Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 dependabot[bot]
|
beaea8ae40 |
No files matched your search
@@ -17,15 +17,11 @@ name: ci
|
||||
#
|
||||
# release.yml calls this workflow (workflow_call) before it builds anything, so a tag passes
|
||||
# exactly these gates and there is one list of them.
|
||||
#
|
||||
# workflow_dispatch reruns the suite on a commit whose push produced no run, such as one
|
||||
# pushed while Actions was unavailable.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
workflow_call:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -91,7 +91,7 @@ jobs:
|
||||
- name: Free disk space
|
||||
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: e2e-assets
|
||||
path: dist
|
||||
@@ -223,7 +223,7 @@ jobs:
|
||||
- name: Free disk space
|
||||
run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
|
||||
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: e2e-assets
|
||||
path: dist
|
||||
|
||||
@@ -105,7 +105,7 @@ jobs:
|
||||
id-token: write # the Sigstore certificate behind the provenance attestation
|
||||
attestations: write
|
||||
steps:
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-assets
|
||||
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
# Felis
|
||||
|
||||
**此项目仍处于早期开发阶段，您不该在任何生产环境使用该项目。若产生任何问题，贵用户的使用行为与 FelisMC 团队无任何民事刑事法律关系。**
|
||||
**THIS PROJECT IS STILL WIP, YOU SHOULD DO NOT USE THIS PROJECT IN ANY PRODUCTION USAGE. WE ARE NOT RESPOND FOR ANY LEGAL OR HUMANLY PROBLEM.**
|
||||
|
||||
一款 Kubernetes 驱动的 Minecraft 服务器托管平台，一行命令部署，自动管理生命周期与安全。
|
||||
A Kubernetes-driven Minecraft server hosting platform — one command to deploy, automatic lifecycle, backup, and security.
|
||||
|
||||
|
||||
@@ -491,12 +491,6 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
|
||||
go pruner.Loop(ctx, registryPruneInterval)
|
||||
}
|
||||
go reapRejectedContexts(ctx, submissions, stderr)
|
||||
if fileStage != nil {
|
||||
go expireFileSessions(ctx, fileStage, fileSessionSweep, stderr)
|
||||
}
|
||||
if exporter != nil {
|
||||
go expireExports(ctx, a, exportSweep)
|
||||
}
|
||||
go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())
|
||||
|
||||
servers := []*http.Server{internalSrv, externalSrv}
|
||||
@@ -803,51 +797,6 @@ func scheduleBackups(ctx context.Context, s *api.BackupScheduler, stderr io.Writ
|
||||
}
|
||||
}
|
||||
|
||||
// fileSessionSweep is how often expireFileSessions looks for idle upload
|
||||
// sessions: small beside fileedit.SessionIdle, so an abandoned one gives its
|
||||
// room back within minutes of going stale.
|
||||
const fileSessionSweep = 10 * time.Minute
|
||||
|
||||
// expireFileSessions drops the file manager's upload sessions left untouched
|
||||
// for fileedit.SessionIdle. Each reserved room on the staging disk for its whole
|
||||
// file when it began, so one abandoned would otherwise hold that room until
|
||||
// felis-api restarts.
|
||||
func expireFileSessions(ctx context.Context, s *fileedit.Stage, every time.Duration, stderr io.Writer) {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if n := s.Expire(); n > 0 {
|
||||
fmt.Fprintf(stderr, "felis api: dropped %d upload session(s) left idle for %s\n", n, fileedit.SessionIdle)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// exportSweep is how often expireExports runs: an export whose Job never
|
||||
// connected is stopped within a minute of going stale.
|
||||
const exportSweep = time.Minute
|
||||
|
||||
// expireExports runs the export sweep (api.API.ExpireExports) on a ticker. The
|
||||
// export routes sweep as they are called, and an owner who closed the tab calls
|
||||
// none; a Job whose Pod never got going would then keep the server from
|
||||
// starting until the Job's deadline.
|
||||
func expireExports(ctx context.Context, a interface{ ExpireExports() }, every time.Duration) {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
a.ExpireExports()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reapRejectedContexts deletes, once an hour, the uploaded contexts of
|
||||
// submissions rejected more than submit.RejectedContextRetention ago, and the
|
||||
// chunked uploads left untouched for submit.StalePartRetention. Without it a
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -9,7 +8,6 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -17,7 +15,6 @@ import (
|
||||
"felis.lolicon.best/internal/api"
|
||||
"felis.lolicon.best/internal/build"
|
||||
"felis.lolicon.best/internal/config"
|
||||
"felis.lolicon.best/internal/fileedit"
|
||||
)
|
||||
|
||||
// The passkey relying party follows the panel host the SPA is served on: an install
|
||||
@@ -238,76 +235,3 @@ func TestInUseImageRefsCoversEverySource(t *testing.T) {
|
||||
t.Fatal("a failing whitelist read produced a reference list")
|
||||
}
|
||||
}
|
||||
|
||||
// TestExpireFileSessions runs the loop against a stage whose clock the test
|
||||
// holds: the session idle past fileedit.SessionIdle goes, the one touched since
|
||||
// stays, and the drop is said once.
|
||||
func TestExpireFileSessions(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
now := time.Date(2026, 9, 28, 10, 0, 0, 0, time.UTC)
|
||||
advance := func(d time.Duration) { mu.Lock(); now = now.Add(d); mu.Unlock() }
|
||||
st := &fileedit.Stage{Dir: t.TempDir(), MinFree: 1e-9, Now: func() time.Time {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return now
|
||||
}}
|
||||
idle, err := st.Begin("u1", "survival", "a.jar", 3)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advance(fileedit.SessionIdle - time.Minute)
|
||||
fresh, err := st.Begin("u1", "survival", "b.jar", 3)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advance(2 * time.Minute)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var out bytes.Buffer
|
||||
done := make(chan struct{})
|
||||
go func() { expireFileSessions(ctx, st, time.Millisecond, &out); close(done) }()
|
||||
for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(time.Millisecond) {
|
||||
if _, err := st.Status("u1", "survival", idle.ID); errors.Is(err, fileedit.ErrNotStaged) {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
cancel()
|
||||
t.Fatal("the idle session was never dropped")
|
||||
}
|
||||
}
|
||||
// A few more ticks with nothing idle, which must stay quiet.
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
<-done
|
||||
if _, err := st.Status("u1", "survival", fresh.ID); err != nil {
|
||||
t.Fatalf("the session touched since went too: %v", err)
|
||||
}
|
||||
if got := out.String(); got != "felis api: dropped 1 upload session(s) left idle for 6h0m0s\n" {
|
||||
t.Fatalf("said %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
type sweepCount struct{ n atomic.Int32 }
|
||||
|
||||
func (s *sweepCount) ExpireExports() { s.n.Add(1) }
|
||||
|
||||
// TestExpireExports: the loop sweeps on each tick, and returns once felis-api
|
||||
// shuts down.
|
||||
func TestExpireExports(t *testing.T) {
|
||||
var s sweepCount
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { expireExports(ctx, &s, time.Millisecond); close(done) }()
|
||||
for deadline := time.Now().Add(5 * time.Second); s.n.Load() < 3; time.Sleep(time.Millisecond) {
|
||||
if time.Now().After(deadline) {
|
||||
cancel()
|
||||
t.Fatalf("swept %d times in 5s at a 1ms tick", s.n.Load())
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the loop outlived its context")
|
||||
}
|
||||
}
|
||||
@@ -551,10 +551,7 @@ func TestPreMigrateBackupExportsServers(t *testing.T) {
|
||||
func TestServerExportStopsWaitingWithTheContext(t *testing.T) {
|
||||
dir := newPodRig(t)
|
||||
writeTestFile(t, filepath.Join(dir, "servers_fail"), "99", 0o600)
|
||||
// Long enough for the first try to run to its refusal: starting the fake
|
||||
// k3s on a busy machine can take a few hundred ms. Still far below the
|
||||
// 10s retry wait, so waiting it out would fail the check below.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
_, err := exportMinecraftServers(ctx)
|
||||
|
||||
+33
-182
@@ -2,35 +2,26 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/backup"
|
||||
"felis.lolicon.best/internal/fileedit"
|
||||
"felis.lolicon.best/internal/worldexport"
|
||||
)
|
||||
|
||||
// cmdExport is the in-Pod entrypoint the export Job runs. internal/worldexport
|
||||
// renders a Pod whose command is `/usr/local/bin/felis export`. It archives the
|
||||
// mounted world, re-streams one archive from the mounted backup store, or sends
|
||||
// one file or folder of the world, PUTs it to felis-api's internal face, and
|
||||
// exits once felis-api says the owner's browser got all of it. It is NOT a
|
||||
// user-facing command and is never invoked by hand.
|
||||
// mounted world (or opens one archive on the mounted backup store), PUTs the
|
||||
// tar.gz to felis-api's internal face, and exits once felis-api says the
|
||||
// owner's browser got all of it. It is NOT a user-facing command and is never
|
||||
// invoked by hand.
|
||||
//
|
||||
// Like cmdRestore it holds no database credentials and never calls config.Load:
|
||||
// felis-api made every decision (who may download what, that the server is
|
||||
@@ -38,29 +29,19 @@ import (
|
||||
// plus the one-time upload token in the environment, which opens this one
|
||||
// export and nothing else.
|
||||
//
|
||||
// Whatever leaves goes through the same guards as the file editor
|
||||
// (fileedit.Guard): the proxy forwarding secret, which every server on the
|
||||
// install shares, never leaves, and server.properties leaves with its RCON
|
||||
// password redacted. A backup is stored with both, since a restore must bring
|
||||
// the world back whole, so it is filtered on the way out rather than handed
|
||||
// over as stored.
|
||||
//
|
||||
// Exit status: 0 once felis-api answers 204 (the download completed), 1 when
|
||||
// the export could not be read or handed over, a backup failed its digest
|
||||
// check, or felis-api refused it (the browser never came or left early), 2 on
|
||||
// the archive could not be read or handed over, or felis-api refused it (the
|
||||
// browser never came, left early, or the backup failed its digest check), 2 on
|
||||
// bad flags. The last stderr line reaches the export's status and the jobs list.
|
||||
func cmdExport(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("export", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
mode := fs.String("mode", "", "what to export: world, backup or files")
|
||||
mode := fs.String("mode", "", "what to export: world or backup")
|
||||
server := fs.String("server", "", "server name being exported (for logging)")
|
||||
target := fs.String("target-url", "", "felis-api URL to PUT the export to")
|
||||
target := fs.String("target-url", "", "felis-api URL to PUT the archive to")
|
||||
ref := fs.String("ref", "", "backup only: absolute path to the archive on the backup mount")
|
||||
backupRoot := fs.String("backup-root", "/backups", "backup only: mount path of the backup PVC (the ref must resolve under it)")
|
||||
sum := fs.String("sha256", "", "backup only: the sha256 recorded when the archive was written; a mismatch fails the export before its end is sent")
|
||||
worldsRoot := fs.String("worlds-root", "/world", "world and files: mount path of the world PVC")
|
||||
path := fs.String("path", "", "files only: the file or folder to send, relative to the world root")
|
||||
dir := fs.Bool("dir", false, "files only: the path is a folder, sent as a zip")
|
||||
worldsRoot := fs.String("worlds-root", "/world", "world only: mount path of the world PVC to archive")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
@@ -69,7 +50,6 @@ func cmdExport(args []string, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintf(stderr, "felis export: --target-url and %s are required\n", worldexport.TokenEnv)
|
||||
return 2
|
||||
}
|
||||
limitHeapToCgroup()
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
@@ -80,17 +60,11 @@ func cmdExport(args []string, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintln(stderr, "felis export: --ref is required for a backup")
|
||||
return 2
|
||||
}
|
||||
err = exportBackup(ctx, *target, token, *ref, *backupRoot, *sum, stdout)
|
||||
err = exportBackup(ctx, *target, token, *ref, *backupRoot)
|
||||
case worldexport.ModeWorld:
|
||||
err = exportWorld(ctx, *target, token, *worldsRoot, stdout)
|
||||
case worldexport.ModeFiles:
|
||||
if *path == "" {
|
||||
fmt.Fprintln(stderr, "felis export: --path is required for files")
|
||||
return 2
|
||||
}
|
||||
err = exportFiles(ctx, *target, token, *worldsRoot, *path, *dir, stdout)
|
||||
default:
|
||||
fmt.Fprintf(stderr, "felis export: --mode must be %s, %s or %s\n", worldexport.ModeWorld, worldexport.ModeBackup, worldexport.ModeFiles)
|
||||
fmt.Fprintf(stderr, "felis export: --mode must be %s or %s\n", worldexport.ModeWorld, worldexport.ModeBackup)
|
||||
return 2
|
||||
}
|
||||
if err != nil {
|
||||
@@ -101,20 +75,9 @@ func cmdExport(args []string, stdout, stderr io.Writer) int {
|
||||
return 0
|
||||
}
|
||||
|
||||
// archiveType is the content type of a world or backup export.
|
||||
const archiveType = "application/gzip"
|
||||
|
||||
// errBackupDigest fails a backup export whose stored archive no longer hashes
|
||||
// to what was recorded when it was written.
|
||||
var errBackupDigest = errors.New("the backup archive does not match the sha256 recorded when it was written")
|
||||
|
||||
// exportBackup re-streams one stored archive through the export guards
|
||||
// (backup.FilterTarGz with archiveFilter). Its length changes on the way, so it
|
||||
// goes chunked. With want set, the stored bytes are hashed as they are read,
|
||||
// and FilterTarGz reads them to their end before it closes its own archive: a
|
||||
// mismatch aborts the upload while what felis-api has passed on still lacks
|
||||
// its end, so the browser never keeps a complete-looking corrupt file.
|
||||
func exportBackup(ctx context.Context, target, token, ref, root, want string, stdout io.Writer) error {
|
||||
// exportBackup hands over one stored archive as it is, with its length, so the
|
||||
// browser shows real progress and felis-api can check its recorded digest.
|
||||
func exportBackup(ctx context.Context, target, token, ref, root string) error {
|
||||
// Defense in depth, as in cmdRestore: the ref comes from felis-api, but this
|
||||
// process opens it, so it confirms the ref stays on the backup mount.
|
||||
if !refWithinRoot(ref, root) {
|
||||
@@ -125,159 +88,47 @@ func exportBackup(ctx context.Context, target, token, ref, root, want string, st
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
var src io.Reader = f
|
||||
if want != "" {
|
||||
src = &digestReader{r: f, sum: sha256.New(), want: want}
|
||||
}
|
||||
var withheld []string
|
||||
err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
|
||||
var err error
|
||||
withheld, err = backup.FilterTarGz(ctx, w, src, archiveFilter)
|
||||
st, err := f.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
})
|
||||
if errors.Is(err, errBackupDigest) {
|
||||
return errBackupDigest // the jobs list shows it as it is, not wrapped as a read error
|
||||
}
|
||||
reportWithheld(stdout, len(withheld))
|
||||
return err
|
||||
return putExport(ctx, target, token, f, st.Size())
|
||||
}
|
||||
|
||||
// exportWorld archives the world straight into the request body: nothing is
|
||||
// staged, so a world bigger than the Pod's memory or any scratch disk exports
|
||||
// the same.
|
||||
// the same. A read error mid-way aborts the chunked body, and felis-api cuts
|
||||
// the browser's download off rather than end it.
|
||||
func exportWorld(ctx context.Context, target, token, root string, stdout io.Writer) error {
|
||||
r, err := os.OpenRoot(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
guard := fileedit.NewGuard(r)
|
||||
r.Close()
|
||||
var skipped, withheld []string
|
||||
err = streamExport(ctx, target, token, archiveType, -1, func(w io.Writer) error {
|
||||
var err error
|
||||
skipped, withheld, err = backup.WriteTarGz(ctx, w, root, worldFilter(guard))
|
||||
return err
|
||||
})
|
||||
if len(skipped) > 0 {
|
||||
pr, pw := io.Pipe()
|
||||
skippedc := make(chan []string, 1)
|
||||
go func() {
|
||||
skipped, err := backup.WriteTarGz(ctx, pw, root)
|
||||
pw.CloseWithError(err)
|
||||
skippedc <- skipped
|
||||
}()
|
||||
err := putExport(ctx, target, token, pr, -1)
|
||||
pr.CloseWithError(io.ErrClosedPipe) // stops the archiver if the PUT ended first
|
||||
if skipped := <-skippedc; len(skipped) > 0 {
|
||||
fmt.Fprintf(stdout, "felis export: left out %d entries a tar cannot hold (symbolic links, devices, sockets)\n", len(skipped))
|
||||
}
|
||||
reportWithheld(stdout, len(withheld))
|
||||
return err
|
||||
}
|
||||
|
||||
// exportFiles sends one file or folder of the world (fileedit.OpenDownload): a
|
||||
// file with its exact length, a folder as a zip made as it streams. dir is
|
||||
// what the owner saw at path when they asked.
|
||||
func exportFiles(ctx context.Context, target, token, root, path string, dir bool, stdout io.Writer) error {
|
||||
d, err := fileedit.OpenDownload(root, path, dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer d.Close()
|
||||
err = streamExport(ctx, target, token, d.ContentType, d.Size, func(w io.Writer) error { return d.WriteTo(ctx, w) })
|
||||
if d.Skipped > 0 {
|
||||
fmt.Fprintf(stdout, "felis export: left out %d entries a zip does not carry (symbolic links, devices, sockets)\n", d.Skipped)
|
||||
}
|
||||
reportWithheld(stdout, d.Withheld)
|
||||
return err
|
||||
}
|
||||
|
||||
func reportWithheld(stdout io.Writer, n int) {
|
||||
if n > 0 {
|
||||
fmt.Fprintf(stdout, "felis export: left out %d files that hold platform secrets\n", n)
|
||||
}
|
||||
}
|
||||
|
||||
// worldFilter guards a live world by file identity, so a link to a guarded
|
||||
// file under another name is caught as well.
|
||||
func worldFilter(g fileedit.Guard) backup.Filter {
|
||||
return func(_ string, info fs.FileInfo) (bool, func([]byte) []byte) {
|
||||
return guardAction(g.Rule(info))
|
||||
}
|
||||
}
|
||||
|
||||
// archiveFilter guards a stored archive, which has only names.
|
||||
func archiveFilter(name string, _ fs.FileInfo) (bool, func([]byte) []byte) {
|
||||
return guardAction(fileedit.ArchiveRule(name))
|
||||
}
|
||||
|
||||
func guardAction(withhold, redact bool) (bool, func([]byte) []byte) {
|
||||
if redact {
|
||||
return withhold, fileedit.RedactProps
|
||||
}
|
||||
return withhold, nil
|
||||
}
|
||||
|
||||
// digestReader passes r through, hashing it, and turns r's EOF into
|
||||
// errBackupDigest when the bytes do not hash to want.
|
||||
type digestReader struct {
|
||||
r io.Reader
|
||||
sum hash.Hash
|
||||
want string
|
||||
}
|
||||
|
||||
func (d *digestReader) Read(p []byte) (int, error) {
|
||||
n, err := d.r.Read(p)
|
||||
d.sum.Write(p[:n])
|
||||
if err == io.EOF && !strings.EqualFold(hex.EncodeToString(d.sum.Sum(nil)), d.want) {
|
||||
return n, errBackupDigest
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// streamExport runs write straight into the body of the PUT, hashing it as it
|
||||
// goes. Once write has finished, the SHA-256 of all it wrote rides the
|
||||
// request's trailer (worldexport.DigestTrailer), and felis-api holds back the
|
||||
// last bytes from the browser until what it received hashes the same. An error
|
||||
// from write aborts the chunked body before the trailer, and felis-api then
|
||||
// cuts the browser's download off rather than end it; that error is the one
|
||||
// reported, since the PUT's own error only wraps it. When the PUT ends first,
|
||||
// write is stopped.
|
||||
func streamExport(ctx context.Context, target, token, contentType string, size int64, write func(io.Writer) error) error {
|
||||
pr, pw := io.Pipe()
|
||||
trailer := http.Header{worldexport.DigestTrailer: nil}
|
||||
werr := make(chan error, 1)
|
||||
go func() {
|
||||
sum := sha256.New()
|
||||
err := write(io.MultiWriter(pw, sum))
|
||||
if err == nil {
|
||||
// Set before the body ends: the transport reads the trailer once it
|
||||
// has read the body to its end.
|
||||
trailer.Set(worldexport.DigestTrailer, "sha-256=:"+base64.StdEncoding.EncodeToString(sum.Sum(nil))+":")
|
||||
}
|
||||
pw.CloseWithError(err)
|
||||
werr <- err
|
||||
}()
|
||||
err := putExport(ctx, target, token, contentType, pr, size, trailer)
|
||||
pr.CloseWithError(io.ErrClosedPipe)
|
||||
if w := <-werr; w != nil && !errors.Is(w, io.ErrClosedPipe) {
|
||||
return w
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// putExport PUTs the export to felis-api. There is no retry: the token opens
|
||||
// putExport PUTs the archive to felis-api. There is no retry: the token opens
|
||||
// the export once, so a second attempt could only be refused. Redirects are
|
||||
// refused because the request carries the token and the internal face never
|
||||
// redirects. felis-api answers only after the whole download, which the Job's
|
||||
// activeDeadlineSeconds bounds, so the header timeout is a backstop for a
|
||||
// wedged endpoint and not the real limit.
|
||||
//
|
||||
// The body always goes chunked, which is what lets it end with a trailer; a
|
||||
// size the Job knows (-1 when it does not) goes as worldexport.LengthHeader in
|
||||
// place of Content-Length.
|
||||
func putExport(ctx context.Context, target, token, contentType string, body io.Reader, size int64, trailer http.Header) error {
|
||||
func putExport(ctx context.Context, target, token string, body io.Reader, size int64) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.ContentLength = -1
|
||||
req.Trailer = trailer
|
||||
if size >= 0 {
|
||||
req.Header.Set(worldexport.LengthHeader, strconv.FormatInt(size, 10))
|
||||
}
|
||||
req.ContentLength = size
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
req.Header.Set("Content-Type", "application/gzip")
|
||||
client := &http.Client{
|
||||
Transport: &http.Transport{ResponseHeaderTimeout: 10 * time.Minute},
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
|
||||
+31
-320
@@ -2,22 +2,13 @@ package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -55,25 +46,6 @@ func receiveExport(t *testing.T, reply func(w http.ResponseWriter)) *exportRecei
|
||||
|
||||
func noContent(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) }
|
||||
|
||||
// sentWhole fails unless the upload rcv got ended with the Content-Digest
|
||||
// trailer of its own bytes, and declared length as its size (-1: none).
|
||||
func sentWhole(t *testing.T, rcv *exportReceiver, length int64) {
|
||||
t.Helper()
|
||||
sum := sha256.Sum256(rcv.body)
|
||||
want := "sha-256=:" + base64.StdEncoding.EncodeToString(sum[:]) + ":"
|
||||
wantLength := ""
|
||||
if length >= 0 {
|
||||
wantLength = strconv.FormatInt(length, 10)
|
||||
}
|
||||
r := rcv.req
|
||||
if rcv.readErr != nil || r.Trailer.Get(worldexport.DigestTrailer) != want || r.Header.Get(worldexport.LengthHeader) != wantLength ||
|
||||
r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
|
||||
t.Fatalf("upload read %v, trailer %v, %s %q, length %d, encoding %v; want trailer %q and %s %q, chunked",
|
||||
rcv.readErr, r.Trailer, worldexport.LengthHeader, r.Header.Get(worldexport.LengthHeader), r.ContentLength, r.TransferEncoding,
|
||||
want, worldexport.LengthHeader, wantLength)
|
||||
}
|
||||
}
|
||||
|
||||
func tarEntries(t *testing.T, archive []byte) map[string]string {
|
||||
t.Helper()
|
||||
gz, err := gzip.NewReader(bytes.NewReader(archive))
|
||||
@@ -95,52 +67,16 @@ func tarEntries(t *testing.T, archive []byte) map[string]string {
|
||||
}
|
||||
}
|
||||
|
||||
// writeTree writes name → body under root, making the folders on the way.
|
||||
func writeTree(t *testing.T, root string, files map[string]string) {
|
||||
t.Helper()
|
||||
for name, body := range files {
|
||||
p := filepath.Join(root, name)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The two secrets a world holds, and what server.properties reads as once
|
||||
// redacted.
|
||||
const (
|
||||
secretProps = "motd=hi\nrcon.password=hunter2\n"
|
||||
redactedProps = "motd=hi\nrcon.password=<redacted by felis>\n"
|
||||
forwardingKey = "secret: aVeryRealForwardingKey\n"
|
||||
)
|
||||
|
||||
// secretWorld is a world holding both secrets, with a hard link to the
|
||||
// forwarding secret under a name nothing would guard by.
|
||||
func secretWorld(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
writeTree(t, root, map[string]string{
|
||||
"server.properties": secretProps,
|
||||
"config/paper-global.yml": forwardingKey,
|
||||
"world/region/r.0.0.mca": "chunks",
|
||||
})
|
||||
if err := os.MkdirAll(filepath.Join(root, "plugins"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Link(filepath.Join(root, "config/paper-global.yml"), filepath.Join(root, "plugins/copy.yml")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func TestCmdExportWorld(t *testing.T) {
|
||||
root := secretWorld(t)
|
||||
if err := os.Symlink("server.properties", filepath.Join(root, "props-link")); err != nil {
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "world", "region"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, body := range map[string]string{"server.properties": "motd=hi\n", "world/region/r.0.0.mca": "chunks"} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
rcv := receiveExport(t, noContent)
|
||||
t.Setenv(worldexport.TokenEnv, "tok")
|
||||
var stdout, stderr bytes.Buffer
|
||||
@@ -154,204 +90,61 @@ func TestCmdExportWorld(t *testing.T) {
|
||||
r.Header.Get("Content-Type") != "application/gzip" || r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
|
||||
t.Fatalf("request = %s %s, headers %v, length %d, encoding %v", r.Method, r.URL.Path, r.Header, r.ContentLength, r.TransferEncoding)
|
||||
}
|
||||
want := map[string]string{
|
||||
"server.properties": redactedProps, "config/": "", "plugins/": "",
|
||||
"world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks",
|
||||
got := tarEntries(t, rcv.body)
|
||||
want := map[string]string{"server.properties": "motd=hi\n", "world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("archive holds %v, want %v", got, want)
|
||||
}
|
||||
if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("archive holds %v\nwant %v", got, want)
|
||||
for name, body := range want {
|
||||
if b, ok := got[name]; !ok || b != body {
|
||||
t.Errorf("%s = %q (present %v), want %q", name, b, ok, body)
|
||||
}
|
||||
sentWhole(t, rcv, -1)
|
||||
want2 := "felis export: left out 1 entries a tar cannot hold (symbolic links, devices, sockets)\n" +
|
||||
"felis export: left out 2 files that hold platform secrets\n" +
|
||||
"felis export: server=survival mode=world downloaded\n"
|
||||
if stdout.String() != want2 {
|
||||
t.Errorf("stdout = %q, want %q", stdout.String(), want2)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "server=survival mode=world downloaded") {
|
||||
t.Errorf("stdout = %q", stdout.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A world root that cannot be opened fails before anything reaches felis-api.
|
||||
func TestCmdExportWorldUnreadable(t *testing.T) {
|
||||
// A world that cannot be read must never reach felis-api as a complete body:
|
||||
// the chunked upload is cut off, so felis-api aborts the browser's download.
|
||||
func TestCmdExportWorldReadErrorAbortsTheUpload(t *testing.T) {
|
||||
rcv := receiveExport(t, noContent)
|
||||
t.Setenv(worldexport.TokenEnv, "tok")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := cmdExport([]string{"--mode", "world", "--target-url", rcv.srv.URL, "--worlds-root", filepath.Join(t.TempDir(), "missing")}, &stdout, &stderr)
|
||||
if code != 1 || rcv.hits.Load() != 0 {
|
||||
t.Fatalf("exit %d with %d requests, want 1 and none", code, rcv.hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// An export that fails part-way must never reach felis-api as a complete body:
|
||||
// the chunked upload is cut off, so felis-api aborts the browser's download,
|
||||
// and the failure itself is what the Job reports.
|
||||
func TestStreamExportWriteErrorAbortsTheUpload(t *testing.T) {
|
||||
broken := errors.New("disk read failed")
|
||||
rcv := receiveExport(t, noContent)
|
||||
err := streamExport(context.Background(), rcv.srv.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
|
||||
if _, err := w.Write(bytes.Repeat([]byte("x"), 100_000)); err != nil {
|
||||
return err
|
||||
}
|
||||
return broken
|
||||
})
|
||||
if err != broken {
|
||||
t.Fatalf("err = %v, want the write's own error, unwrapped", err)
|
||||
if code != 1 {
|
||||
t.Fatalf("exit %d, want 1", code)
|
||||
}
|
||||
select {
|
||||
case <-rcv.served:
|
||||
if rcv.readErr == nil {
|
||||
t.Fatalf("felis-api read a complete %d-byte body from a failed export", len(rcv.body))
|
||||
t.Fatalf("felis-api read a complete %d-byte body from an unreadable world", len(rcv.body))
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the request never reached felis-api")
|
||||
case <-time.After(2 * time.Second): // the request never reached the handler
|
||||
}
|
||||
}
|
||||
|
||||
// When felis-api refuses first, its reason is reported, not the closed pipe
|
||||
// that then stops the writer.
|
||||
func TestStreamExportRefusalStopsTheWriter(t *testing.T) {
|
||||
refusing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer refusing.Close()
|
||||
stopped := make(chan error, 1)
|
||||
err := streamExport(context.Background(), refusing.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
|
||||
for {
|
||||
if _, err := w.Write(make([]byte, 32<<10)); err != nil {
|
||||
stopped <- err
|
||||
return err
|
||||
}
|
||||
}
|
||||
})
|
||||
if err == nil || err.Error() != "felis-api answered 404 Not Found" {
|
||||
t.Fatalf("err = %v, want felis-api's answer", err)
|
||||
}
|
||||
if werr := <-stopped; !errors.Is(werr, io.ErrClosedPipe) {
|
||||
t.Fatalf("the writer stopped on %v, want the closed pipe", werr)
|
||||
}
|
||||
|
||||
// A PUT that never starts leaves no transport to close the body: the writer
|
||||
// is still stopped, and the export fails rather than hangs.
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- streamExport(context.Background(), "http://[::1", "tok", "application/gzip", -1, func(w io.Writer) error {
|
||||
_, err := w.Write([]byte("x"))
|
||||
return err
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil || !strings.Contains(err.Error(), "missing ']'") {
|
||||
t.Fatalf("err = %v, want the bad URL", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("an export whose PUT never started hung")
|
||||
}
|
||||
}
|
||||
|
||||
// storedBackup writes, at path, a gzip+tar like one the backup store holds:
|
||||
// the world whole, both secrets included, and a region file that does not
|
||||
// compress. It returns the archive's sha256.
|
||||
func storedBackup(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
region := make([]byte, 64<<10)
|
||||
if _, err := rand.Read(region); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
for _, e := range []struct{ name, body string }{
|
||||
{"server.properties", secretProps},
|
||||
{"config/paper-global.yml", forwardingKey},
|
||||
{"world/level.dat", "level"},
|
||||
{"world/region/r.0.0.mca", string(region)},
|
||||
} {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: e.name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(e.body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := io.WriteString(tw, e.body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(buf.Bytes())
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func TestCmdExportBackup(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
archive := bytes.Repeat([]byte("felis"), 10_000)
|
||||
ref := filepath.Join(root, "survival-1.tar.gz")
|
||||
sum := storedBackup(t, ref)
|
||||
stored, err := os.ReadFile(ref)
|
||||
if err != nil {
|
||||
if err := os.WriteFile(ref, archive, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
region := tarEntries(t, stored)["world/region/r.0.0.mca"]
|
||||
args := func(url, ref string) []string {
|
||||
return []string{"--mode", "backup", "--server", "survival", "--target-url", url, "--ref", ref, "--backup-root", root}
|
||||
}
|
||||
|
||||
for name, extra := range map[string][]string{
|
||||
"no digest recorded": nil,
|
||||
"recorded digest matches": {"--sha256", sum},
|
||||
} {
|
||||
t.Run(name+": re-streamed through the guards", func(t *testing.T) {
|
||||
t.Run("hands the archive over with its length", func(t *testing.T) {
|
||||
rcv := receiveExport(t, noContent)
|
||||
t.Setenv(worldexport.TokenEnv, "tok")
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := cmdExport(append(args(rcv.srv.URL, ref), extra...), &stdout, &stderr); code != 0 {
|
||||
if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("exit %d, stderr %q", code, stderr.String())
|
||||
}
|
||||
r := rcv.req
|
||||
if r.ContentLength != -1 || r.Header.Get("Content-Type") != "application/gzip" || r.Header.Get("Authorization") != "Bearer tok" {
|
||||
t.Fatalf("length %d, headers %v", r.ContentLength, r.Header)
|
||||
}
|
||||
want := map[string]string{"server.properties": redactedProps, "world/level.dat": "level", "world/region/r.0.0.mca": region}
|
||||
if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("archive holds %d entries, want exactly the redacted properties, level.dat and the region file", len(got))
|
||||
}
|
||||
sentWhole(t, rcv, -1)
|
||||
if want := "felis export: left out 1 files that hold platform secrets\nfelis export: server=survival mode=backup downloaded\n"; stdout.String() != want {
|
||||
t.Errorf("stdout = %q, want %q", stdout.String(), want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The stored bytes are checked as they stream, and the archive the Job sends
|
||||
// is only closed once they are all read: a mismatch cuts the upload off
|
||||
// short of its end, so felis-api never passes on a complete-looking copy.
|
||||
t.Run("a digest mismatch cuts the upload off before its end", func(t *testing.T) {
|
||||
rcv := receiveExport(t, noContent)
|
||||
t.Setenv(worldexport.TokenEnv, "tok")
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := cmdExport(append(args(rcv.srv.URL, ref), "--sha256", strings.Repeat("ab", 32)), &stdout, &stderr); code != 1 {
|
||||
t.Fatalf("exit %d, want 1", code)
|
||||
}
|
||||
if want := "felis export: the backup archive does not match the sha256 recorded when it was written\n"; stderr.String() != want {
|
||||
t.Fatalf("stderr = %q, want %q", stderr.String(), want)
|
||||
}
|
||||
select {
|
||||
case <-rcv.served:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the upload never reached felis-api")
|
||||
}
|
||||
if rcv.readErr == nil {
|
||||
t.Fatalf("felis-api read a complete %d-byte body", len(rcv.body))
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(rcv.body))
|
||||
if err == nil {
|
||||
_, err = io.ReadAll(zr)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("what felis-api got is a complete archive")
|
||||
if rcv.req.ContentLength != int64(len(archive)) || !bytes.Equal(rcv.body, archive) || rcv.req.Header.Get("Authorization") != "Bearer tok" {
|
||||
t.Fatalf("got %d bytes (length %d, auth %q), want the %d archive bytes",
|
||||
len(rcv.body), rcv.req.ContentLength, rcv.req.Header.Get("Authorization"), len(archive))
|
||||
}
|
||||
})
|
||||
|
||||
@@ -411,87 +204,6 @@ func TestCmdExportBackup(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCmdExportFiles(t *testing.T) {
|
||||
root := secretWorld(t)
|
||||
writeTree(t, root, map[string]string{"plugins/Essentials/config.yml": "x: 1"})
|
||||
if err := os.Symlink("config.yml", filepath.Join(root, "plugins/Essentials/link.yml")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
export := func(t *testing.T, path string, dir bool) (*exportReceiver, int, string, string) {
|
||||
t.Helper()
|
||||
rcv := receiveExport(t, noContent)
|
||||
t.Setenv(worldexport.TokenEnv, "tok")
|
||||
args := []string{"--mode", "files", "--server", "survival", "--target-url", rcv.srv.URL, "--worlds-root", root, "--path", path}
|
||||
if dir {
|
||||
args = append(args, "--dir")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := cmdExport(args, &stdout, &stderr)
|
||||
return rcv, code, stdout.String(), stderr.String()
|
||||
}
|
||||
|
||||
for path, want := range map[string]string{
|
||||
"world/region/r.0.0.mca": "chunks",
|
||||
"server.properties": redactedProps,
|
||||
} {
|
||||
t.Run("a file goes with its exact length: "+path, func(t *testing.T) {
|
||||
rcv, code, stdout, stderr := export(t, path, false)
|
||||
if code != 0 || stdout != "felis export: server=survival mode=files downloaded\n" {
|
||||
t.Fatalf("exit %d, stdout %q, stderr %q", code, stdout, stderr)
|
||||
}
|
||||
if string(rcv.body) != want || rcv.req.Header.Get("Content-Type") != "application/octet-stream" {
|
||||
t.Fatalf("body %q, type %q; want %q", rcv.body, rcv.req.Header.Get("Content-Type"), want)
|
||||
}
|
||||
sentWhole(t, rcv, int64(len(want)))
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a folder goes as a zip, guarded", func(t *testing.T) {
|
||||
rcv, code, stdout, stderr := export(t, "plugins", true)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit %d, stderr %q", code, stderr)
|
||||
}
|
||||
if rcv.req.ContentLength != -1 || rcv.req.Header.Get("Content-Type") != "application/zip" {
|
||||
t.Fatalf("length %d, type %q", rcv.req.ContentLength, rcv.req.Header.Get("Content-Type"))
|
||||
}
|
||||
zr, err := zip.NewReader(bytes.NewReader(rcv.body), int64(len(rcv.body)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for _, f := range zr.File {
|
||||
names = append(names, f.Name)
|
||||
}
|
||||
if want := []string{"plugins/", "plugins/Essentials/", "plugins/Essentials/config.yml"}; !slices.Equal(names, want) {
|
||||
t.Fatalf("zip holds %v, want %v", names, want)
|
||||
}
|
||||
want := "felis export: left out 1 entries a zip does not carry (symbolic links, devices, sockets)\n" +
|
||||
"felis export: left out 1 files that hold platform secrets\n" +
|
||||
"felis export: server=survival mode=files downloaded\n"
|
||||
if stdout != want {
|
||||
t.Errorf("stdout = %q, want %q", stdout, want)
|
||||
}
|
||||
})
|
||||
|
||||
for _, c := range []struct {
|
||||
path string
|
||||
dir bool
|
||||
want string
|
||||
}{
|
||||
{"config/paper-global.yml", false, "forwarding secret"},
|
||||
{"plugins/copy.yml", false, "forwarding secret"},
|
||||
{"plugins", false, "is a folder now"},
|
||||
{"server.properties", true, "is not a folder now"},
|
||||
} {
|
||||
t.Run("refused before any request: "+c.path, func(t *testing.T) {
|
||||
rcv, code, _, stderr := export(t, c.path, c.dir)
|
||||
if code != 1 || rcv.hits.Load() != 0 || !strings.Contains(stderr, c.want) {
|
||||
t.Fatalf("exit %d, %d requests, stderr %q; want 1, none, and %q", code, rcv.hits.Load(), stderr, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCmdExportUsage(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
token string
|
||||
@@ -501,7 +213,6 @@ func TestCmdExportUsage(t *testing.T) {
|
||||
"no target": {"tok", []string{"--mode", "world"}},
|
||||
"unknown mode": {"tok", []string{"--mode", "both", "--target-url", "http://api/x"}},
|
||||
"backup without ref": {"tok", []string{"--mode", "backup", "--target-url", "http://api/x"}},
|
||||
"files without path": {"tok", []string{"--mode", "files", "--target-url", "http://api/x"}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Setenv(worldexport.TokenEnv, tc.token)
|
||||
|
||||
+6
-51
@@ -39,7 +39,7 @@ import (
|
||||
func cmdFiles(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("files", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
op := fs.String("op", "", "operation: list, read, write, mkdir, delete, rename, upload or unzip")
|
||||
op := fs.String("op", "", "operation: list, read, write, mkdir, delete, rename or upload")
|
||||
path := fs.String("path", "", "path to operate on, relative to the world root (empty = the root itself)")
|
||||
worldsRoot := fs.String("worlds-root", "/data", "mount path of the world PVC; every path resolves under it")
|
||||
expect := fs.String("expect-sha256", "", "write only: refuse unless the file's current SHA-256 (hex) is this")
|
||||
@@ -47,8 +47,8 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
|
||||
to := fs.String("to", "", "rename only: the destination path")
|
||||
sourceURL := fs.String("source-url", "", "upload only: felis-api URL to fetch the bytes from")
|
||||
size := fs.Int64("size", -1, "upload only: the byte count the fetched file must have")
|
||||
sum := fs.String("sha256", "", "write and upload: the SHA-256 (hex) the content or the fetched file must have")
|
||||
overwrite := fs.Bool("overwrite", false, "upload and unzip: replace files already there")
|
||||
sum := fs.String("sha256", "", "upload only: the SHA-256 (hex) the fetched file must have")
|
||||
overwrite := fs.Bool("overwrite", false, "upload only: replace a file already at the path")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
@@ -57,7 +57,6 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintln(stderr, "felis files: --op is required")
|
||||
return 2
|
||||
}
|
||||
limitHeapToCgroup()
|
||||
req := fileedit.Request{
|
||||
Op: *op, Path: *path, To: *to, Expect: *expect, CreateOnly: *createOnly, Overwrite: *overwrite,
|
||||
}
|
||||
@@ -70,18 +69,12 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
|
||||
// channel that must be a valid string.
|
||||
switch *op {
|
||||
case fileedit.OpWrite:
|
||||
// The content's SHA-256 comes with it, so bytes that changed on the way
|
||||
// to this Job are refused rather than written (Request.ContentSHA256).
|
||||
if *sum == "" {
|
||||
fmt.Fprintln(stderr, "felis files: a write needs --sha256")
|
||||
return 2
|
||||
}
|
||||
content, err := fileedit.ContentFromEnv(os.LookupEnv)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis files: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
req.Content, req.ContentSHA256 = content, *sum
|
||||
req.Content = content
|
||||
case fileedit.OpUpload:
|
||||
token := os.Getenv(fileedit.UploadTokenEnv)
|
||||
if *sourceURL == "" || token == "" {
|
||||
@@ -93,19 +86,8 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
|
||||
req.Upload = &fileedit.Upload{
|
||||
Size: *size, SHA256: *sum,
|
||||
Open: func() (io.ReadCloser, error) { return fetchUpload(ctx, *sourceURL, token) },
|
||||
Landed: func() {
|
||||
if err := reportLanded(ctx, *sourceURL, token); err != nil {
|
||||
// The file is in place; felis-api drops its copy when it
|
||||
// has sat idle long enough, and the panel cancels it too.
|
||||
fmt.Fprintf(stderr, "felis files: tell felis-api the upload landed: %v\n", err)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
// An upload or an unzip (the only ops that report progress) can run long
|
||||
// enough that felis-api does not wait on its Job, and the panel shows how far
|
||||
// it has got from the latest of these lines (fileedit.K8sRunner.Ops).
|
||||
req.Progress = fileedit.ThrottledProgress(stdout, time.Second, time.Now)
|
||||
|
||||
res, err := fileedit.Execute(*worldsRoot, req)
|
||||
if err != nil {
|
||||
@@ -125,9 +107,8 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
|
||||
|
||||
// fetchUpload opens the staged upload on felis-api's internal face. There is no
|
||||
// retry: the token opens the upload once (fileedit.Stage), so a second attempt
|
||||
// could only be refused, and the caller retries the failed Job whole (a file
|
||||
// sent in parts stays staged until its Job reports it landed, so that retry
|
||||
// does not send it again). Redirects are refused because the request carries the
|
||||
// could only be refused, and felis-api answers the failed Job with a 500 the
|
||||
// caller can retry whole. Redirects are refused because the request carries the
|
||||
// token and the internal face never redirects; the header timeout catches a
|
||||
// wedged endpoint, and the Job's activeDeadlineSeconds bounds the body.
|
||||
func fetchUpload(ctx context.Context, url, token string) (io.ReadCloser, error) {
|
||||
@@ -150,29 +131,3 @@ func fetchUpload(ctx context.Context, url, token string) (io.ReadCloser, error)
|
||||
}
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// reportLanded tells felis-api the upload's file is in place (DELETE on the URL
|
||||
// it was fetched from, with the same token), so it deletes the copy it staged.
|
||||
// One try: the file has landed whatever the answer, and a copy nobody deletes
|
||||
// is dropped once it has sat idle for fileedit.SessionIdle.
|
||||
func reportLanded(ctx context.Context, url, token string) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
client := &http.Client{
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
return fmt.Errorf("DELETE returned %s", resp.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+12
-167
@@ -1,14 +1,11 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"cmp"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
@@ -20,12 +17,10 @@ import (
|
||||
"felis.lolicon.best/internal/fileedit"
|
||||
)
|
||||
|
||||
// filesResult is the Result a `felis files` run printed on its marked line,
|
||||
// the last it prints.
|
||||
// filesResult is the Result a `felis files` run printed on its marked line.
|
||||
func filesResult(t *testing.T, stdout string) fileedit.Result {
|
||||
t.Helper()
|
||||
lines := strings.Split(strings.TrimSpace(stdout), "\n")
|
||||
line, ok := strings.CutPrefix(lines[len(lines)-1], fileedit.ResultPrefix)
|
||||
line, ok := strings.CutPrefix(strings.TrimSpace(stdout), fileedit.ResultPrefix)
|
||||
if !ok {
|
||||
t.Fatalf("stdout has no result line: %q", stdout)
|
||||
}
|
||||
@@ -36,43 +31,19 @@ func filesResult(t *testing.T, stdout string) fileedit.Result {
|
||||
return res
|
||||
}
|
||||
|
||||
// stagedSource is felis-api's internal face for one staged upload. It serves
|
||||
// body to a GET carrying Bearer token and 404 to any other, and answers the
|
||||
// DELETE that reports the file landed with landedCode (204 when unset),
|
||||
// redirecting to landedTo when that is a redirect. reports counts those
|
||||
// DELETEs, each with the token and at the path the bytes came from.
|
||||
type stagedSource struct {
|
||||
*httptest.Server
|
||||
reports, strays atomic.Int32
|
||||
landedCode int
|
||||
landedTo string
|
||||
}
|
||||
|
||||
func stagedUpload(t *testing.T, token string, body []byte) *stagedSource {
|
||||
// stagedUpload serves body to a request carrying Bearer token, and 404 to any
|
||||
// other, the way felis-api's internal face does.
|
||||
func stagedUpload(t *testing.T, token string, body []byte) *httptest.Server {
|
||||
t.Helper()
|
||||
s := &stagedSource{}
|
||||
s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer "+token || r.URL.Path != "/u" {
|
||||
s.strays.Add(1)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer "+token {
|
||||
http.Error(w, "no such upload", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
w.Write(body)
|
||||
case http.MethodDelete:
|
||||
s.reports.Add(1)
|
||||
if s.landedTo != "" {
|
||||
w.Header().Set("Location", s.landedTo)
|
||||
}
|
||||
w.WriteHeader(cmp.Or(s.landedCode, http.StatusNoContent))
|
||||
default:
|
||||
s.strays.Add(1)
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(s.Close)
|
||||
return s
|
||||
t.Cleanup(srv.Close)
|
||||
return srv
|
||||
}
|
||||
|
||||
func uploadArgs(root, sourceURL string, body []byte) []string {
|
||||
@@ -104,9 +75,6 @@ func TestCmdFilesUpload(t *testing.T) {
|
||||
if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("exit %d, stderr %q", code, stderr.String())
|
||||
}
|
||||
if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix+`{"done":4,"total":4}`+"\n") {
|
||||
t.Fatalf("stdout %q does not start with the progress to the last byte", stdout.String())
|
||||
}
|
||||
if res := filesResult(t, stdout.String()); res.Code != "" {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
@@ -114,58 +82,8 @@ func TestCmdFilesUpload(t *testing.T) {
|
||||
if err != nil || !bytes.Equal(got, body) {
|
||||
t.Fatalf("landed %q, %v", got, err)
|
||||
}
|
||||
if n, strays := srv.reports.Load(), srv.strays.Load(); n != 1 || strays != 0 || stderr.Len() != 0 {
|
||||
t.Fatalf("reported landed %d times, %d stray requests, stderr %q; want once", n, strays, stderr.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a file already there is a result, and nothing is reported landed", func(t *testing.T) {
|
||||
root := uploadRoot(t)
|
||||
if err := os.WriteFile(filepath.Join(root, "plugins", "a.jar"), []byte("old!"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := stagedUpload(t, "tok", body)
|
||||
t.Setenv(fileedit.UploadTokenEnv, "tok")
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("exit %d, stderr %q", code, stderr.String())
|
||||
}
|
||||
if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeExists || srv.reports.Load() != 0 {
|
||||
t.Fatalf("result = %+v, reported landed %d times", res, srv.reports.Load())
|
||||
}
|
||||
})
|
||||
|
||||
// The file is in place whatever felis-api answers, so the Job still succeeds
|
||||
// and says why the staged copy may linger. A redirect is not followed, since
|
||||
// the request carries the token.
|
||||
for name, tc := range map[string]struct {
|
||||
code int
|
||||
stderr string
|
||||
}{
|
||||
"refused": {http.StatusNotFound, "felis files: tell felis-api the upload landed: DELETE returned 404 Not Found\n"},
|
||||
"redirected": {http.StatusFound, "felis files: tell felis-api the upload landed: DELETE returned 302 Found\n"},
|
||||
} {
|
||||
t.Run("a landed report "+name+" still lands the file", func(t *testing.T) {
|
||||
var elsewhere atomic.Int32
|
||||
away := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { elsewhere.Add(1) }))
|
||||
defer away.Close()
|
||||
root := uploadRoot(t)
|
||||
srv := stagedUpload(t, "tok", body)
|
||||
srv.landedCode, srv.landedTo = tc.code, away.URL+"/u"
|
||||
t.Setenv(fileedit.UploadTokenEnv, "tok")
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("exit %d, stderr %q", code, stderr.String())
|
||||
}
|
||||
if res := filesResult(t, stdout.String()); res.Code != "" || stderr.String() != tc.stderr || elsewhere.Load() != 0 {
|
||||
t.Fatalf("result = %+v, stderr %q, redirect followed %d times", res, stderr.String(), elsewhere.Load())
|
||||
}
|
||||
if got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar")); err != nil || !bytes.Equal(got, body) {
|
||||
t.Fatalf("landed %q, %v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A refused fetch is the Job failing, never a Result: the API answers it with a
|
||||
// 500 the caller retries whole.
|
||||
t.Run("a refused fetch exits 1 and lands nothing", func(t *testing.T) {
|
||||
@@ -179,9 +97,6 @@ func TestCmdFilesUpload(t *testing.T) {
|
||||
if !strings.Contains(stderr.String(), "404") {
|
||||
t.Fatalf("stderr %q does not name the status", stderr.String())
|
||||
}
|
||||
if srv.reports.Load() != 0 {
|
||||
t.Fatal("a refused fetch was reported landed")
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(root, "plugins", "a.jar")); !os.IsNotExist(err) {
|
||||
t.Fatalf("a refused fetch left a file: %v", err)
|
||||
}
|
||||
@@ -238,11 +153,10 @@ func TestCmdFilesUpload(t *testing.T) {
|
||||
|
||||
func TestCmdFilesWrite(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
content := []byte("[]\r\n")
|
||||
sum := sha256.Sum256(content)
|
||||
args := []string{"--op", "write", "--path", "ops.json", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}
|
||||
args := []string{"--op", "write", "--path", "ops.json", "--worlds-root", root}
|
||||
|
||||
t.Run("reassembles the content parts", func(t *testing.T) {
|
||||
content := []byte("[]\r\n")
|
||||
t.Setenv(fileedit.ContentPartsEnv, "1")
|
||||
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
|
||||
var stdout, stderr bytes.Buffer
|
||||
@@ -262,40 +176,11 @@ func TestCmdFilesWrite(t *testing.T) {
|
||||
t.Setenv(fileedit.ContentPartsEnv, "2")
|
||||
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("x")))
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 2 {
|
||||
t.Fatalf("exit %d, want 2", code)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("an incomplete spec wrote a file: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Without the content's SHA-256 the Job could not tell bytes changed on the
|
||||
// way from the bytes felis-api sent.
|
||||
t.Run("a write without its SHA-256 exits 2 and writes nothing", func(t *testing.T) {
|
||||
t.Setenv(fileedit.ContentPartsEnv, "1")
|
||||
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root}, &stdout, &stderr); code != 2 {
|
||||
t.Fatalf("exit %d, want 2", code)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("a write without its SHA-256 wrote a file: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("content that changed on the way is a result and writes nothing", func(t *testing.T) {
|
||||
t.Setenv(fileedit.ContentPartsEnv, "1")
|
||||
t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("[]\n")))
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root, "--sha256", hex.EncodeToString(sum[:])}, &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("exit %d, stderr %q", code, stderr.String())
|
||||
}
|
||||
if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeDigestMismatch {
|
||||
t.Fatalf("result = %+v, want %s", res, fileedit.CodeDigestMismatch)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("changed content wrote a file: %v", err)
|
||||
t.Fatalf("an incomplete spec wrote a file: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -316,43 +201,3 @@ func TestCmdFilesCallerFaultIsAResult(t *testing.T) {
|
||||
t.Fatalf("no --op: exit %d, want 2", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCmdFilesUnzip checks an unzip extracts next to the archive and reports its
|
||||
// progress before its result, the same way an upload does.
|
||||
func TestCmdFilesUnzip(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "maps"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var zb bytes.Buffer
|
||||
zw := zip.NewWriter(&zb)
|
||||
for name, body := range map[string]string{"world/level.dat": "level", "world/region/r.0.0.mca": "region!"} {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
io.WriteString(w, body)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "maps", "a.zip"), zb.Bytes(), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := cmdFiles([]string{"--op", "unzip", "--path", "maps/a.zip", "--worlds-root", root}, &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("exit %d, stderr %q", code, stderr.String())
|
||||
}
|
||||
if res := filesResult(t, stdout.String()); res.Code != "" || res.Files != 2 || res.Bytes != 12 {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
if !strings.HasPrefix(stdout.String(), fileedit.ProgressPrefix) ||
|
||||
!strings.Contains(stdout.String(), fileedit.ProgressPrefix+`{"done":12,"total":12}`+"\n") {
|
||||
t.Fatalf("stdout %q does not report the progress to the last byte", stdout.String())
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(root, "maps", "world", "region", "r.0.0.mca"))
|
||||
if err != nil || string(got) != "region!" {
|
||||
t.Fatalf("extracted %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// cgroupMemoryFiles are where a container reads the memory it is allowed:
|
||||
// cgroup v2 first, then v1.
|
||||
var cgroupMemoryFiles = []string{"/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"}
|
||||
|
||||
// limitHeapToCgroup sets the Go heap's soft limit from the container's memory
|
||||
// limit, so the collector works harder as a Job nears it and the kernel does not
|
||||
// kill the Job first. An extraction or a folder zipped for download keeps a few
|
||||
// hundred bytes per entry for as long as it runs; without the limit the heap
|
||||
// grows to twice that before a collection, and a 256 MiB Job was killed at
|
||||
// 400,000 entries whose live heap was 115 MB. GOMEMLIMIT set by hand wins.
|
||||
func limitHeapToCgroup() {
|
||||
if os.Getenv("GOMEMLIMIT") != "" {
|
||||
return
|
||||
}
|
||||
for _, f := range cgroupMemoryFiles {
|
||||
b, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if n, ok := softMemoryLimit(string(b)); ok {
|
||||
debug.SetMemoryLimit(n)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// softMemoryLimit answers three fifths of the limit a cgroup memory file holds,
|
||||
// or false for "max" (no limit) and anything unreadable. The rest is left for
|
||||
// what the kernel charges the container beyond the Go heap: the page cache of
|
||||
// the files it reads and writes, and the inodes it creates. Under a 256 MiB
|
||||
// limit, 400,000 extracted entries peaked at 184 MB resident with the heap held
|
||||
// to 150 MiB.
|
||||
func softMemoryLimit(content string) (int64, bool) {
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(content), 10, 64)
|
||||
if err != nil || n <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return n / 5 * 3, true
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSoftMemoryLimit(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
in string
|
||||
want int64
|
||||
ok bool
|
||||
}{
|
||||
{"268435456\n", 161061273, true}, // 256 MiB, as memory.max holds it
|
||||
{"max\n", 0, false},
|
||||
{"0\n", 0, false},
|
||||
{"-1", 0, false},
|
||||
{"", 0, false},
|
||||
} {
|
||||
got, ok := softMemoryLimit(c.in)
|
||||
if got != c.want || ok != c.ok {
|
||||
t.Errorf("softMemoryLimit(%q) = %d %v, want %d %v", c.in, got, ok, c.want, c.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLimitHeapToCgroup checks the limit comes from the first cgroup file there
|
||||
// is, and that GOMEMLIMIT set by hand leaves the heap alone.
|
||||
func TestLimitHeapToCgroup(t *testing.T) {
|
||||
prevFiles, prevLimit := cgroupMemoryFiles, debug.SetMemoryLimit(-1)
|
||||
t.Cleanup(func() { cgroupMemoryFiles = prevFiles; debug.SetMemoryLimit(prevLimit) })
|
||||
dir := t.TempDir()
|
||||
v1, v1b := filepath.Join(dir, "v1"), filepath.Join(dir, "v1b")
|
||||
if err := os.WriteFile(v1, []byte("268435456\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(v1b, []byte("536870912\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cgroupMemoryFiles = []string{filepath.Join(dir, "missing"), v1, v1b}
|
||||
|
||||
t.Setenv("GOMEMLIMIT", "")
|
||||
debug.SetMemoryLimit(math.MaxInt64)
|
||||
limitHeapToCgroup()
|
||||
if got := debug.SetMemoryLimit(-1); got != 161061273 {
|
||||
t.Fatalf("limit = %d, want three fifths of 256 MiB", got)
|
||||
}
|
||||
|
||||
t.Setenv("GOMEMLIMIT", "1GiB")
|
||||
debug.SetMemoryLimit(math.MaxInt64)
|
||||
limitHeapToCgroup()
|
||||
if got := debug.SetMemoryLimit(-1); got != math.MaxInt64 {
|
||||
t.Fatalf("limit = %d with GOMEMLIMIT set, want it left alone", got)
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -21,7 +21,7 @@ Commands:
|
||||
restore Extract a world archive into a world volume (internal Job entrypoint)
|
||||
backup Archive a world into the backup store and record it (internal Job entrypoint)
|
||||
backup-now Archive every user server's world now, one at a time (or the named ones; -stop stops running ones first; prints the plan, -yes applies; requires root/sudo)
|
||||
files List, read, write, mkdir, delete, rename, upload or unzip one path in a stopped server's world (internal Job entrypoint)
|
||||
files List, read, write, mkdir, delete, rename or upload one path in a stopped server's world (internal Job entrypoint)
|
||||
export Archive a stopped server's world, or read one of its backups, and hand it to felis-api for download (internal Job entrypoint)
|
||||
egress-gate Hold a build or game server pod until its egress NetworkPolicy is enforced (internal init container entrypoint)
|
||||
fetch-context Fetch and extract a submission's build context (internal Job entrypoint)
|
||||
|
||||
+8
-10
@@ -3438,18 +3438,16 @@ atomic_install_file() {
|
||||
mv -fT "$staged" "$target"
|
||||
}
|
||||
|
||||
# install_if_changed is atomic_install_file that leaves the target alone when it already
|
||||
# has the same bytes, owner and mode, so its mtime keeps meaning "the content changed".
|
||||
# felis domain check reads a proxy started before felis-link.properties' mtime as one
|
||||
# still on the old names, and a re-run that rewrote the same bytes made every install look
|
||||
# behind (and `felis domain set` restart the proxy for nothing).
|
||||
# It never fixes a target in place: the proxy's account owns these directories and can
|
||||
# swap the file for a symlink after the checks, and a chown or chmod by path would follow
|
||||
# it to, say, k3s.yaml. Owner and group compare by name, as the callers pass them.
|
||||
# install_if_changed is atomic_install_file that leaves a target with the same bytes in
|
||||
# place, fixing only its mode and owner, so the target's mtime keeps meaning "the content
|
||||
# changed". felis domain check reads a proxy started before felis-link.properties' mtime
|
||||
# as one still on the old names, and a re-run that rewrote the same bytes made every
|
||||
# install look behind (and `felis domain set` restart the proxy for nothing).
|
||||
install_if_changed() {
|
||||
local source="$1" target="$2" mode="$3" owner="$4" group="$5"
|
||||
if [ -f "$target" ] && [ ! -L "$target" ] && cmp -s "$source" "$target" \
|
||||
&& [ "$(stat -c '%U:%G %a' "$target")" = "${owner}:${group} ${mode#0}" ]; then
|
||||
if [ -f "$target" ] && [ ! -L "$target" ] && cmp -s "$source" "$target"; then
|
||||
chown "${owner}:${group}" "$target"
|
||||
chmod "$mode" "$target"
|
||||
return 0
|
||||
fi
|
||||
atomic_install_file "$@"
|
||||
|
||||
@@ -1775,10 +1775,7 @@ expect "a write that fails halfway leaves the old file whole" \
|
||||
out="$(run_atomic "sync() { return 1; }; write_file_atomic '$wadir/old.env' 0600 < '$wadir/in.replace'")"
|
||||
expect "content that did not reach the disk does not replace the old file" "DIE: could not write $wadir/old.env" "$out"
|
||||
expect "the old file survives a failed sync" "DB_PASSWORD=old-and-whole" "$(cat "$wadir/old.env")"
|
||||
left=""
|
||||
for p in "$wadir"/old.env.*; do
|
||||
if [ -e "$p" ]; then left="$left${left:+ }${p##*/}"; fi
|
||||
done
|
||||
left="$(cd "$wadir" && ls -a | grep '^old\.env\.' || true)"
|
||||
if [ -z "$left" ]; then
|
||||
echo "PASS a failed write leaves no temp file beside the old one"
|
||||
else
|
||||
@@ -2064,8 +2061,7 @@ else
|
||||
fi
|
||||
|
||||
# A re-run that writes the same felis-link.properties leaves the file alone: felis domain
|
||||
# check reads a proxy started before the file's mtime as still on the old names. stat answers
|
||||
# as for the file the first install left, root:v 0640, which the test's user cannot make.
|
||||
# check reads a proxy started before the file's mtime as still on the old names.
|
||||
run_link() { # velocity-dir [root-domain]
|
||||
VD="$1" RD="${2:-r.example.com}" TMPDIR="$1" FNFILE="$fnfile" bash -c '
|
||||
set -Eeuo pipefail
|
||||
@@ -2074,7 +2070,6 @@ run_link() { # velocity-dir [root-domain]
|
||||
prepare_velocity_layout() { :; }
|
||||
atomic_install_file() { echo "REPLACED $(basename "$2")"; cp "$1" "$2"; }
|
||||
chown() { echo "CHOWN $*"; }; chmod() { echo "CHMOD $*"; }
|
||||
stat() { echo "root:v 640"; }
|
||||
. "$FNFILE"
|
||||
STATE_DIR="$VD" FELIS_ROOT_DOMAIN="$RD" FORWARDING_SECRET=f SERVICE_TOKEN=t LOGIN_SERVER=login \
|
||||
LOBBY_SERVER=lobby FELIS_GAME_PORT=25565 VELOCITY_DIR="$VD" VELOCITY_USER=v NODE_IP=10.0.0.5
|
||||
@@ -2091,10 +2086,8 @@ case "$out" in
|
||||
*"REPLACED felis-link.properties"*) echo "FAIL: a re-run with the same names replaced felis-link.properties"; fails=$((fails + 1)) ;;
|
||||
*) echo "PASS a re-run with the same names leaves felis-link.properties in place" ;;
|
||||
esac
|
||||
case "$out" in
|
||||
*CHOWN*|*CHMOD*) echo "FAIL: the re-run changed felis-link.properties by path:"; printf '%s\n' "$out"; fails=$((fails + 1)) ;;
|
||||
*) echo "PASS the re-run changes nothing by path" ;;
|
||||
esac
|
||||
expect "the re-run still fixes the owner" "CHOWN root:v $lprops" "$out"
|
||||
expect "the re-run still fixes the mode" "CHMOD 0640 $lprops" "$out"
|
||||
expect "the kept file keeps its mtime" "$before" "$(ls -l --time-style=+%s "$lprops" 2>/dev/null || stat -f '%m' "$lprops")"
|
||||
expect "a re-run on other names replaces felis-link.properties" "REPLACED felis-link.properties" "$(run_link "$ldir2" other.example.net)"
|
||||
expect "the replaced file has the new root domain" "root-domain=other.example.net" "$(grep '^root-domain=' "$lprops")"
|
||||
@@ -4497,39 +4490,6 @@ expect "from FELIS_ARTIFACT_DIR it stops the install" "DIE: FELIS_ARTIFACT_DIR:
|
||||
expect "a source build builds the plugin" "ENSURE
|
||||
BUILD" "$(run_plugin "" 0)"
|
||||
|
||||
# --- install_if_changed never fixes a target in place -----------------------------------------
|
||||
# The proxy's account owns the directories these files land in and can swap one for a symlink
|
||||
# after the checks, so a chown or chmod by path would land on whatever the link names. The cmp
|
||||
# stub makes that swap right after the content check; chown and chmod report every call.
|
||||
iicblock="$(bsfn install_if_changed)"
|
||||
[ -n "$iicblock" ] || { echo "FAIL: no install_if_changed in $BS"; exit 1; }
|
||||
mkdir "$adir/iic"
|
||||
printf 'plugin\n' > "$adir/iic/src"
|
||||
printf 'not the plugin\n' > "$adir/iic/decoy"
|
||||
chmod 600 "$adir/iic/decoy"
|
||||
mine="$(stat -c '%U:%G' "$adir/iic/src")"
|
||||
run_iic() { # target's mode, the owner:group asked for, then "edited" or "swapped"
|
||||
rm -f "$adir/iic/dst"
|
||||
if [ "${3-}" = edited ]; then printf 'an older plugin\n' > "$adir/iic/dst"; else cp "$adir/iic/src" "$adir/iic/dst"; fi
|
||||
chmod "$1" "$adir/iic/dst"
|
||||
D="$adir/iic" OG="$2" HOW="${3-}" bash -c '
|
||||
atomic_install_file() { echo "ATOMIC $2"; }
|
||||
chown() { echo "CHOWN $*"; command chown "$@"; }
|
||||
chmod() { echo "CHMOD $*"; command chmod "$@"; }
|
||||
cmp() { command cmp "$@" || return; [ "$HOW" != swapped ] || ln -sfn "$D/decoy" "$3"; }
|
||||
'"$iicblock"'
|
||||
install_if_changed "$D/src" "$D/dst" 0644 "${OG%%:*}" "${OG#*:}"' 2>&1
|
||||
}
|
||||
iic_is() { # label want got
|
||||
[ "$3" = "$2" ] && echo "PASS $1" || { printf 'FAIL %s: got\n%s\nwant\n%s\n' "$1" "$3" "$2"; fails=$((fails + 1)); }
|
||||
}
|
||||
iic_is "the same bytes, owner and mode are left alone" "" "$(run_iic 644 "$mine")"
|
||||
iic_is "the same bytes with the wrong mode are reinstalled" "ATOMIC $adir/iic/dst" "$(run_iic 600 "$mine")"
|
||||
iic_is "the same bytes with the wrong owner are reinstalled" "ATOMIC $adir/iic/dst" "$(run_iic 644 "felis-nobody:${mine#*:}")"
|
||||
iic_is "new bytes are installed" "ATOMIC $adir/iic/dst" "$(run_iic 644 "$mine" edited)"
|
||||
iic_is "a target swapped for a symlink after the checks is reinstalled" "ATOMIC $adir/iic/dst" "$(run_iic 644 "$mine" swapped)"
|
||||
iic_is "and the file the link named keeps its mode" 600 "$(stat -c %a "$adir/iic/decoy")"
|
||||
|
||||
# --- k3s's own images from its GitHub release ----------------------------------------------------
|
||||
kablock="$(bsfn stage_k3s_airgap_images)"
|
||||
mkdir -p "$adir/k3simg" "$adir/k3srel"
|
||||
|
||||
+33
-696
@@ -743,72 +743,6 @@ components:
|
||||
state: { type: string, const: pending }
|
||||
filename: { type: string, description: What the download saves as. }
|
||||
|
||||
FileUploadSession:
|
||||
type: object
|
||||
description: Where an upload session stands (internal/api/handlers_fileops.go fileSessionView).
|
||||
required: [id, path, size, received, part_max_bytes, parts]
|
||||
properties:
|
||||
id: { type: string, description: 32 hex characters. }
|
||||
path: { type: string, description: Where the file lands, relative to the world root. }
|
||||
size: { type: integer, format: int64, description: The file's length. }
|
||||
received: { type: integer, format: int64, description: Bytes here so far; the next part starts here. }
|
||||
part_max_bytes: { type: integer, format: int64, description: The most one part may carry. }
|
||||
parts:
|
||||
type: array
|
||||
description: >-
|
||||
The parts taken so far, in order, each with the SHA-256 it arrived
|
||||
with. A client resuming from a file it still holds hashes the same
|
||||
ranges and starts over when one differs.
|
||||
items:
|
||||
type: object
|
||||
required: [size, sha256]
|
||||
properties:
|
||||
size: { type: integer, format: int64 }
|
||||
sha256: { type: string, pattern: '^[0-9a-f]{64}$' }
|
||||
|
||||
StartFileOp:
|
||||
type: object
|
||||
properties:
|
||||
overwrite: { type: boolean, description: Replace files already there. }
|
||||
|
||||
FileOp:
|
||||
type: object
|
||||
description: One background upload or extraction (internal/api/handlers_fileops.go fileOpView).
|
||||
required: [id, op, path, state, started_at, done, total]
|
||||
properties:
|
||||
id: { type: string }
|
||||
op: { type: string, enum: [upload, unzip] }
|
||||
path: { type: string, description: The file landed, or the archive extracted. }
|
||||
state: { type: string, enum: [running, succeeded, failed] }
|
||||
started_at: { type: string, format: date-time }
|
||||
finished_at: { type: string, format: date-time, description: Omitted while it runs. }
|
||||
done: { type: integer, format: int64, description: Bytes landed or extracted so far; 0 before the first report. }
|
||||
total: { type: integer, format: int64, description: Bytes in all; 0 before the first report. }
|
||||
files: { type: integer, description: Files an extraction wrote. Omitted otherwise. }
|
||||
bytes: { type: integer, format: int64, description: Bytes an extraction wrote. Omitted otherwise. }
|
||||
error: { $ref: '#/components/schemas/FileOpError' }
|
||||
|
||||
FileOpError:
|
||||
type: object
|
||||
description: >-
|
||||
Why an op failed (internal/api/handlers_fileops.go fileOpError). code is
|
||||
what the synchronous file routes answer for the same refusal
|
||||
(file_exists, volume_full, file_changed, not_found, bad_path), an
|
||||
extraction's own (archive_invalid, archive_unsafe, archive_symlink,
|
||||
type_conflict), or job_failed for a Job that ended without saying why.
|
||||
required: [code, message]
|
||||
properties:
|
||||
code: { type: string }
|
||||
message: { type: string }
|
||||
entry: { type: string, description: The archive entry refused, or the path it collides with. }
|
||||
conflicts:
|
||||
type: array
|
||||
items: { type: string }
|
||||
description: On file_exists from an extraction, the first 200 files it would replace, sorted.
|
||||
conflict_count: { type: integer, description: How many files it would replace in all. }
|
||||
need: { type: integer, format: int64, description: On volume_full, the bytes needed. }
|
||||
avail: { type: integer, format: int64, description: On volume_full, the bytes free; left out when none are. }
|
||||
|
||||
ExportStatus:
|
||||
type: object
|
||||
description: Where an export stands (internal/api/exports.go exportStatusView).
|
||||
@@ -1372,10 +1306,7 @@ paths:
|
||||
face and the bearer token minted with the upload is the whole check. The
|
||||
token opens its upload once. An unknown id, a wrong or missing token and a
|
||||
spent token are all the same 404, so the route says nothing about which
|
||||
uploads exist. An upload session committed through POST
|
||||
…/files/uploads/{id}/commit is fetched here the same way, under the
|
||||
session id; it stays staged until its Job reports the file landed
|
||||
(DELETE), so a Job that failed at any point can be committed again.
|
||||
uploads exist.
|
||||
x-felis-face: [internal]
|
||||
x-felis-tier: public
|
||||
security: []
|
||||
@@ -1394,30 +1325,6 @@ paths:
|
||||
schema: { type: string, format: binary }
|
||||
'404':
|
||||
$ref: '#/components/responses/NotFound'
|
||||
delete:
|
||||
tags: [files]
|
||||
operationId: internalFileUploadLanded
|
||||
summary: The Job reports a staged upload landed (the same one-time bearer token).
|
||||
description: >-
|
||||
Sent once the file is in place. An upload session is then dropped from
|
||||
felis-api's disk; a single-request upload goes when its request ends in
|
||||
any case. Only the token of the session's latest commit is taken. As for
|
||||
the fetch, every refusal is the same 404.
|
||||
x-felis-face: [internal]
|
||||
x-felis-tier: public
|
||||
security: []
|
||||
parameters:
|
||||
- { name: id, in: path, required: true, schema: { type: string } }
|
||||
- name: Authorization
|
||||
in: header
|
||||
required: true
|
||||
description: Bearer followed by the token the Job fetched the upload with.
|
||||
schema: { type: string }
|
||||
responses:
|
||||
'204':
|
||||
$ref: '#/components/responses/NoContent'
|
||||
'404':
|
||||
$ref: '#/components/responses/NotFound'
|
||||
|
||||
/api/v1/internal/exports/{id}:
|
||||
put:
|
||||
@@ -1425,13 +1332,8 @@ paths:
|
||||
operationId: internalExportUpload
|
||||
summary: Hand one export's archive over for download (one-time bearer token).
|
||||
description: >-
|
||||
The export Job PUTs the archive or file here, always chunked, with its
|
||||
size as X-Felis-Export-Length when it knows it and, once the body has
|
||||
ended, the SHA-256 of all it sent as the Content-Digest trailer
|
||||
(sha-256=:<base64>:). felis-api holds the last bytes back from the
|
||||
browser until the bytes it received number and hash as the Job said, so
|
||||
a body changed on the way, or one without the trailer, ends the
|
||||
download short and the browser reports it failed. The Job holds no service token, so the route
|
||||
The export Job PUTs the tar.gz here, chunked for a world and with its
|
||||
Content-Length for a backup. The Job holds no service token, so the route
|
||||
is public on the internal face and the bearer token minted with the
|
||||
export is the whole check; an unknown id, a wrong or missing token and a
|
||||
token already used are all the same 404. The request then waits, body
|
||||
@@ -1449,11 +1351,6 @@ paths:
|
||||
required: true
|
||||
description: Bearer followed by the token minted with the export.
|
||||
schema: { type: string }
|
||||
- name: X-Felis-Export-Length
|
||||
in: header
|
||||
required: false
|
||||
description: The body's length in bytes, when the Job knows it; the download then carries it as Content-Length.
|
||||
schema: { type: integer, format: int64, minimum: 0 }
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
@@ -1462,11 +1359,6 @@ paths:
|
||||
responses:
|
||||
'204':
|
||||
$ref: '#/components/responses/NoContent'
|
||||
'400':
|
||||
description: X-Felis-Export-Length is not a byte count (bad_request); the token is not spent.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'404':
|
||||
$ref: '#/components/responses/NotFound'
|
||||
'409':
|
||||
@@ -1475,7 +1367,7 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'410':
|
||||
description: Nobody opened the download within 90 seconds, the browser left before the archive ended, or what arrived did not number or hash as the Job declared, so the download was cut off (export_expired).
|
||||
description: Nobody opened the download within 90 seconds, or the browser left before the archive ended (export_expired).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
@@ -4257,14 +4149,11 @@ paths:
|
||||
description: >-
|
||||
Starts a Job that reads the archive from the backup store and hands it to
|
||||
felis-api, which streams it to the browser (poll GET /exports/{ticket},
|
||||
then open its download). The Job checks the archive against the sha256
|
||||
recorded when it was written as it streams; a mismatch cuts the
|
||||
download off short of its end. On the way out config/paper-global.yml
|
||||
(the cluster's forwarding secret) is left out and server.properties has
|
||||
its rcon.password redacted, so the download carries no Content-Length.
|
||||
A user gets 404 for a backup outside their scope, as their list never
|
||||
shows it. One export per user at a time, 2 across the install, 6 per
|
||||
user per hour.
|
||||
then open its download). The archive is checked against the sha256
|
||||
recorded when it was written as it streams; a mismatch aborts the
|
||||
download. A user gets 404 for a backup outside their scope, as their
|
||||
list never shows it. One export per user at a time, 2 across the
|
||||
install, 6 per user per hour.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
@@ -4317,9 +4206,7 @@ paths:
|
||||
it to felis-api, which streams it to the browser (poll GET
|
||||
/exports/{ticket}, then open its download). The server must be fully
|
||||
stopped, and it cannot start until the download has ended or the Job's
|
||||
2 hour deadline passes. The same two files are guarded as in a backup
|
||||
export, matched by the file itself, so a link to either under another
|
||||
name is guarded too. Same limits as a backup export.
|
||||
2 hour deadline passes. Same limits as a backup export.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
@@ -4398,10 +4285,8 @@ paths:
|
||||
description: >-
|
||||
The first request spends the ticket, whatever becomes of it. The archive
|
||||
streams as the Job sends it, with Content-Length when it is known; a
|
||||
download that cannot finish (the Job died, a backup did not match its
|
||||
recorded sha256, or the bytes did not hash to the SHA-256 the Job sent
|
||||
with them) is cut off before its last bytes, so the browser reports it
|
||||
failed. HEAD is
|
||||
download that cannot finish (the Job died, or a backup did not match its
|
||||
recorded sha256) is cut off, so the browser reports it failed. HEAD is
|
||||
refused, since it would spend the ticket on no body.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
@@ -4410,19 +4295,13 @@ paths:
|
||||
- { name: ticket, in: path, required: true, schema: { type: string } }
|
||||
responses:
|
||||
'200':
|
||||
description: >-
|
||||
The export as an attachment: a world or a backup as a tar.gz, a
|
||||
downloaded folder as a zip, a downloaded file as its bytes.
|
||||
description: The tar.gz, as an attachment.
|
||||
headers:
|
||||
Content-Disposition:
|
||||
schema: { type: string }
|
||||
content:
|
||||
application/gzip:
|
||||
schema: { type: string, format: binary }
|
||||
application/zip:
|
||||
schema: { type: string, format: binary }
|
||||
application/octet-stream:
|
||||
schema: { type: string, format: binary }
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'404':
|
||||
@@ -4476,7 +4355,7 @@ paths:
|
||||
required: [name, kind, state]
|
||||
properties:
|
||||
name: { type: string }
|
||||
kind: { type: string, enum: [backup, restore, export_world, export_backup, export_files] }
|
||||
kind: { type: string, enum: [backup, restore, export_world, export_backup] }
|
||||
state: { type: string, enum: [running, succeeded, failed] }
|
||||
message: { type: string }
|
||||
started_at: { type: string, format: date-time }
|
||||
@@ -4545,11 +4424,10 @@ paths:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [path, entries, truncated, free_bytes]
|
||||
required: [path, entries, truncated]
|
||||
properties:
|
||||
path: { type: string }
|
||||
truncated: { type: boolean, description: The listing hit the entry cap and is incomplete. }
|
||||
free_bytes: { type: integer, format: int64, nullable: true, description: Bytes free on the world volume, for a client to check an upload fits before sending it; 0 is a full volume, and null a volume whose free space the Job could not read. }
|
||||
entries:
|
||||
type: array
|
||||
items:
|
||||
@@ -4615,7 +4493,7 @@ paths:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [path, content, sha256, content_sha256]
|
||||
required: [path, content, sha256]
|
||||
properties:
|
||||
path: { type: string }
|
||||
content: { type: string, format: byte, description: Base64-encoded file bytes. }
|
||||
@@ -4625,13 +4503,6 @@ paths:
|
||||
description: >-
|
||||
SHA-256 of the file as stored (before the rcon.password redaction in
|
||||
server.properties). Send it back as expect_sha256 on the next write.
|
||||
content_sha256:
|
||||
type: string
|
||||
pattern: '^[0-9a-f]{64}$'
|
||||
description: >-
|
||||
SHA-256 of the decoded content as sent (after any redaction). A
|
||||
client that gets content hashing otherwise got it damaged on the
|
||||
way, and reads it again.
|
||||
'400':
|
||||
description: Missing path, invalid server name, or a path that escapes the world root.
|
||||
content:
|
||||
@@ -4656,13 +4527,6 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'502':
|
||||
description: >-
|
||||
The file's bytes do not hash to the digest the file Job sent with them
|
||||
(read_damaged): they changed on the way to felis-api. Read it again.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'503':
|
||||
$ref: '#/components/responses/ServiceUnavailable'
|
||||
'504':
|
||||
@@ -4685,10 +4549,7 @@ paths:
|
||||
root is refused. The replacement is atomic (a synced temporary sibling renamed
|
||||
over the file, keeping its mode), so a failed write leaves the old file whole.
|
||||
With expect_sha256 the write lands only if the file still has that hash;
|
||||
otherwise 409 file_changed. content_sha256 is the SHA-256 of the content:
|
||||
content that hashes otherwise changed on the way and is refused (400
|
||||
digest_mismatch) before a Job starts, and the Job checks the bytes it received
|
||||
the same way before writing. Audited as file.write.
|
||||
otherwise 409 file_changed. Audited as file.write.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
@@ -4705,16 +4566,9 @@ paths:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [content, content_sha256]
|
||||
required: [content]
|
||||
properties:
|
||||
content: { type: string, format: byte, description: Base64-encoded file bytes. }
|
||||
content_sha256:
|
||||
type: string
|
||||
pattern: '^[0-9a-f]{64}$'
|
||||
description: >-
|
||||
The SHA-256 (lowercase hex) of the decoded content. Absent is 400
|
||||
digest_required, malformed 400 bad_digest, and content that does not
|
||||
hash to it 400 digest_mismatch; nothing is written.
|
||||
expect_sha256:
|
||||
type: string
|
||||
pattern: '^[0-9a-f]{64}$'
|
||||
@@ -4741,7 +4595,7 @@ paths:
|
||||
status: { type: string, const: written }
|
||||
sha256: { type: string, pattern: '^[0-9a-f]{64}$', description: SHA-256 of the bytes written. }
|
||||
'400':
|
||||
description: Missing path, malformed body, invalid server name, or a path that escapes the world root (bad_request, bad_path), or content that came without its SHA-256 (digest_required), with a malformed one (bad_digest), or changed on the way (digest_mismatch).
|
||||
description: Missing path, malformed body, invalid server name, or a path that escapes the world root.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
@@ -4964,75 +4818,6 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
|
||||
/api/v1/servers/{name}/files/download:
|
||||
post:
|
||||
tags: [files]
|
||||
operationId: downloadServerFile
|
||||
summary: Start downloading one file or folder of a stopped server's world (owner-or-admin).
|
||||
description: >-
|
||||
An export (poll GET /exports/{ticket}, then open its download): a Job
|
||||
reads the file, or zips the folder, from the world volume read-only and
|
||||
hands it to felis-api, which streams it to the browser. A file saves
|
||||
under its own name with its length; a folder as NAME.zip, streamed
|
||||
without one, with symbolic links, devices and sockets left out.
|
||||
config/paper-global.yml, the cluster's forwarding secret, is refused as
|
||||
a file and left out of a folder, and server.properties goes out with
|
||||
its rcon.password redacted; both are matched by the file itself, so a
|
||||
link to either under another name is guarded too. The server cannot
|
||||
start until the download has ended. Two file downloads per user at a
|
||||
time, 4 across the install, 30 per user per hour, counted apart from
|
||||
world and backup exports. Audited as file.download.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
parameters:
|
||||
- { name: name, in: path, required: true, schema: { type: string } }
|
||||
- name: path
|
||||
in: query
|
||||
required: true
|
||||
description: File or folder to download, relative to the world root. The root itself is refused.
|
||||
schema: { type: string }
|
||||
- name: dir
|
||||
in: query
|
||||
required: false
|
||||
description: true when path is a folder, which is sent as a zip. The Job refuses a path that is not what dir says.
|
||||
schema: { type: string, enum: ["true", "false"] }
|
||||
responses:
|
||||
'202':
|
||||
description: Download started.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/ExportTicket' }
|
||||
'400':
|
||||
description: Missing path (bad_request), the world root (bad_path), or a malformed server name (bad_name).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
$ref: '#/components/responses/Forbidden'
|
||||
'404':
|
||||
description: Unknown server.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'409':
|
||||
description: Server is not stopped (not_stopped), has no world volume yet (no_world_volume), or a restore, backup, file change or another export already holds its world volume (maintenance_in_progress).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
description: A file download limit is reached (export_busy); Retry-After gives the seconds to wait.
|
||||
headers:
|
||||
Retry-After:
|
||||
schema: { type: integer }
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'503':
|
||||
$ref: '#/components/responses/ServiceUnavailable'
|
||||
|
||||
/api/v1/servers/{name}/files/upload:
|
||||
put:
|
||||
tags: [files]
|
||||
@@ -5040,19 +4825,15 @@ paths:
|
||||
summary: Upload a file into a server's world volume (owner-or-admin; server must be stopped).
|
||||
description: >-
|
||||
Lands the raw request body as the file at path, up to 64 MiB — a plugin jar,
|
||||
a datapack, a world region; a bigger file goes up as an upload session
|
||||
(POST …/files/uploads). Content-Length is required (411
|
||||
a datapack, a world region. Content-Length is required (411
|
||||
length_required). An existing file is 409 file_exists unless overwrite=true;
|
||||
a folder at the path is 400 bad_path either way. The body is staged on
|
||||
felis-api's disk first and then fetched by the file Job with a one-time
|
||||
token, so the world lock is taken only after the body has arrived and a slow
|
||||
upload holds off no backup. The file lands atomically: a synced temporary
|
||||
sibling is checked against the staged size and SHA-256, then renamed into
|
||||
place, so a failed upload leaves the old file whole. The body carries its
|
||||
SHA-256 as Content-Digest; felis-api checks it as the body arrives, and
|
||||
the Job checks the same digest again as it fetches the staged copy, so
|
||||
every hop between the browser and the world volume is verified. Same
|
||||
stopped-gate and os.Root containment as a write. Audited as file.upload.
|
||||
place, so a failed upload leaves the old file whole. Same stopped-gate and
|
||||
os.Root containment as a write. Audited as file.upload.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
@@ -5068,14 +4849,6 @@ paths:
|
||||
required: false
|
||||
description: true replaces an existing file, keeping its mode. Anything else refuses to.
|
||||
schema: { type: string, enum: ["true", "false"] }
|
||||
- name: Content-Digest
|
||||
in: header
|
||||
required: true
|
||||
description: >-
|
||||
The SHA-256 of the body as RFC 9530 sends it, sha-256=:<base64>:.
|
||||
Other algorithms listed beside it are ignored. Bytes that do not hash
|
||||
to it were changed on the way and are refused whole.
|
||||
schema: { type: string, example: 'sha-256=:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=:' }
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
@@ -5097,10 +4870,8 @@ paths:
|
||||
'400':
|
||||
description: >-
|
||||
Missing path, invalid server name, a folder or the world root at the path,
|
||||
a path that escapes the world root, a body that ended before
|
||||
Content-Length bytes arrived (upload_incomplete), no Content-Digest
|
||||
(digest_required), a malformed one (bad_digest), or bytes that do not
|
||||
hash to it (digest_mismatch; nothing is staged, so send it again).
|
||||
a path that escapes the world root, or a body that ended before
|
||||
Content-Length bytes arrived (upload_incomplete).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
@@ -5124,7 +4895,7 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'413':
|
||||
description: The file is over 64 MiB, the most one request carries (too_large); send it as an upload session instead.
|
||||
description: The file is over 64 MiB (too_large).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
@@ -5144,404 +4915,6 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
|
||||
/api/v1/servers/{name}/files/uploads:
|
||||
post:
|
||||
tags: [files]
|
||||
operationId: beginServerFileUpload
|
||||
summary: Begin an upload session for a file too big for one request (owner-or-admin; server must be stopped).
|
||||
description: >-
|
||||
A file of any size goes up in parts: this begins a session for path and
|
||||
the file's size, PUT …/uploads/{id}?offset= sends each part (at most
|
||||
part_max_bytes, 32 MiB, so each fits the edge's body limit), and POST
|
||||
…/uploads/{id}/commit lands it. There is no size ceiling but felis-api's
|
||||
staging disk, and room for the whole file is reserved here, so an upload
|
||||
that begins is one the disk can finish (507 upload_staging_full
|
||||
otherwise). A session belongs to the account and server it was begun
|
||||
for, answers no one else, and is dropped after 6 hours untouched. Four
|
||||
sessions per account at a time. Sessions do not survive a felis-api
|
||||
restart.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
parameters:
|
||||
- { name: name, in: path, required: true, schema: { type: string } }
|
||||
- name: path
|
||||
in: query
|
||||
required: true
|
||||
description: File to create, relative to the world root. It must stay inside it (400 bad_path); its folder is checked when the file lands.
|
||||
schema: { type: string }
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [size]
|
||||
properties:
|
||||
size: { type: integer, format: int64, minimum: 0, description: The file's length in bytes. }
|
||||
responses:
|
||||
'201':
|
||||
description: Session begun.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/FileUploadSession' }
|
||||
'400':
|
||||
description: Missing path or size, or a negative size (bad_request), a path leaving the world folder or naming the folder itself (bad_path), or a malformed server name (bad_name).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
$ref: '#/components/responses/Forbidden'
|
||||
'404':
|
||||
description: Unknown server.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'409':
|
||||
description: Server is not stopped (not_stopped) or has no world volume yet (no_world_volume).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'429':
|
||||
description: The account already has 4 uploads in progress (too_many_uploads).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'503':
|
||||
$ref: '#/components/responses/ServiceUnavailable'
|
||||
'507':
|
||||
description: felis-api's staging disk has no room for a file this size right now (upload_staging_full).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
|
||||
/api/v1/servers/{name}/files/uploads/{id}:
|
||||
get:
|
||||
tags: [files]
|
||||
operationId: getServerFileUpload
|
||||
summary: Where an upload session stands (owner-or-admin, the account that began it).
|
||||
description: >-
|
||||
received is where the next part starts: after a lost answer or a 409
|
||||
upload_offset_mismatch, read it here and continue from there. Needs no
|
||||
stopped server.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
parameters:
|
||||
- { name: name, in: path, required: true, schema: { type: string } }
|
||||
- { name: id, in: path, required: true, schema: { type: string } }
|
||||
responses:
|
||||
'200':
|
||||
description: The session.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/FileUploadSession' }
|
||||
'400':
|
||||
description: Malformed server name (bad_name).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
$ref: '#/components/responses/Forbidden'
|
||||
'404':
|
||||
description: Unknown server, or no such session for this account on this server (upload_not_found).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'503':
|
||||
$ref: '#/components/responses/ServiceUnavailable'
|
||||
put:
|
||||
tags: [files]
|
||||
operationId: putServerFileUploadPart
|
||||
summary: Send one part of an upload session (owner-or-admin, the account that began it).
|
||||
description: >-
|
||||
The raw body is appended at offset, which must be where the session
|
||||
ends. Content-Length and the part's own Content-Digest are required, and
|
||||
the part is taken whole or not at all: one cut short, or one whose bytes
|
||||
do not hash to its digest, leaves the session where it was. Parts go one at a
|
||||
time (409 upload_busy while one arrives). Needs no stopped server, so
|
||||
starting the server midway costs only the commit's refusal until it is
|
||||
stopped again.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
parameters:
|
||||
- { name: name, in: path, required: true, schema: { type: string } }
|
||||
- { name: id, in: path, required: true, schema: { type: string } }
|
||||
- name: offset
|
||||
in: query
|
||||
required: true
|
||||
description: The byte position the part starts at, the session's received.
|
||||
schema: { type: integer, format: int64, minimum: 0 }
|
||||
- name: Content-Digest
|
||||
in: header
|
||||
required: true
|
||||
description: >-
|
||||
The SHA-256 of the body as RFC 9530 sends it, sha-256=:<base64>:.
|
||||
Other algorithms listed beside it are ignored. Bytes that do not hash
|
||||
to it were changed on the way and are refused whole.
|
||||
schema: { type: string, example: 'sha-256=:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=:' }
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/octet-stream:
|
||||
schema: { type: string, format: binary }
|
||||
responses:
|
||||
'200':
|
||||
description: Part taken.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/FileUploadSession' }
|
||||
'400':
|
||||
description: >-
|
||||
A missing or malformed offset (bad_request), a body that ended before
|
||||
its Content-Length (upload_incomplete), no Content-Digest
|
||||
(digest_required), a malformed one (bad_digest), bytes that do not hash
|
||||
to it (digest_mismatch; the part was not taken, so send it again), or a
|
||||
malformed server name (bad_name).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
$ref: '#/components/responses/Forbidden'
|
||||
'404':
|
||||
description: Unknown server, or no such session for this account on this server (upload_not_found).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'409':
|
||||
description: offset is not where the session ends (upload_offset_mismatch), or another part is still arriving (upload_busy).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'411':
|
||||
description: The request has no Content-Length (length_required).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'413':
|
||||
description: The part is over part_max_bytes, or runs past the size the session began with (part_too_large).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'503':
|
||||
$ref: '#/components/responses/ServiceUnavailable'
|
||||
'507':
|
||||
description: felis-api's staging disk ran out of room (upload_staging_full).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
delete:
|
||||
tags: [files]
|
||||
operationId: deleteServerFileUpload
|
||||
summary: Cancel an upload session and free its room (owner-or-admin, the account that began it).
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
parameters:
|
||||
- { name: name, in: path, required: true, schema: { type: string } }
|
||||
- { name: id, in: path, required: true, schema: { type: string } }
|
||||
responses:
|
||||
'204':
|
||||
description: Cancelled.
|
||||
'400':
|
||||
description: Malformed server name (bad_name).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
$ref: '#/components/responses/Forbidden'
|
||||
'404':
|
||||
description: Unknown server, or no such session for this account on this server (upload_not_found).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'409':
|
||||
description: A part is still arriving (upload_busy).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'503':
|
||||
$ref: '#/components/responses/ServiceUnavailable'
|
||||
|
||||
/api/v1/servers/{name}/files/uploads/{id}/commit:
|
||||
post:
|
||||
tags: [files]
|
||||
operationId: commitServerFileUpload
|
||||
summary: Land a finished upload session in the world volume (owner-or-admin; server must be stopped).
|
||||
description: >-
|
||||
Starts the Job that fetches the session's bytes from felis-api and lands
|
||||
them at its path, checked against their size and SHA-256 and renamed into
|
||||
place, so a failed landing leaves the old file whole. It answers at once
|
||||
with the op; GET …/files/ops reports how it ends (file_exists when a file
|
||||
is at the path and overwrite is not true). The Job holds the world volume
|
||||
while it runs, so the server cannot start meanwhile. A Job that fails
|
||||
before it has every byte leaves the session to commit again; once the
|
||||
bytes have gone to the Job the session is gone. Audited as file.upload.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
parameters:
|
||||
- { name: name, in: path, required: true, schema: { type: string } }
|
||||
- { name: id, in: path, required: true, schema: { type: string } }
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/StartFileOp' }
|
||||
responses:
|
||||
'202':
|
||||
description: Landing started.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [op]
|
||||
properties:
|
||||
op: { $ref: '#/components/schemas/FileOp' }
|
||||
'400':
|
||||
description: Malformed body, or a malformed server name (bad_name).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
$ref: '#/components/responses/Forbidden'
|
||||
'404':
|
||||
description: Unknown server, or no such session for this account on this server (upload_not_found).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'409':
|
||||
description: >-
|
||||
Not every byte has arrived (upload_incomplete; the world lock is not
|
||||
asked for), a part is still arriving (upload_busy), the server is not stopped (not_stopped) or has
|
||||
no world volume yet (no_world_volume), or a restore, backup, file
|
||||
change or export already holds its world volume, this session's
|
||||
earlier commit included (maintenance_in_progress).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'503':
|
||||
$ref: '#/components/responses/ServiceUnavailable'
|
||||
|
||||
/api/v1/servers/{name}/files/unzip:
|
||||
post:
|
||||
tags: [files]
|
||||
operationId: unzipServerFile
|
||||
summary: Extract a .zip into the folder holding it (owner-or-admin; server must be stopped).
|
||||
description: >-
|
||||
Starts a Job that extracts the archive into a temporary folder beside it
|
||||
and moves the result into place, and answers at once with the op; GET
|
||||
…/files/ops reports how it ends. Nothing changes unless every entry is
|
||||
safe: an entry leaving the folder, an absolute path, or a link ends
|
||||
archive_unsafe or archive_symlink; an entry whose size differs from what
|
||||
the archive declares ends archive_invalid; a file where the archive has
|
||||
a folder, or the reverse, ends type_conflict. Without overwrite an
|
||||
archive that would replace any file ends file_exists with the files it
|
||||
would replace, for the caller to confirm and run again with overwrite.
|
||||
Names stored in GBK, as Windows zips in a Chinese locale have them, are
|
||||
read as such. The Job holds the world volume while it runs. Audited as
|
||||
file.unzip.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
parameters:
|
||||
- { name: name, in: path, required: true, schema: { type: string } }
|
||||
- name: path
|
||||
in: query
|
||||
required: true
|
||||
description: The .zip to extract, relative to the world root.
|
||||
schema: { type: string }
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/StartFileOp' }
|
||||
responses:
|
||||
'202':
|
||||
description: Extraction started.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [op]
|
||||
properties:
|
||||
op: { $ref: '#/components/schemas/FileOp' }
|
||||
'400':
|
||||
description: Missing path or malformed body (bad_request), a path not ending in .zip (bad_path), or a malformed server name (bad_name).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
$ref: '#/components/responses/Forbidden'
|
||||
'404':
|
||||
description: Unknown server.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'409':
|
||||
description: Server is not stopped (not_stopped), has no world volume yet (no_world_volume), or a restore, backup, file change or export already holds its world volume (maintenance_in_progress).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'503':
|
||||
$ref: '#/components/responses/ServiceUnavailable'
|
||||
|
||||
/api/v1/servers/{name}/files/ops:
|
||||
get:
|
||||
tags: [files]
|
||||
operationId: listServerFileOps
|
||||
summary: A server's background uploads and extractions (owner-or-admin).
|
||||
description: >-
|
||||
Newest first: the one running, if any, and those that ended within the
|
||||
last 30 minutes, at most 10. Needs no stopped server.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
parameters:
|
||||
- { name: name, in: path, required: true, schema: { type: string } }
|
||||
responses:
|
||||
'200':
|
||||
description: The ops.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [ops]
|
||||
properties:
|
||||
ops:
|
||||
type: array
|
||||
items: { $ref: '#/components/schemas/FileOp' }
|
||||
'400':
|
||||
description: Malformed server name (bad_name).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
$ref: '#/components/responses/Forbidden'
|
||||
'404':
|
||||
description: Unknown server.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
'503':
|
||||
$ref: '#/components/responses/ServiceUnavailable'
|
||||
|
||||
|
||||
# --------------------------------------------------- scheduled tasks (app) ---
|
||||
/api/v1/servers/{name}/schedules:
|
||||
get:
|
||||
@@ -5864,7 +5237,7 @@ paths:
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
description: >-
|
||||
Not an owner (forbidden); a change to the caller's own role (self_protected); a role change on the owner account (owner_protected), which only the host's break-glass console (sudo felis breakGlass) may make; or a change to the caller's own email without a reauth in the last 5 minutes (reauth_required).
|
||||
Not an owner (forbidden); a change to the caller's own role (self_protected); or a role change on the owner account (owner_protected), which only the host's break-glass console (sudo felis breakGlass) may make.
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
@@ -6136,11 +5509,7 @@ paths:
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
description: >-
|
||||
Not an owner (forbidden); or unbinding the caller's own passkeys without a reauth in the last 5 minutes (reauth_required).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
$ref: '#/components/responses/Forbidden'
|
||||
|
||||
/api/v1/users/{id}/links:
|
||||
post:
|
||||
@@ -7395,22 +6764,13 @@ paths:
|
||||
the API, a larger context goes through the chunked upload at
|
||||
/api/v1/me/submissions/{id}/context/upload instead. An upload that would push the
|
||||
caller past their per-user stored-context budget is refused with 403
|
||||
before the excess is persisted. The body's SHA-256 is required as
|
||||
Content-Digest; bytes that do not hash to it were changed on the way,
|
||||
and none of them replace the context stored before. Returns 503 when
|
||||
the deployment's context store has no implemented upload transport.
|
||||
before the excess is persisted. Returns 503 when the deployment's context
|
||||
store has no implemented upload transport.
|
||||
x-felis-face: [external]
|
||||
x-felis-tier: app
|
||||
security: [{ sessionCookie: [] }]
|
||||
parameters:
|
||||
- { name: id, in: path, required: true, schema: { type: string } }
|
||||
- name: Content-Digest
|
||||
in: header
|
||||
required: true
|
||||
description: >-
|
||||
The SHA-256 of the body as RFC 9530 sends it, sha-256=:<base64>:.
|
||||
Other algorithms listed beside it are ignored.
|
||||
schema: { type: string, example: 'sha-256=:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=:' }
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
@@ -7423,14 +6783,7 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Submission' }
|
||||
'400':
|
||||
description: >-
|
||||
A body that is not a gzip tarball or is over the context cap
|
||||
(bad_request), no Content-Digest (digest_required), a malformed one
|
||||
(bad_digest), or bytes that do not hash to it (digest_mismatch;
|
||||
nothing was stored, so send it again).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
$ref: '#/components/responses/BadRequest'
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
@@ -7491,9 +6844,8 @@ paths:
|
||||
upload_offset_mismatch and the client reads GET for where to resume. The
|
||||
first part must open with the gzip magic (400). The staged total meets
|
||||
the same context cap (400) and storage budget (403) as a single upload.
|
||||
A part that breaks off is cut back off, and so is one whose bytes do not
|
||||
hash to its Content-Digest, so the staged bytes are always a prefix of
|
||||
the file. One request per upload at a time (409 upload_busy).
|
||||
A part that breaks off is cut back off, so the staged bytes are always a
|
||||
prefix of the file. One request per upload at a time (409 upload_busy).
|
||||
Staged bytes untouched for 24 hours are deleted. The budget check reads
|
||||
blob sizes remembered for up to a minute; when a size has to be read and
|
||||
the uploads store does not answer, the answer is 503
|
||||
@@ -7505,13 +6857,6 @@ paths:
|
||||
parameters:
|
||||
- { name: id, in: path, required: true, schema: { type: string } }
|
||||
- { name: offset, in: query, required: true, schema: { type: integer, format: int64, minimum: 0 } }
|
||||
- name: Content-Digest
|
||||
in: header
|
||||
required: true
|
||||
description: >-
|
||||
The SHA-256 of the part as RFC 9530 sends it, sha-256=:<base64>:.
|
||||
Other algorithms listed beside it are ignored.
|
||||
schema: { type: string, example: 'sha-256=:47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=:' }
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
@@ -7524,15 +6869,7 @@ paths:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/ContextUploadProgress' }
|
||||
'400':
|
||||
description: >-
|
||||
A missing or malformed offset, a first part without the gzip magic
|
||||
or a total over the context cap (bad_request), no Content-Digest
|
||||
(digest_required), a malformed one (bad_digest), or bytes that do
|
||||
not hash to it (digest_mismatch; the part was cut back off, so read
|
||||
where the upload stands and send it again).
|
||||
content:
|
||||
application/json:
|
||||
schema: { $ref: '#/components/schemas/Error' }
|
||||
$ref: '#/components/responses/BadRequest'
|
||||
'401':
|
||||
$ref: '#/components/responses/Unauthorized'
|
||||
'403':
|
||||
|
||||
+8
-63
@@ -3164,9 +3164,9 @@ for 10 seconds (the Free plan's limits).
|
||||
The panel's Files page is for the server's owner or an admin, and only while
|
||||
the server is fully stopped. Each call runs a one-shot `felis files` Job in the
|
||||
`minecraft` namespace, labelled `app.kubernetes.io/managed-by=felis-files` and
|
||||
`felis.lolicon.best/files-mode=<list|read|write|mkdir|delete|rename|upload|unzip>`.
|
||||
`felis.lolicon.best/files-mode=<list|read|write|mkdir|delete|rename|upload>`.
|
||||
A listing or a read holds nothing. Every change (a save, a new file or folder,
|
||||
a rename, a delete, an upload, an unzip) holds the world for its Job (§3b), so a wake or a
|
||||
a rename, a delete, an upload) holds the world for its Job (§3b), so a wake or a
|
||||
second change in the meantime gets `409 maintenance_in_progress`. The panel
|
||||
sends uploads one at a time and greys its other changes until they finish.
|
||||
|
||||
@@ -3178,19 +3178,13 @@ sends uploads one at a time and greys its other changes until they finish.
|
||||
| `409` | `file_changed` | The file changed after the editor read it | The editor offers to load the latest or overwrite it |
|
||||
| `400` | `bad_path` | The path leaves the world volume (`..`, an absolute path, a link pointing out), or it would move `server.properties`, `config` or `config/paper-global.yml`, or read `config/paper-global.yml` | Those three keep their names: the read path withholds their secrets by name, and `paper-global.yml` holds the proxy forwarding secret every server shares |
|
||||
| `404` | `not_found` | The path, or a new folder's parent, is gone | Refresh the listing |
|
||||
| `413` | `too_large` | A read over 1 MiB, a save over 256 KiB, or a one-request upload over 64 MiB | Upload a large file whole instead of editing it; the panel sends a file over 64 MiB in parts on its own |
|
||||
| `413` | `too_large` | A read over 1 MiB, a save over 256 KiB, or an upload over 64 MiB | Upload a large file whole instead of editing it |
|
||||
| `411` | `length_required` | An upload without `Content-Length` (a chunked body) | Upload from the panel, or with `curl -T`, which sends the length |
|
||||
| `400` | `upload_incomplete` | The body ended before its declared length | Retry; nothing was changed |
|
||||
| `507` | `upload_staging_full` | Staging this upload would leave felis-api's staging filesystem under 10% free | Free space on the uploads volume |
|
||||
| `507` | `volume_full` | The world volume ran out of space; the old file is left as it was | Delete files the server no longer needs, or grow its volume |
|
||||
| `504` | `files_timeout` | felis-api stopped waiting after 90 s | See below: the Job may still finish |
|
||||
| `503` | `files_unavailable` | felis-api runs without the file Job runner, or (for an upload) without a staging directory or its internal address | Check felis-api's startup log |
|
||||
| `404` | `upload_not_found` | A large upload's session is gone: cancelled, already landed, idle for 6 hours, or felis-api restarted | Upload the file again |
|
||||
| `409` | `upload_offset_mismatch` | A part did not start where the session ends (a lost answer, a second tab) | The panel reads where it stands and continues; nothing to do |
|
||||
| `409` | `upload_busy` | Another part of the same upload is still arriving | Same |
|
||||
| `413` | `part_too_large` | A part over 32 MiB, or past the size the upload began with | A client bug; upload from the panel |
|
||||
| `409` | `upload_incomplete` | The commit came before every part had arrived | Same |
|
||||
| `429` | `too_many_uploads` | The account already has 4 large uploads in progress | Finish or cancel one |
|
||||
|
||||
**An upload travels in two legs.** The browser sends the body to felis-api,
|
||||
which stages it under `/var/lib/felis/uploads/.file-staging` on the uploads
|
||||
@@ -3214,58 +3208,9 @@ change waits on it with `maintenance_in_progress`; refresh the listing once it
|
||||
is gone to see whether the change landed. A Job is kept for two minutes after
|
||||
it ends, with its log.
|
||||
|
||||
**A file over 64 MiB goes up in parts.** The panel begins an upload session
|
||||
(`POST …/files/uploads`), which reserves room for the whole file on the staging
|
||||
filesystem at once, so an upload that starts can finish (`507
|
||||
upload_staging_full` otherwise). It then sends 32 MiB parts, each under the
|
||||
Cloudflare edge's 100 MB body limit, retrying a part that fails and resuming
|
||||
from where the session ends; there is no size cap beyond the room. Starting the
|
||||
server midway costs only the commit, which needs it stopped again. The commit
|
||||
starts the Job and answers at once (`202`); the Job fetches the file the same
|
||||
way as above and runs up to 2 hours. A session belongs to the account and server
|
||||
it was begun for, and one untouched for 6 hours is dropped (felis-api logs
|
||||
`dropped N upload session(s) left idle`). A Job that fails before it has every
|
||||
byte leaves the session, so committing again does not mean sending it again.
|
||||
Folders cannot be uploaded: the panel asks for a `.zip` instead, because loose
|
||||
files cut off midway would leave half a world.
|
||||
|
||||
**Unzip** (`POST …/files/unzip`, `.zip` only) extracts into a hidden
|
||||
`.felis-unzip-*` folder beside the archive and moves the result into place only
|
||||
once every entry has been written and checked, so a failure changes nothing and
|
||||
the working folder is removed. It checks, before writing a byte, that no entry
|
||||
leaves the folder or is a link (`archive_unsafe`, `archive_symlink`), that the
|
||||
archive does not put a file where the server has a folder or the reverse
|
||||
(`type_conflict`), and that the volume has room (`volume_full`); an entry whose
|
||||
size differs from what the archive declares ends `archive_invalid`. Names stored
|
||||
in GBK, as Windows zips in a Chinese locale have them, are read as such. Without
|
||||
replace, an archive that would overwrite files ends `file_exists` with the list,
|
||||
which the panel shows for confirmation before running it again with replace.
|
||||
|
||||
Large uploads and unzips run in the background: they keep going when the page is
|
||||
closed, and `GET …/files/ops` lists the one running and those that ended in the
|
||||
last 30 minutes, with bytes done and, on failure, the code above or `job_failed`
|
||||
with the Job's condition (`DeadlineExceeded` after 2 hours). Their Jobs carry
|
||||
`felis.lolicon.best/files-async=true`. A Job that fails without a result keeps
|
||||
its log for 30 minutes:
|
||||
|
||||
```sh
|
||||
kubectl -n minecraft get jobs -l felis.lolicon.best/server=<name>,felis.lolicon.best/files-async=true
|
||||
```
|
||||
|
||||
**Downloading a file or folder** (`POST …/files/download`) is an export (§10,
|
||||
"Downloading a backup or the world"): a `felis-export` Job with
|
||||
`felis.lolicon.best/export-mode=files` reads the file, or zips the folder, from
|
||||
the world read-only and felis-api streams it to the browser. It needs the server
|
||||
stopped and holds the world until the download ends. `config/paper-global.yml`
|
||||
is refused as a file and left out of a folder, and `server.properties` goes out
|
||||
with `rcon.password` redacted, matched by the file itself so a link to either is
|
||||
guarded too; world and backup exports filter the same two files. Two per user at
|
||||
a time, four across the install, 30 per user per hour (`429 export_busy`).
|
||||
|
||||
Every change is audited as `file.write`, `file.mkdir`, `file.delete`,
|
||||
`file.rename` (with `to`), `file.upload` (with `size_bytes`, `sha256` and
|
||||
`overwrite`, in one request or in parts), `file.unzip` (with `overwrite`) or
|
||||
`file.download`, with `server_name` set to `<server>:<path>`:
|
||||
`file.rename` (with `to`) or `file.upload` (with `size_bytes`, `sha256` and
|
||||
`overwrite`), with `server_name` set to `<server>:<path>`:
|
||||
|
||||
```sh
|
||||
sudo k3s kubectl -n felis exec deploy/felis-postgres -c postgres -- psql -U postgres felis -c "
|
||||
@@ -3273,8 +3218,8 @@ sudo k3s kubectl -n felis exec deploy/felis-postgres -c postgres -- psql -U post
|
||||
FROM audit_logs WHERE action LIKE 'file.%' ORDER BY created_at DESC LIMIT 20;"
|
||||
```
|
||||
|
||||
[GO-TESTED: `internal/fileedit`, `handlers_files_test.go`, `handlers_fileops_test.go`,
|
||||
`cmd/felis/files_test.go`, `TestExpireFileSessions`, `internal/maintenance`.] [VM-TESTED: a pod labelled as a files Job in `minecraft`
|
||||
[GO-TESTED: `internal/fileedit`, `handlers_files_test.go`, `cmd/felis/files_test.go`,
|
||||
`internal/maintenance`.] [VM-TESTED: a pod labelled as a files Job in `minecraft`
|
||||
reaches `felis-api-internal:8081`; a 256 KiB save's content, split across six
|
||||
variables, lands byte for byte through the real binary, where one 140 KB variable
|
||||
fails with `argument list too long`. An upload through the API, both legs end to
|
||||
@@ -3386,5 +3331,5 @@ PG-TESTED: `TestScheduleStoreRunCAS`, `TestDueSchedules`, `TestSchedulesFollowTh
|
||||
| `FelisAuditWriteFailing` | §17 |
|
||||
| `felis breakGlass` sends no code / shows `Root override`; `otp_skipped` in the audit | §17 |
|
||||
| How long sessions, codes and audit rows are kept; export audit rows | §17 |
|
||||
| Files page: a change, upload or unzip refused (`file_exists`, `bad_path`, `too_large`, `upload_staging_full`, `upload_not_found`, `volume_full`, `archive_unsafe`, `job_failed`, `files_timeout`) | §18 |
|
||||
| Files page: a change or upload refused (`file_exists`, `bad_path`, `too_large`, `upload_staging_full`, `volume_full`, `files_timeout`) | §18 |
|
||||
| A scheduled task shows `skipped`, `missed` or `failed`; a task switched itself off after an owner change | §19 |
|
||||
@@ -16,7 +16,6 @@ require (
|
||||
github.com/minio/minio-go/v7 v7.2.1
|
||||
github.com/prometheus/client_golang v1.19.1
|
||||
github.com/prometheus/client_model v0.6.1
|
||||
golang.org/x/text v0.39.0
|
||||
k8s.io/api v0.31.3
|
||||
k8s.io/apimachinery v0.31.3
|
||||
k8s.io/client-go v0.31.0
|
||||
@@ -96,10 +95,11 @@ require (
|
||||
golang.org/x/crypto v0.52.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
|
||||
golang.org/x/net v0.55.0 // indirect
|
||||
golang.org/x/oauth2 v0.27.0 // indirect
|
||||
golang.org/x/oauth2 v0.21.0 // indirect
|
||||
golang.org/x/sync v0.21.0 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
golang.org/x/term v0.43.0 // indirect
|
||||
golang.org/x/text v0.39.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
|
||||
google.golang.org/protobuf v1.36.10 // indirect
|
||||
|
||||
@@ -249,8 +249,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
|
||||
golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
|
||||
golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
|
||||
golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
||||
+3
-18
@@ -464,12 +464,10 @@ func (a *API) internalAPIRoutes() []apiRoute {
|
||||
// Principal); the shared enqueueBackup tail enforces the RWO stopped-gate.
|
||||
{Method: "POST", Pattern: "/api/v1/internal/servers/{name}/backup", Callers: ops, h: a.handleInternalBackup},
|
||||
|
||||
// A file upload's staged bytes, fetched once by the Job landing them, which
|
||||
// then reports them landed so a file sent in parts is deleted. Public
|
||||
// A file upload's staged bytes, fetched once by the Job landing them. Public
|
||||
// because that Job holds no service token; the one-time bearer token minted
|
||||
// with the upload is the check (handlers_files.go).
|
||||
{Method: "GET", Pattern: "/api/v1/internal/file-uploads/{id}", Public: true, h: a.handleInternalFileUpload},
|
||||
{Method: "DELETE", Pattern: "/api/v1/internal/file-uploads/{id}", Public: true, h: a.handleInternalFileUploadLanded},
|
||||
|
||||
// An export Job's archive, held open until the owner's browser downloads
|
||||
// it. Public for the same reason as file uploads: the Job holds no service
|
||||
@@ -584,8 +582,8 @@ func (a *API) externalAPIRoutes() []apiRoute {
|
||||
{Method: "POST", Pattern: "/api/v1/servers/{name}/world/export", h: a.handleExportWorld},
|
||||
{Method: "GET", Pattern: "/api/v1/exports/{ticket}", h: a.handleExportStatus},
|
||||
{Method: "GET", Pattern: "/api/v1/exports/{ticket}/download", h: a.handleExportDownload},
|
||||
// Server file manager: list, read, write, make a folder, delete, rename,
|
||||
// upload, unzip and download in a STOPPED server's world volume (handlers_files.go). App-tier,
|
||||
// Server file manager: list, read, write, make a folder, delete, rename and
|
||||
// upload in a STOPPED server's world volume (handlers_files.go). App-tier,
|
||||
// exactly like the backup pair above and for the same reason — every route
|
||||
// gates on owner-or-admin inside the handler, so an owner repairs their own
|
||||
// broken server without an admin's Zero-Trust path. The path travels as ?path=
|
||||
@@ -607,19 +605,6 @@ func (a *API) externalAPIRoutes() []apiRoute {
|
||||
{Method: "POST", Pattern: "/api/v1/servers/{name}/files/mkdir", h: a.handleMkdir},
|
||||
{Method: "POST", Pattern: "/api/v1/servers/{name}/files/rename", h: a.handleRenameFile},
|
||||
{Method: "PUT", Pattern: "/api/v1/servers/{name}/files/upload", h: a.handleUploadFile},
|
||||
// A file too big for one request goes up in parts as an upload session, and
|
||||
// lands, like an unzip, as a Job the request does not wait on; files/ops
|
||||
// reports how those went (handlers_fileops.go).
|
||||
{Method: "POST", Pattern: "/api/v1/servers/{name}/files/uploads", h: a.handleBeginFileUpload},
|
||||
{Method: "GET", Pattern: "/api/v1/servers/{name}/files/uploads/{id}", h: a.handleFileUploadStatus},
|
||||
{Method: "PUT", Pattern: "/api/v1/servers/{name}/files/uploads/{id}", h: a.handleFileUploadPart},
|
||||
{Method: "DELETE", Pattern: "/api/v1/servers/{name}/files/uploads/{id}", h: a.handleDropFileUpload},
|
||||
{Method: "POST", Pattern: "/api/v1/servers/{name}/files/uploads/{id}/commit", h: a.handleCommitFileUpload},
|
||||
{Method: "POST", Pattern: "/api/v1/servers/{name}/files/unzip", h: a.handleUnzipFile},
|
||||
{Method: "GET", Pattern: "/api/v1/servers/{name}/files/ops", h: a.handleListFileOps},
|
||||
// A file or folder download is an export (exports.go): it answers 202
|
||||
// with a ticket the export routes above serve.
|
||||
{Method: "POST", Pattern: "/api/v1/servers/{name}/files/download", h: a.handleDownloadFile},
|
||||
// Scheduled tasks (handlers_schedules.go): a console command, restart, stop,
|
||||
// start or backup at set times, which felis-api's runner fires. App-tier and
|
||||
// owner-or-admin inside the handler, like the console and power routes they
|
||||
|
||||
+98
-276
@@ -8,33 +8,30 @@ import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"log"
|
||||
"mime"
|
||||
"net/http"
|
||||
pathpkg "path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
||||
"felis.lolicon.best/internal/fileedit"
|
||||
"felis.lolicon.best/internal/maintenance"
|
||||
"felis.lolicon.best/internal/naming"
|
||||
"felis.lolicon.best/internal/worldexport"
|
||||
)
|
||||
|
||||
// World export and file download. The owner downloads a tar.gz of their world,
|
||||
// either as it is now (the server stopped) or as one of its backups, or one file
|
||||
// or folder of a stopped world (a folder as a zip), straight into the browser:
|
||||
// World export. The owner downloads a tar.gz of their world, either as it is
|
||||
// now (the server stopped) or as one of its backups, straight into the browser:
|
||||
//
|
||||
// 1. POST /servers/{name}/world/export, /servers/{name}/backups/{id}/export or
|
||||
// /servers/{name}/files/download checks the caller and the server, admits
|
||||
// the export against the limits below and starts a one-shot felis-export
|
||||
// Job (internal/worldexport) that reads the world or the archive read-only.
|
||||
// It answers 202 with a ticket: 256 random bits, good for the caller who
|
||||
// started it and nobody else.
|
||||
// 1. POST /servers/{name}/world/export or /servers/{name}/backups/{id}/export
|
||||
// checks the caller and the server, admits the export against the limits
|
||||
// below and starts a one-shot felis-export Job (internal/worldexport) that
|
||||
// reads the world or the archive read-only. It answers 202 with a ticket:
|
||||
// 256 random bits, good for the caller who started it and nobody else.
|
||||
// 2. The Job PUTs the archive to the internal face (PUT
|
||||
// /api/v1/internal/exports/{id} with the one-time token it was started
|
||||
// with), and that request waits, body unread, for the browser.
|
||||
@@ -44,80 +41,31 @@ import (
|
||||
// archive passes through felis-api's memory once and never touches a disk
|
||||
// it owns, and the Job moves at the browser's pace.
|
||||
//
|
||||
// Nothing on the way is trusted to deliver the bytes intact. The Job hashes
|
||||
// what it sends and ends its chunked PUT with the SHA-256 as a trailer;
|
||||
// felis-api hashes what it receives and holds the last buffer back from the
|
||||
// browser until the two agree (relayExport), so a download whose bytes changed
|
||||
// between the Job and here fails in the browser rather than lands complete.
|
||||
//
|
||||
// A backup is checked by the Job against the sha256 recorded when it was
|
||||
// written (cmd/felis export): on a mismatch the Job aborts its upload short of
|
||||
// the archive's end, the download aborts with it, and the browser reports a
|
||||
// failed download and never keeps a complete-looking corrupt file.
|
||||
// A backup is checked against the sha256 recorded when it was written as it
|
||||
// streams, and the last read is held back until the digest is known: a mismatch
|
||||
// aborts the response, so the browser reports a failed download and never keeps
|
||||
// a complete-looking corrupt file, and the Job is told backup_corrupt.
|
||||
//
|
||||
// Tickets live in felis-api's memory. A restart forgets them, and a Job that
|
||||
// then PUTs finds nothing and fails, which the jobs list shows.
|
||||
|
||||
// Exporter starts the Job that archives a world or a backup and hands it to the
|
||||
// internal upload route (internal/worldexport). Optional: when nil the export
|
||||
// routes answer 503. Stop deletes a Job felis-api has given up on.
|
||||
// routes answer 503.
|
||||
type Exporter interface {
|
||||
Start(ctx context.Context, r worldexport.Request) (job string, err error)
|
||||
Stop(ctx context.Context, job string) error
|
||||
}
|
||||
|
||||
// Limits on exports. Each one keeps a Job, a connection and a 64 KiB copy
|
||||
// buffer alive for as long as a download takes, and a world export or a file
|
||||
// download also keeps its server from starting.
|
||||
// buffer alive for as long as a download takes, and a world export also keeps
|
||||
// its server from starting.
|
||||
const (
|
||||
exportMaxActive = 2 // admitted and not yet over, install-wide
|
||||
exportMaxPerUser = 1
|
||||
exportPerHour = 6 // started by one user in any hour
|
||||
exportCopyBuffer = 32 << 10
|
||||
|
||||
// File downloads count apart from the world and backup exports, with
|
||||
// their own limits: one is a file or a folder, usually small and over in
|
||||
// seconds, and an owner fetching a few configs one after another must
|
||||
// neither wait on an export nor hold one off.
|
||||
fileExportMaxActive = 4
|
||||
fileExportMaxPerUser = 2
|
||||
fileExportPerHour = 30
|
||||
)
|
||||
|
||||
// exportClass is one set of export limits and how a refusal names them.
|
||||
type exportClass struct {
|
||||
maxActive, perUser, perHour int
|
||||
busyUser, busyActive, busyHour string // each formats its limit
|
||||
}
|
||||
|
||||
var (
|
||||
worldExports = exportClass{
|
||||
maxActive: exportMaxActive, perUser: exportMaxPerUser, perHour: exportPerHour,
|
||||
busyUser: "you already have %d export in progress; download it or let it expire first",
|
||||
busyActive: "%d exports are already in progress; retry in a few minutes",
|
||||
busyHour: "you have started %d exports in the last hour; retry later",
|
||||
}
|
||||
fileExports = exportClass{
|
||||
maxActive: fileExportMaxActive, perUser: fileExportMaxPerUser, perHour: fileExportPerHour,
|
||||
busyUser: "you already have %d file downloads in progress; let one finish first",
|
||||
busyActive: "%d file downloads are already in progress; retry in a minute",
|
||||
busyHour: "you have started %d file downloads in the last hour; retry later",
|
||||
}
|
||||
)
|
||||
|
||||
func (e *exportEntry) class() exportClass {
|
||||
if e.files {
|
||||
return fileExports
|
||||
}
|
||||
return worldExports
|
||||
}
|
||||
|
||||
// exportStarts keys a user's recent starts within one class.
|
||||
type exportStarts struct {
|
||||
userID string
|
||||
files bool
|
||||
}
|
||||
|
||||
// Timings. Vars only so a test can shrink them.
|
||||
var (
|
||||
// exportClaimTTL is how long the Job's upload waits for the browser.
|
||||
@@ -154,6 +102,8 @@ type exportStatusView struct {
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
var errExportDigest = errors.New("the archive does not match the sha256 recorded when it was written")
|
||||
|
||||
func errExportExpired() error {
|
||||
return newError(http.StatusGone, "export_expired", "this export has expired or was already downloaded; start a new one")
|
||||
}
|
||||
@@ -169,11 +119,8 @@ type exportEntry struct {
|
||||
userID string
|
||||
server string
|
||||
mode string
|
||||
files bool // a file download, counted in fileExports
|
||||
sha256 string // what a backup must hash to; empty for a world
|
||||
filename string
|
||||
// contentType is what the download is served as: a gzip archive, a zip,
|
||||
// or a single file's raw bytes.
|
||||
contentType string
|
||||
job string
|
||||
state string
|
||||
message string
|
||||
@@ -184,9 +131,7 @@ type exportEntry struct {
|
||||
// exportUpload is the Job's PUT, parked until a browser claims it.
|
||||
type exportUpload struct {
|
||||
body io.Reader
|
||||
size int64 // what the Job declared (worldexport.LengthHeader); -1 when it did not
|
||||
// digest reads the Job's trailer, which is there once body has ended.
|
||||
digest func() []string
|
||||
size int64 // -1 when the Job streams it chunked
|
||||
claimed chan struct{}
|
||||
done chan error // how the download ended; buffered
|
||||
}
|
||||
@@ -199,10 +144,7 @@ type exportRegistry struct {
|
||||
mu sync.Mutex
|
||||
byTicket map[string]*exportEntry
|
||||
byID map[string]*exportEntry
|
||||
starts map[exportStarts][]time.Time // oldest first
|
||||
// stop deletes the Job of an export the sweep expired while it was
|
||||
// pending, and is run apart from the lock.
|
||||
stop func(job string)
|
||||
starts map[string][]time.Time // per user, oldest first
|
||||
}
|
||||
|
||||
func (a *API) exportTickets() *exportRegistry {
|
||||
@@ -210,42 +152,18 @@ func (a *API) exportTickets() *exportRegistry {
|
||||
a.exports = &exportRegistry{
|
||||
byTicket: map[string]*exportEntry{},
|
||||
byID: map[string]*exportEntry{},
|
||||
starts: map[exportStarts][]time.Time{},
|
||||
stop: a.stopExportJob,
|
||||
starts: map[string][]time.Time{},
|
||||
}
|
||||
})
|
||||
return a.exports
|
||||
}
|
||||
|
||||
// ExpireExports runs the export sweep on its own, for felis-api's loop: the
|
||||
// routes sweep as they are called, and an owner who closed the tab calls none.
|
||||
func (a *API) ExpireExports() {
|
||||
g := a.exportTickets()
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.sweepLocked(a.now())
|
||||
}
|
||||
|
||||
// stopExportJob deletes one export Job. A delete that fails leaves the Job to
|
||||
// its own deadline.
|
||||
func (a *API) stopExportJob(job string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := a.Exporter.Stop(ctx, job); err != nil {
|
||||
log.Printf("api: stop export job %s, which never connected: %v", job, err)
|
||||
}
|
||||
}
|
||||
|
||||
// sweepLocked expires what has waited too long and forgets what ended long ago.
|
||||
// A Job still pending at its TTL never connected: its Pod is stuck unscheduled
|
||||
// or pulling, and until its deadline it would keep the server from starting,
|
||||
// so it is deleted.
|
||||
func (g *exportRegistry) sweepLocked(now time.Time) {
|
||||
for t, e := range g.byTicket {
|
||||
switch {
|
||||
case e.state == exportPending && now.Sub(e.at) >= exportPendingTTL:
|
||||
e.state, e.at = exportSpent, now
|
||||
go g.stop(e.job)
|
||||
case !e.active() && now.Sub(e.at) >= exportKeepSpent:
|
||||
delete(g.byTicket, t)
|
||||
delete(g.byID, e.id)
|
||||
@@ -269,37 +187,38 @@ func randomHex(n int) string {
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// admit reserves the export e describes (its user, server, mode, class,
|
||||
// filename and content type) and returns its upload token, or refuses it with
|
||||
// export_busy. Only exports of e's class count against it.
|
||||
// admit reserves the export e describes (its user, server, mode, filename and
|
||||
// digest) and returns its upload token, or refuses it with export_busy.
|
||||
func (g *exportRegistry) admit(e *exportEntry, now time.Time) (string, error) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.sweepLocked(now)
|
||||
active, mine := 0, 0
|
||||
for _, o := range g.byTicket {
|
||||
if o.active() && o.files == e.files {
|
||||
if o.active() {
|
||||
active++
|
||||
if o.userID == e.userID {
|
||||
mine++
|
||||
}
|
||||
}
|
||||
}
|
||||
c, key := e.class(), exportStarts{e.userID, e.files}
|
||||
switch starts := g.starts[key]; {
|
||||
case mine >= c.perUser:
|
||||
return "", newError(http.StatusTooManyRequests, "export_busy", c.busyUser, c.perUser).retryAfter(exportClaimTTL)
|
||||
case active >= c.maxActive:
|
||||
return "", newError(http.StatusTooManyRequests, "export_busy", c.busyActive, c.maxActive).retryAfter(time.Minute)
|
||||
case len(starts) >= c.perHour:
|
||||
return "", newError(http.StatusTooManyRequests, "export_busy", c.busyHour, c.perHour).retryAfter(starts[0].Add(time.Hour).Sub(now))
|
||||
switch starts := g.starts[e.userID]; {
|
||||
case mine >= exportMaxPerUser:
|
||||
return "", newError(http.StatusTooManyRequests, "export_busy",
|
||||
"you already have an export in progress; download it or let it expire first").retryAfter(exportClaimTTL)
|
||||
case active >= exportMaxActive:
|
||||
return "", newError(http.StatusTooManyRequests, "export_busy",
|
||||
"%d exports are already in progress; retry in a few minutes", exportMaxActive).retryAfter(time.Minute)
|
||||
case len(starts) >= exportPerHour:
|
||||
return "", newError(http.StatusTooManyRequests, "export_busy",
|
||||
"you have started %d exports in the last hour; retry later", exportPerHour).retryAfter(starts[0].Add(time.Hour).Sub(now))
|
||||
}
|
||||
token := randomHex(32)
|
||||
e.ticket, e.id, e.tokenHash = randomHex(32), randomHex(8), sha256.Sum256([]byte(token))
|
||||
e.state, e.at = exportPending, now
|
||||
g.byTicket[e.ticket] = e
|
||||
g.byID[e.id] = e
|
||||
g.starts[key] = append(g.starts[key], now)
|
||||
g.starts[e.userID] = append(g.starts[e.userID], now)
|
||||
return token, nil
|
||||
}
|
||||
|
||||
@@ -309,15 +228,10 @@ func (g *exportRegistry) drop(e *exportEntry) {
|
||||
defer g.mu.Unlock()
|
||||
delete(g.byTicket, e.ticket)
|
||||
delete(g.byID, e.id)
|
||||
key := exportStarts{e.userID, e.files}
|
||||
ts := g.starts[key]
|
||||
ts := g.starts[e.userID]
|
||||
for i := len(ts) - 1; i >= 0; i-- {
|
||||
if ts[i].Equal(e.at) {
|
||||
if rest := append(ts[:i:i], ts[i+1:]...); len(rest) > 0 {
|
||||
g.starts[key] = rest
|
||||
} else {
|
||||
delete(g.starts, key)
|
||||
}
|
||||
g.starts[e.userID] = append(ts[:i:i], ts[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -463,13 +377,13 @@ func (a *API) handleExportBackup(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
e := &exportEntry{userID: p.UserID, server: name, mode: worldexport.ModeBackup,
|
||||
filename: fmt.Sprintf("%s-backup-%s.tar.gz", name, backup.ID), contentType: archiveContentType}
|
||||
filename: fmt.Sprintf("%s-backup-%s.tar.gz", name, backup.ID), sha256: backup.SHA256}
|
||||
token, err := a.exportTickets().admit(e, a.now())
|
||||
if err != nil {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
if !a.startExport(w, r, e, token, worldexport.Request{BackupRef: backup.BackupRef, BackupSHA256: backup.SHA256}) {
|
||||
if !a.startExport(w, r, e, token, backup.BackupRef) {
|
||||
return
|
||||
}
|
||||
a.auditEntry(r, AuditEntry{Actor: auditActor(p), ActorUserID: p.UserID, Action: "backup.export", ServerName: rec.Name,
|
||||
@@ -511,8 +425,7 @@ func (a *API) handleExportWorld(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
reg := a.exportTickets()
|
||||
e := &exportEntry{userID: p.UserID, server: name, mode: worldexport.ModeWorld,
|
||||
filename: fmt.Sprintf("%s-world-%s.tar.gz", name, a.now().UTC().Format("20060102-150405")),
|
||||
contentType: archiveContentType}
|
||||
filename: fmt.Sprintf("%s-world-%s.tar.gz", name, a.now().UTC().Format("20060102-150405"))}
|
||||
token, err := reg.admit(e, a.now())
|
||||
if err != nil {
|
||||
writeError(w, r, err)
|
||||
@@ -524,79 +437,15 @@ func (a *API) handleExportWorld(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
if !a.startExport(w, r, e, token, worldexport.Request{}) {
|
||||
if !a.startExport(w, r, e, token, "") {
|
||||
return
|
||||
}
|
||||
a.auditEntry(r, AuditEntry{Actor: auditActor(p), ActorUserID: p.UserID, Action: "world.export", ServerName: rec.Name})
|
||||
writeJSON(w, http.StatusAccepted, exportTicketView{Ticket: e.ticket, State: exportPending, Filename: e.filename})
|
||||
}
|
||||
|
||||
// archiveContentType is how a world or a backup export is served.
|
||||
const archiveContentType = "application/gzip"
|
||||
|
||||
// handleDownloadFile starts the download of one file or folder of a stopped
|
||||
// server's world (POST /api/v1/servers/{name}/files/download?path=…&dir=true
|
||||
// for a folder). The gate is the file manager's (authorizeFileOp), plus an
|
||||
// account to bind the ticket to. The download is an export: a felis-export Job
|
||||
// in files mode reads the file, or zips the folder, from the world volume and
|
||||
// hands it over through a ticket like any other, under the world-volume lock
|
||||
// (as KindExport) so the server cannot start mid-zip.
|
||||
//
|
||||
// The path is passed to the Job as it came, as every file route does (see
|
||||
// handleListFiles): the Job's os.Root is the containment, and the guards run
|
||||
// there. Only the world root is refused here, since a whole world is what the
|
||||
// world export is for.
|
||||
func (a *API) handleDownloadFile(w http.ResponseWriter, r *http.Request) {
|
||||
name, ok := a.authorizeFileOp(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
p := principalFromContext(r.Context())
|
||||
if p.UserID == "" {
|
||||
writeError(w, r, errForbidden)
|
||||
return
|
||||
}
|
||||
path, ok := requirePath(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
dir := r.URL.Query().Get("dir") == "true"
|
||||
base := pathpkg.Base(pathpkg.Clean("/" + path))
|
||||
if base == "/" {
|
||||
writeError(w, r, newError(http.StatusBadRequest, "bad_path",
|
||||
"the whole world is not a file download; export the world from the backups page instead"))
|
||||
return
|
||||
}
|
||||
if a.Exporter == nil || a.InternalBaseURL == "" {
|
||||
writeError(w, r, errExportUnavailable())
|
||||
return
|
||||
}
|
||||
e := &exportEntry{userID: p.UserID, server: name, mode: worldexport.ModeFiles, files: true,
|
||||
filename: base, contentType: fileedit.DownloadFileType}
|
||||
if dir {
|
||||
e.filename, e.contentType = base+".zip", fileedit.DownloadZipType
|
||||
}
|
||||
reg := a.exportTickets()
|
||||
token, err := reg.admit(e, a.now())
|
||||
if err != nil {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
release, ok := a.acquireWorld(w, r, name, maintenance.KindExport, "stop the server before downloading its files")
|
||||
if !ok {
|
||||
reg.drop(e)
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
if !a.startExport(w, r, e, token, worldexport.Request{Path: path, Dir: dir}) {
|
||||
return
|
||||
}
|
||||
a.auditFile(r, "file.download", name, path, map[string]any{"dir": dir})
|
||||
writeJSON(w, http.StatusAccepted, exportTicketView{Ticket: e.ticket, State: exportPending, Filename: e.filename})
|
||||
}
|
||||
|
||||
func errExportUnavailable() error {
|
||||
return newError(http.StatusServiceUnavailable, "export_unavailable", "world export and downloads are not configured")
|
||||
return newError(http.StatusServiceUnavailable, "export_unavailable", "world export is not configured")
|
||||
}
|
||||
|
||||
// exportGate is the front half both export routes share: a valid name, a known
|
||||
@@ -622,12 +471,12 @@ func (a *API) exportGate(w http.ResponseWriter, r *http.Request) (string, *Serve
|
||||
}
|
||||
|
||||
// startExport creates the admitted export's Job, or forgets the export and
|
||||
// writes the error. what carries the mode's own fields (the backup and its
|
||||
// digest, or the path); the rest comes from e.
|
||||
func (a *API) startExport(w http.ResponseWriter, r *http.Request, e *exportEntry, token string, what worldexport.Request) bool {
|
||||
what.Server, what.Mode, what.ID, what.Token = e.server, e.mode, e.id, token
|
||||
what.TargetURL = a.InternalBaseURL + "/api/v1/internal/exports/" + e.id
|
||||
job, err := a.Exporter.Start(r.Context(), what)
|
||||
// writes the error.
|
||||
func (a *API) startExport(w http.ResponseWriter, r *http.Request, e *exportEntry, token, backupRef string) bool {
|
||||
job, err := a.Exporter.Start(r.Context(), worldexport.Request{
|
||||
Server: e.server, Mode: e.mode, BackupRef: backupRef, ID: e.id, Token: token,
|
||||
TargetURL: a.InternalBaseURL + "/api/v1/internal/exports/" + e.id,
|
||||
})
|
||||
if err != nil {
|
||||
a.exportTickets().drop(e)
|
||||
writeError(w, r, err)
|
||||
@@ -684,7 +533,7 @@ func (a *API) handleExportDownload(w http.ResponseWriter, r *http.Request) {
|
||||
defer reg.finish(e.ticket, a.now())
|
||||
|
||||
h := w.Header()
|
||||
h.Set("Content-Type", e.contentType)
|
||||
h.Set("Content-Type", "application/gzip")
|
||||
if cd := mime.FormatMediaType("attachment", map[string]string{"filename": e.filename}); cd != "" {
|
||||
h.Set("Content-Disposition", cd)
|
||||
} else {
|
||||
@@ -697,7 +546,7 @@ func (a *API) handleExportDownload(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
n, sum, err := relayExport(w, e.upload)
|
||||
err = copyExport(w, e.upload.body, e.sha256)
|
||||
e.upload.done <- err
|
||||
if err != nil {
|
||||
log.Printf("api: export %s of %s ended early: %v", e.id, e.server, err)
|
||||
@@ -705,77 +554,56 @@ func (a *API) handleExportDownload(w http.ResponseWriter, r *http.Request) {
|
||||
// read as failed in the browser, never as a complete file.
|
||||
panic(http.ErrAbortHandler)
|
||||
}
|
||||
log.Printf("api: export %s of %s sent %d bytes, sha256 %s", e.id, e.server, n, sum)
|
||||
}
|
||||
|
||||
var (
|
||||
errExportDigest = errors.New("the bytes the export Job sent do not match the SHA-256 it sent with them")
|
||||
errExportLength = errors.New("the export Job sent a different number of bytes than it declared")
|
||||
)
|
||||
|
||||
// relayExport copies the Job's upload into the download through fixed 32 KiB
|
||||
// buffers, restarting w's write deadline on every write, and checks it on the
|
||||
// way: the bytes must hash to the SHA-256 the Job's trailer carries once it has
|
||||
// sent them all (worldexport.DigestTrailer), and number what it declared, when
|
||||
// it did. The latest buffer read is held back until both hold, so bytes changed
|
||||
// between the Job and here, or a Job that sent no digest, end the download
|
||||
// short of its end, and the browser reports it failed rather than keep a
|
||||
// complete-looking corrupt file. It returns the bytes sent and their SHA-256.
|
||||
func relayExport(w http.ResponseWriter, up *exportUpload) (int64, string, error) {
|
||||
out := &stallWriter{w: w, rc: http.NewResponseController(w)}
|
||||
h := sha256.New()
|
||||
var n int64
|
||||
next, spare := make([]byte, exportCopyBuffer), make([]byte, exportCopyBuffer)
|
||||
var held []byte
|
||||
for {
|
||||
k, err := up.body.Read(next)
|
||||
if k > 0 {
|
||||
if len(held) > 0 {
|
||||
if _, werr := out.Write(held); werr != nil {
|
||||
return n, "", werr
|
||||
// copyExport copies body into w through a fixed 32 KiB buffer, restarting w's
|
||||
// write deadline on every write. With want set, the last read is held back
|
||||
// until the whole body hashes to it.
|
||||
func copyExport(w http.ResponseWriter, body io.Reader, want string) error {
|
||||
out := &heldWriter{w: w, rc: http.NewResponseController(w)}
|
||||
var sum hash.Hash
|
||||
if want != "" {
|
||||
sum = sha256.New()
|
||||
body = io.TeeReader(body, sum)
|
||||
}
|
||||
if _, err := io.CopyBuffer(out, body, make([]byte, exportCopyBuffer)); err != nil {
|
||||
return err
|
||||
}
|
||||
h.Write(next[:k])
|
||||
n += int64(k)
|
||||
held = next[:k]
|
||||
next, spare = spare, next
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return n, "", err
|
||||
}
|
||||
}
|
||||
want, err := parseContentDigest(up.digest())
|
||||
switch {
|
||||
case up.size >= 0 && n != up.size:
|
||||
return n, "", fmt.Errorf("%w: %d of %d", errExportLength, n, up.size)
|
||||
case err != nil:
|
||||
return n, "", fmt.Errorf("%w: %v", errExportDigest, err)
|
||||
case subtle.ConstantTimeCompare(h.Sum(nil), want) != 1:
|
||||
return n, "", errExportDigest
|
||||
}
|
||||
if len(held) > 0 {
|
||||
if _, err := out.Write(held); err != nil {
|
||||
return n, "", err
|
||||
if sum != nil && !strings.EqualFold(hex.EncodeToString(sum.Sum(nil)), want) {
|
||||
return errExportDigest
|
||||
}
|
||||
if err := out.flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = out.rc.SetWriteDeadline(time.Time{}) // the connection may serve another request
|
||||
return n, hex.EncodeToString(h.Sum(nil)), nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// stallWriter restarts the connection's write deadline before every write, so
|
||||
// a write fails only once the browser has taken nothing for exportStall. It
|
||||
// has no ReadFrom, so io.CopyBuffer uses the buffer it is given.
|
||||
type stallWriter struct {
|
||||
// heldWriter passes each write on one behind, keeping the latest back until
|
||||
// flush, so the end of an archive reaches the browser only once it is checked.
|
||||
// It has no ReadFrom, so io.CopyBuffer uses the buffer it is given.
|
||||
type heldWriter struct {
|
||||
w io.Writer
|
||||
rc *http.ResponseController
|
||||
held []byte
|
||||
}
|
||||
|
||||
func (s *stallWriter) Write(p []byte) (int, error) {
|
||||
_ = s.rc.SetWriteDeadline(time.Now().Add(exportStall))
|
||||
return s.w.Write(p)
|
||||
func (h *heldWriter) Write(p []byte) (int, error) {
|
||||
if err := h.flush(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
h.held = append(h.held[:0], p...)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (h *heldWriter) flush() error {
|
||||
if len(h.held) == 0 {
|
||||
return nil
|
||||
}
|
||||
_ = h.rc.SetWriteDeadline(time.Now().Add(exportStall))
|
||||
_, err := h.w.Write(h.held)
|
||||
h.held = h.held[:0]
|
||||
return err
|
||||
}
|
||||
|
||||
// handleInternalExportUpload takes an export Job's archive (PUT
|
||||
@@ -783,27 +611,18 @@ func (s *stallWriter) Write(p []byte) (int, error) {
|
||||
// export is the check, as for file uploads: an unknown id, a wrong token and a
|
||||
// token already used all read the same 404. The request then waits for the
|
||||
// browser, up to exportClaimTTL, and answers once the download has ended: 204
|
||||
// when it got the whole archive, 410 export_expired when nobody came for it,
|
||||
// the browser left early or the Job itself cut the upload short.
|
||||
// when it got the whole archive, 409 backup_corrupt when the archive did not
|
||||
// match its recorded digest, 410 export_expired when nobody came for it or the
|
||||
// browser left early.
|
||||
func (a *API) handleInternalExportUpload(w http.ResponseWriter, r *http.Request) {
|
||||
token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
|
||||
if !ok {
|
||||
writeError(w, r, errNoExport())
|
||||
return
|
||||
}
|
||||
size := int64(-1)
|
||||
if v := r.Header.Get(worldexport.LengthHeader); v != "" {
|
||||
n, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil || n < 0 {
|
||||
writeError(w, r, newError(http.StatusBadRequest, "bad_request", "%s must be a byte count", worldexport.LengthHeader))
|
||||
return
|
||||
}
|
||||
size = n
|
||||
}
|
||||
rc := takeBodyDeadline(w, r)
|
||||
up := &exportUpload{
|
||||
body: &stallBody{r: r.Body, rc: rc}, size: size,
|
||||
digest: func() []string { return r.Trailer.Values(worldexport.DigestTrailer) },
|
||||
body: &stallBody{r: r.Body, rc: rc}, size: r.ContentLength,
|
||||
claimed: make(chan struct{}), done: make(chan error, 1),
|
||||
}
|
||||
reg := a.exportTickets()
|
||||
@@ -825,11 +644,14 @@ func (a *API) handleInternalExportUpload(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := <-up.done; err != nil {
|
||||
writeError(w, r, newError(http.StatusGone, "export_expired", "the download ended before the archive did: %v", err))
|
||||
return
|
||||
}
|
||||
switch err := <-up.done; {
|
||||
case err == nil:
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
case errors.Is(err, errExportDigest):
|
||||
writeError(w, r, newError(http.StatusConflict, "backup_corrupt", "%v", err))
|
||||
default:
|
||||
writeError(w, r, newError(http.StatusGone, "export_expired", "the download ended before the archive did: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// stallBody restarts the connection's read deadline on every read, so reading
|
||||
|
||||
+85
-425
@@ -4,12 +4,13 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -18,7 +19,6 @@ import (
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
||||
"felis.lolicon.best/internal/fileedit"
|
||||
"felis.lolicon.best/internal/maintenance"
|
||||
"felis.lolicon.best/internal/worldexport"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
@@ -31,14 +31,6 @@ import (
|
||||
type fakeExporter struct {
|
||||
reqs []worldexport.Request
|
||||
err error
|
||||
// stopped receives each Job Stop is asked to delete; the sweep asks from
|
||||
// a goroutine of its own.
|
||||
stopped chan string
|
||||
}
|
||||
|
||||
func (f *fakeExporter) Stop(_ context.Context, job string) error {
|
||||
f.stopped <- job
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeExporter) Start(_ context.Context, r worldexport.Request) (string, error) {
|
||||
@@ -100,7 +92,7 @@ func exportFixture() (*API, *fakeRepo, *fakeCluster, *fakeExporter) {
|
||||
for _, name := range []string{"survival", "gamma"} {
|
||||
cl.byName[name] = &ServerInfo{Name: name, Phase: "Stopped", DesiredState: string(v1alpha1.DesiredStopped)}
|
||||
}
|
||||
ex := &fakeExporter{stopped: make(chan string, 64)}
|
||||
ex := &fakeExporter{}
|
||||
a := newTestAPI(repo, cl)
|
||||
a.External = exportUsers
|
||||
a.Exporter, a.InternalBaseURL = ex, exportBase
|
||||
@@ -148,15 +140,11 @@ func waitExportReady(t *testing.T, h http.Handler, ticket, user string) {
|
||||
}
|
||||
|
||||
// uploadExport serves the Job's PUT on the internal face in the background.
|
||||
// digest, when set, is the trailer the Job sends once its body has ended.
|
||||
func uploadExport(h http.Handler, id, token string, body io.Reader, digest string) <-chan *httptest.ResponseRecorder {
|
||||
func uploadExport(h http.Handler, id, token string, body io.Reader) <-chan *httptest.ResponseRecorder {
|
||||
out := make(chan *httptest.ResponseRecorder, 1)
|
||||
go func() {
|
||||
r := httptest.NewRequest("PUT", "/api/v1/internal/exports/"+id, body)
|
||||
r.Header.Set("Authorization", "Bearer "+token)
|
||||
if digest != "" {
|
||||
r.Trailer = http.Header{worldexport.DigestTrailer: {digest}}
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
out <- w
|
||||
@@ -179,18 +167,6 @@ func awaitUpload(t *testing.T, up <-chan *httptest.ResponseRecorder) *httptest.R
|
||||
// claimed the export by mistake would block on the parked body, and an upload
|
||||
// let in by mistake would wait for a browser, so either fails the test after a
|
||||
// bound instead of hanging it.
|
||||
// awaitStop returns the next Job the sweep asked to delete.
|
||||
func awaitStop(t *testing.T, ex *fakeExporter) string {
|
||||
t.Helper()
|
||||
select {
|
||||
case job := <-ex.stopped:
|
||||
return job
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("no Job was stopped")
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func doSoon(t *testing.T, h http.Handler, method, path, body string, headers map[string]string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
out := make(chan *httptest.ResponseRecorder, 1)
|
||||
@@ -213,7 +189,6 @@ func randomBytes(n int) []byte {
|
||||
func TestExportBackupGate(t *testing.T) {
|
||||
t.Run("former owner starts a backup export", func(t *testing.T) {
|
||||
a, repo, cl, ex := exportFixture()
|
||||
repo.backups[0].sha256 = strings.Repeat("cd", 32)
|
||||
v := beginExport(t, a.ExternalHandler(), backupPath, "owner1")
|
||||
if !hex64.MatchString(v.Ticket) || v.State != "pending" || v.Filename != "survival-backup-bk1.tar.gz" {
|
||||
t.Fatalf("ticket = %+v", v)
|
||||
@@ -223,7 +198,6 @@ func TestExportBackupGate(t *testing.T) {
|
||||
}
|
||||
r := ex.reqs[0]
|
||||
if r.Server != "survival" || r.Mode != worldexport.ModeBackup || r.BackupRef != "/backups/survival-bk1.tar.gz" ||
|
||||
r.BackupSHA256 != strings.Repeat("cd", 32) || r.Path != "" || r.Dir ||
|
||||
!hex16.MatchString(r.ID) || !hex64.MatchString(r.Token) || r.Token == v.Ticket ||
|
||||
r.TargetURL != exportBase+"/api/v1/internal/exports/"+r.ID {
|
||||
t.Fatalf("export request = %+v", r)
|
||||
@@ -297,8 +271,7 @@ func TestExportWorldGate(t *testing.T) {
|
||||
if v.Filename != "survival-world-"+exportStamp+".tar.gz" || v.State != "pending" {
|
||||
t.Fatalf("ticket = %+v", v)
|
||||
}
|
||||
if len(ex.reqs) != 1 || ex.reqs[0].Mode != worldexport.ModeWorld || ex.reqs[0].BackupRef != "" || ex.reqs[0].BackupSHA256 != "" ||
|
||||
ex.reqs[0].Path != "" || ex.reqs[0].Server != "survival" {
|
||||
if len(ex.reqs) != 1 || ex.reqs[0].Mode != worldexport.ModeWorld || ex.reqs[0].BackupRef != "" || ex.reqs[0].Server != "survival" {
|
||||
t.Fatalf("export requests = %+v", ex.reqs)
|
||||
}
|
||||
if strings.Join(cl.acquired, ",") != "survival:"+maintenance.KindExport || strings.Join(cl.released, ",") != "survival" {
|
||||
@@ -365,7 +338,7 @@ func TestExportWorldGate(t *testing.T) {
|
||||
w := do(a.ExternalHandler(), "POST", worldPath, "", as("admin1"))
|
||||
var raw map[string]map[string]string
|
||||
_ = json.Unmarshal(w.Body.Bytes(), &raw)
|
||||
if want := "a world export or file download is running on this server's world; retry once it finishes"; w.Code != http.StatusConflict ||
|
||||
if want := "a world export is running on this server's world; retry once it finishes"; w.Code != http.StatusConflict ||
|
||||
raw["error"]["code"] != "maintenance_in_progress" || raw["error"]["message"] != want {
|
||||
t.Fatalf("busy = %d %s, want 409 %q", w.Code, w.Body.String(), want)
|
||||
}
|
||||
@@ -390,7 +363,7 @@ func TestExportWorldGate(t *testing.T) {
|
||||
}
|
||||
ex.err = nil
|
||||
beginExport(t, h, worldPath, "owner1")
|
||||
if n := len(a.exportTickets().starts[exportStarts{userID: "owner1"}]); n != 1 {
|
||||
if n := len(a.exportTickets().starts["owner1"]); n != 1 {
|
||||
t.Fatalf("hourly starts = %d, want only the export that got a Job", n)
|
||||
}
|
||||
})
|
||||
@@ -483,17 +456,9 @@ func TestExportRendezvous(t *testing.T) {
|
||||
t.Fatalf("PUT to another id = %d", w.Code)
|
||||
}
|
||||
|
||||
// A length that is no byte count is refused before the token is spent.
|
||||
for _, bad := range []string{"-1", "12abc", "0x10"} {
|
||||
if w := doSoon(t, in, "PUT", "/api/v1/internal/exports/"+job.ID, "x", map[string]string{
|
||||
"Authorization": "Bearer " + job.Token, worldexport.LengthHeader: bad}); w.Code != http.StatusBadRequest || decodeErr(t, w) != "bad_request" {
|
||||
t.Fatalf("PUT declaring %q bytes = %d %s", bad, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
archive := randomBytes(3*exportCopyBuffer + 4321)
|
||||
pr, pw := io.Pipe()
|
||||
up := uploadExport(in, job.ID, job.Token, pr, contentDigestOf(string(archive)))
|
||||
up := uploadExport(in, job.ID, job.Token, pr)
|
||||
waitExportReady(t, ext, v.Ticket, "owner1")
|
||||
if w := doSoon(t, in, "PUT", "/api/v1/internal/exports/"+job.ID, "x", map[string]string{"Authorization": "Bearer " + job.Token}); w.Code != http.StatusNotFound {
|
||||
t.Fatalf("a second PUT with the spent token = %d, want 404", w.Code)
|
||||
@@ -573,55 +538,11 @@ func TestExportExpiry(t *testing.T) {
|
||||
t.Fatalf("past the pending TTL: %d %+v", code, s)
|
||||
}
|
||||
job := ex.reqs[0]
|
||||
if got := awaitStop(t, ex); got != worldexport.JobName(job.Server, job.ID) {
|
||||
t.Fatalf("stopped %q, want the export's own Job", got)
|
||||
}
|
||||
if w := doSoon(t, a.InternalHandler(), "PUT", "/api/v1/internal/exports/"+job.ID, "x", map[string]string{"Authorization": "Bearer " + job.Token}); w.Code != http.StatusNotFound {
|
||||
t.Fatalf("a late Job's PUT = %d, want 404", w.Code)
|
||||
}
|
||||
})
|
||||
|
||||
// The owner closed the tab, so no route sweeps; felis-api's loop does.
|
||||
t.Run("the loop stops a Job left pending, and only that one", func(t *testing.T) {
|
||||
a, _, _, ex := exportFixture()
|
||||
var clock atomic.Int64
|
||||
clock.Store(1_700_000_000)
|
||||
a.Now = func() time.Time { return time.Unix(clock.Load(), 0) }
|
||||
ext := a.ExternalHandler()
|
||||
beginExport(t, ext, worldPath, "owner1")
|
||||
stuck := ex.reqs[0]
|
||||
v := beginExport(t, ext, "/api/v1/servers/creative/backups/bk2/export", "admin1")
|
||||
moving := ex.reqs[1]
|
||||
up := uploadExport(a.InternalHandler(), moving.ID, moving.Token, strings.NewReader("archive"), contentDigestOf("archive"))
|
||||
waitExportReady(t, ext, v.Ticket, "admin1")
|
||||
|
||||
clock.Add(int64(exportPendingTTL/time.Second) - 1)
|
||||
a.ExpireExports()
|
||||
select {
|
||||
case job := <-ex.stopped:
|
||||
t.Fatalf("stopped %s inside the pending TTL", job)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
clock.Add(1)
|
||||
a.ExpireExports()
|
||||
if got := awaitStop(t, ex); got != worldexport.JobName(stuck.Server, stuck.ID) {
|
||||
t.Fatalf("stopped %q, want the Job that never connected", got)
|
||||
}
|
||||
select {
|
||||
case job := <-ex.stopped:
|
||||
t.Fatalf("also stopped %s, whose upload was waiting for its browser", job)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
// Its browser takes it, so the upload's handler has returned before the
|
||||
// next subtest changes the claim TTL that handler reads.
|
||||
if w := do(ext, "GET", "/api/v1/exports/"+v.Ticket+"/download", "", as("admin1")); w.Code != http.StatusOK || w.Body.String() != "archive" {
|
||||
t.Fatalf("download of the waiting export = %d %q", w.Code, w.Body.String())
|
||||
}
|
||||
if w := awaitUpload(t, up); w.Code != http.StatusNoContent {
|
||||
t.Fatalf("upload answered %d %s, want 204", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a browser that never comes", func(t *testing.T) {
|
||||
defer func(old time.Duration) { exportClaimTTL = old }(exportClaimTTL)
|
||||
exportClaimTTL = 30 * time.Millisecond
|
||||
@@ -629,7 +550,7 @@ func TestExportExpiry(t *testing.T) {
|
||||
ext := a.ExternalHandler()
|
||||
v := beginExport(t, ext, backupPath, "owner1")
|
||||
job := ex.reqs[0]
|
||||
w := awaitUpload(t, uploadExport(a.InternalHandler(), job.ID, job.Token, strings.NewReader("archive"), contentDigestOf("archive")))
|
||||
w := awaitUpload(t, uploadExport(a.InternalHandler(), job.ID, job.Token, strings.NewReader("archive")))
|
||||
if w.Code != http.StatusGone || decodeErr(t, w) != "export_expired" {
|
||||
t.Fatalf("unclaimed upload = %d %s, want 410 export_expired", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -715,20 +636,12 @@ func exportServers(t *testing.T, a *API) (ext, in *httptest.Server) {
|
||||
return ext, in
|
||||
}
|
||||
|
||||
// realUpload sends the Job's PUT as cmd/felis export does: chunked, with the
|
||||
// size when it is known (not -1) and, when set, digest as the trailer.
|
||||
func realUpload(t *testing.T, in *httptest.Server, job worldexport.Request, body io.Reader, size int64, digest string) <-chan *http.Response {
|
||||
func realUpload(t *testing.T, in *httptest.Server, job worldexport.Request, body io.Reader, size int64) <-chan *http.Response {
|
||||
req, err := http.NewRequest("PUT", in.URL+"/api/v1/internal/exports/"+job.ID, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.ContentLength = -1
|
||||
if size >= 0 {
|
||||
req.Header.Set(worldexport.LengthHeader, strconv.FormatInt(size, 10))
|
||||
}
|
||||
if digest != "" {
|
||||
req.Trailer = http.Header{worldexport.DigestTrailer: {digest}}
|
||||
}
|
||||
req.ContentLength = size
|
||||
req.Header.Set("Authorization", "Bearer "+job.Token)
|
||||
out := make(chan *http.Response, 1)
|
||||
go func() {
|
||||
@@ -768,339 +681,82 @@ func awaitResponse(t *testing.T, ch <-chan *http.Response) *http.Response {
|
||||
}
|
||||
}
|
||||
|
||||
// TestExportStreamsWhatTheJobSends: the archive reaches the browser byte for
|
||||
// byte, with the length the Job declared when it declared one. The Job checks a
|
||||
// backup against its recorded digest itself and, on a mismatch, cuts its upload
|
||||
// short of the end; the download then aborts too, so the browser never holds a
|
||||
// complete-looking file.
|
||||
func TestExportStreamsWhatTheJobSends(t *testing.T) {
|
||||
// TestExportBackupDigest: a backup streams with its length and is checked
|
||||
// against the sha256 recorded when it was written. A match downloads whole; a
|
||||
// mismatch withholds the tail and aborts, so the browser never holds a
|
||||
// complete-looking corrupt file, and the Job hears backup_corrupt.
|
||||
func TestExportBackupDigest(t *testing.T) {
|
||||
archive := randomBytes(3*exportCopyBuffer + 4321)
|
||||
|
||||
t.Run("whole, with its length", func(t *testing.T) {
|
||||
a, _, _, ex := exportFixture()
|
||||
ext, in := exportServers(t, a)
|
||||
v := beginExport(t, a.ExternalHandler(), backupPath, "owner1")
|
||||
up := realUpload(t, in, ex.reqs[0], bytes.NewReader(archive), int64(len(archive)), contentDigestOf(string(archive)))
|
||||
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
|
||||
resp, got, err := realDownload(ext, v.Ticket)
|
||||
if resp == nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Length") != strconv.Itoa(len(archive)) ||
|
||||
err != nil || !bytes.Equal(got, archive) {
|
||||
t.Fatalf("download = %d, Content-Length %q, read %d of %d bytes, err %v",
|
||||
resp.StatusCode, resp.Header.Get("Content-Length"), len(got), len(archive), err)
|
||||
}
|
||||
if upResp := awaitResponse(t, up); upResp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("upload answered %d, want 204", upResp.StatusCode)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an upload the Job cuts short aborts the download", func(t *testing.T) {
|
||||
a, _, _, ex := exportFixture()
|
||||
ext, in := exportServers(t, a)
|
||||
v := beginExport(t, a.ExternalHandler(), backupPath, "owner1")
|
||||
// As the Job's digest check fails: all but the end, then a read error,
|
||||
// which aborts the chunked PUT.
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
_, _ = pw.Write(archive[:len(archive)-100])
|
||||
pw.CloseWithError(errors.New("the backup archive does not match the sha256 recorded when it was written"))
|
||||
}()
|
||||
up := realUpload(t, in, ex.reqs[0], pr, -1, contentDigestOf(string(archive)))
|
||||
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
|
||||
resp, got, err := realDownload(ext, v.Ticket)
|
||||
if resp == nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
if err == nil || len(got) > len(archive)-100 || !bytes.Equal(got, archive[:len(got)]) {
|
||||
t.Fatalf("read %d of %d bytes, err %v; want an error short of the end", len(got), len(archive), err)
|
||||
}
|
||||
if upResp := awaitResponse(t, up); upResp.StatusCode != -1 {
|
||||
t.Fatalf("the aborted upload answered %d", upResp.StatusCode)
|
||||
}
|
||||
})
|
||||
|
||||
// The Job's trailer is the SHA-256 of what it sent. Bytes that arrive
|
||||
// otherwise, or without it, or not as many as it declared, never reach the
|
||||
// browser to their end, and the Job hears the download failed.
|
||||
t.Run("bytes changed on the way never reach the browser whole", func(t *testing.T) {
|
||||
flipped := bytes.Clone(archive)
|
||||
flipped[len(flipped)/2] ^= 1
|
||||
for _, c := range []struct {
|
||||
name, digest string
|
||||
size int64
|
||||
sum := sha256.Sum256(archive)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
digest string
|
||||
}{
|
||||
{"another archive's digest", contentDigestOf(string(flipped)), int64(len(archive))},
|
||||
{"no digest", "", -1},
|
||||
{"a digest that is no SHA-256", "sha-256=:AAAA:", -1},
|
||||
{"one byte more declared than sent", contentDigestOf(string(archive)), int64(len(archive)) + 1},
|
||||
{"recorded digest matches", hex.EncodeToString(sum[:])},
|
||||
{"no digest recorded", ""},
|
||||
{"digest mismatch", strings.Repeat("ab", 32)},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
a, _, _, ex := exportFixture()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a, repo, _, ex := exportFixture()
|
||||
repo.backups[0].sha256 = tc.digest
|
||||
ext, in := exportServers(t, a)
|
||||
v := beginExport(t, a.ExternalHandler(), backupPath, "owner1")
|
||||
up := realUpload(t, in, ex.reqs[0], bytes.NewReader(archive), c.size, c.digest)
|
||||
up := realUpload(t, in, ex.reqs[0], bytes.NewReader(archive), int64(len(archive)))
|
||||
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
|
||||
resp, got, err := realDownload(ext, v.Ticket)
|
||||
if resp == nil {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Length") != strconv.Itoa(len(archive)) {
|
||||
t.Fatalf("download = %d, Content-Length %q", resp.StatusCode, resp.Header.Get("Content-Length"))
|
||||
}
|
||||
upResp := awaitResponse(t, up)
|
||||
if tc.digest == strings.Repeat("ab", 32) {
|
||||
if err == nil || len(got) >= len(archive) || !bytes.Equal(got, archive[:len(got)]) {
|
||||
t.Fatalf("read %d of %d bytes, err %v; want an error short of the end", len(got), len(archive), err)
|
||||
t.Fatalf("mismatch: read %d of %d bytes, err %v; want an error short of the end", len(got), len(archive), err)
|
||||
}
|
||||
if upResp := awaitResponse(t, up); upResp.StatusCode != http.StatusGone {
|
||||
t.Fatalf("the upload answered %d, want 410", upResp.StatusCode)
|
||||
if upResp.StatusCode != http.StatusConflict || errCode(mustRead(t, upResp.Body)) != "backup_corrupt" {
|
||||
t.Fatalf("upload answered %d, want 409 backup_corrupt", upResp.StatusCode)
|
||||
}
|
||||
})
|
||||
return
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a browser that leaves early is what the Job hears", func(t *testing.T) {
|
||||
a, _, _, ex := exportFixture()
|
||||
ext, _ := exportServers(t, a)
|
||||
v := beginExport(t, a.ExternalHandler(), worldPath, "owner1")
|
||||
up := uploadExport(a.InternalHandler(), ex.reqs[0].ID, ex.reqs[0].Token, io.LimitReader(zeros{}, 1<<30), "")
|
||||
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
|
||||
req, _ := http.NewRequest("GET", ext.URL+"/api/v1/exports/"+v.Ticket+"/download", nil)
|
||||
req.Header.Set("X-Test-User", "owner1")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
if err != nil || !bytes.Equal(got, archive) {
|
||||
t.Fatalf("read %d of %d bytes, err %v", len(got), len(archive), err)
|
||||
}
|
||||
if _, err := io.ReadFull(resp.Body, make([]byte, 1000)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if w := awaitUpload(t, up); w.Code != http.StatusGone || decodeErr(t, w) != "export_expired" {
|
||||
t.Fatalf("upload answered %d %s, want 410 export_expired", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const fileDownloadPath = "/api/v1/servers/survival/files/download?path="
|
||||
|
||||
// fileDownloadFixture is exportFixture with the file manager wired, which the
|
||||
// file routes' gate requires.
|
||||
func fileDownloadFixture() (*API, *fakeRepo, *fakeCluster, *fakeExporter) {
|
||||
a, repo, cl, ex := exportFixture()
|
||||
a.Files = &fakeFileEditor{}
|
||||
return a, repo, cl, ex
|
||||
}
|
||||
|
||||
func TestFileDownloadGate(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, query, filename, contentType, payload string
|
||||
want worldexport.Request
|
||||
}{
|
||||
{"a file", "plugins/Essentials/config.yml", "config.yml", fileedit.DownloadFileType, `{"dir":false}`,
|
||||
worldexport.Request{Server: "survival", Mode: worldexport.ModeFiles, Path: "plugins/Essentials/config.yml"}},
|
||||
{"a folder, as a zip named after it", "plugins/Essentials/&dir=true", "Essentials.zip", fileedit.DownloadZipType, `{"dir":true}`,
|
||||
worldexport.Request{Server: "survival", Mode: worldexport.ModeFiles, Path: "plugins/Essentials/", Dir: true}},
|
||||
{"dir other than true is a file", "a.yml&dir=false", "a.yml", fileedit.DownloadFileType, `{"dir":false}`,
|
||||
worldexport.Request{Server: "survival", Mode: worldexport.ModeFiles, Path: "a.yml"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a, repo, cl, ex := fileDownloadFixture()
|
||||
v := beginExport(t, a.ExternalHandler(), fileDownloadPath+tc.query, "owner1")
|
||||
if !hex64.MatchString(v.Ticket) || v.State != "pending" || v.Filename != tc.filename {
|
||||
t.Fatalf("ticket = %+v", v)
|
||||
}
|
||||
if len(ex.reqs) != 1 {
|
||||
t.Fatalf("exporter started %d Jobs, want 1", len(ex.reqs))
|
||||
}
|
||||
r := ex.reqs[0]
|
||||
if !hex16.MatchString(r.ID) || !hex64.MatchString(r.Token) || r.TargetURL != exportBase+"/api/v1/internal/exports/"+r.ID {
|
||||
t.Fatalf("export request = %+v", r)
|
||||
}
|
||||
r.ID, r.Token, r.TargetURL = "", "", ""
|
||||
if r != tc.want {
|
||||
t.Fatalf("export request = %+v, want %+v", r, tc.want)
|
||||
}
|
||||
if e := a.exportTickets().byTicket[v.Ticket]; e.contentType != tc.contentType || !e.files {
|
||||
t.Fatalf("export served as %q, file download %v", e.contentType, e.files)
|
||||
}
|
||||
if strings.Join(cl.acquired, ",") != "survival:"+maintenance.KindExport || strings.Join(cl.released, ",") != "survival" {
|
||||
t.Fatalf("lock acquired %v, released %v", cl.acquired, cl.released)
|
||||
}
|
||||
if len(repo.audits) != 1 || repo.audits[0].Action != "file.download" || repo.audits[0].ActorUserID != "owner1" ||
|
||||
repo.audits[0].ServerName != "survival:"+tc.want.Path || string(repo.audits[0].Payload) != tc.payload {
|
||||
t.Fatalf("audit = %+v", repo.audits)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("admin", func(t *testing.T) {
|
||||
a, _, _, ex := fileDownloadFixture()
|
||||
beginExport(t, a.ExternalHandler(), fileDownloadPath+"server.properties", "admin1")
|
||||
if len(ex.reqs) != 1 {
|
||||
t.Fatalf("exporter started %d Jobs, want 1", len(ex.reqs))
|
||||
}
|
||||
})
|
||||
|
||||
busy := func(_ *API, c *fakeCluster) {
|
||||
c.maintErr["survival"] = &MaintenanceBusyError{Kind: maintenance.KindFileWrite}
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name, user, query string
|
||||
edit func(*API, *fakeCluster)
|
||||
code int
|
||||
errCode string
|
||||
// msg, when set, is what the refusal must say.
|
||||
msg string
|
||||
}{
|
||||
{name: "stranger", user: "stranger", query: "a.yml", code: http.StatusForbidden, errCode: "forbidden"},
|
||||
{name: "principal without an account", user: "nouser", query: "a.yml", code: http.StatusForbidden, errCode: "forbidden"},
|
||||
{name: "running", user: "owner1", query: "a.yml", edit: func(_ *API, c *fakeCluster) { c.byName["survival"].Ready = true },
|
||||
code: http.StatusConflict, errCode: "not_stopped", msg: "stop the server before working with its files"},
|
||||
{name: "no world volume", user: "owner1", query: "a.yml", edit: func(_ *API, c *fakeCluster) { c.noWorld["survival"] = true },
|
||||
code: http.StatusConflict, errCode: "no_world_volume"},
|
||||
{name: "no file editor", user: "owner1", query: "a.yml", edit: func(a *API, _ *fakeCluster) { a.Files = nil },
|
||||
code: http.StatusServiceUnavailable, errCode: "files_unavailable"},
|
||||
{name: "no path", user: "owner1", query: "", code: http.StatusBadRequest, errCode: "bad_request"},
|
||||
{name: "the root", user: "owner1", query: ".&dir=true", code: http.StatusBadRequest, errCode: "bad_path"},
|
||||
{name: "the root, slashed", user: "owner1", query: "/&dir=true", code: http.StatusBadRequest, errCode: "bad_path"},
|
||||
{name: "the root, dotted", user: "owner1", query: "./", code: http.StatusBadRequest, errCode: "bad_path"},
|
||||
{name: "the root, walked back", user: "owner1", query: "plugins/..", code: http.StatusBadRequest, errCode: "bad_path"},
|
||||
{name: "no exporter", user: "owner1", query: "a.yml", edit: func(a *API, _ *fakeCluster) { a.Exporter = nil },
|
||||
code: http.StatusServiceUnavailable, errCode: "export_unavailable"},
|
||||
{name: "no internal URL", user: "owner1", query: "a.yml", edit: func(a *API, _ *fakeCluster) { a.InternalBaseURL = "" },
|
||||
code: http.StatusServiceUnavailable, errCode: "export_unavailable"},
|
||||
{name: "world busy", user: "owner1", query: "a.yml", edit: busy, code: http.StatusConflict, errCode: "maintenance_in_progress"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a, repo, cl, ex := fileDownloadFixture()
|
||||
if tc.edit != nil {
|
||||
tc.edit(a, cl)
|
||||
}
|
||||
w := do(a.ExternalHandler(), "POST", fileDownloadPath+tc.query, "", as(tc.user))
|
||||
if w.Code != tc.code {
|
||||
t.Fatalf("code = %d, want %d (%s)", w.Code, tc.code, w.Body.String())
|
||||
}
|
||||
if got := decodeErr(t, w); got != tc.errCode {
|
||||
t.Errorf("error code = %q, want %q", got, tc.errCode)
|
||||
}
|
||||
if tc.msg != "" && !strings.Contains(w.Body.String(), tc.msg) {
|
||||
t.Errorf("body %s does not say %q", w.Body.String(), tc.msg)
|
||||
}
|
||||
// Nothing is left behind: no Job, no audit, no ticket, no start
|
||||
// counted against the hour.
|
||||
reg := a.exportTickets()
|
||||
if len(ex.reqs) != 0 || len(repo.audits) != 0 || len(cl.released) != 0 || len(reg.byTicket) != 0 || len(reg.starts) != 0 {
|
||||
t.Errorf("a refused download started %d Jobs, wrote %d audits, released %v, left %d tickets and %v",
|
||||
len(ex.reqs), len(repo.audits), cl.released, len(reg.byTicket), reg.starts)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a failed Job is refunded and releases the lock", func(t *testing.T) {
|
||||
a, _, cl, ex := fileDownloadFixture()
|
||||
ex.err = errors.New("jobs is forbidden")
|
||||
if w := do(a.ExternalHandler(), "POST", fileDownloadPath+"a.yml", "", as("owner1")); w.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("failed Job: code = %d, want 500 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if reg := a.exportTickets(); len(reg.byTicket) != 0 || len(reg.starts) != 0 || strings.Join(cl.released, ",") != "survival" {
|
||||
t.Fatalf("after a failed Job: %d tickets, starts %v, released %v", len(reg.byTicket), reg.starts, cl.released)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFileDownloadLimits(t *testing.T) {
|
||||
busy := func(t *testing.T, w *httptest.ResponseRecorder, message, retry string) {
|
||||
t.Helper()
|
||||
var raw map[string]map[string]string
|
||||
_ = json.Unmarshal(w.Body.Bytes(), &raw)
|
||||
if w.Code != http.StatusTooManyRequests || raw["error"]["code"] != "export_busy" || raw["error"]["message"] != message ||
|
||||
w.Header().Get("Retry-After") != retry {
|
||||
t.Fatalf("refusal = %d %s Retry-After %q; want 429 %q Retry-After %s", w.Code, w.Body.String(), w.Header().Get("Retry-After"), message, retry)
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("two per user, counted apart from exports", func(t *testing.T) {
|
||||
a, _, _, ex := fileDownloadFixture()
|
||||
h := a.ExternalHandler()
|
||||
beginExport(t, h, fileDownloadPath+"a.yml", "owner1")
|
||||
beginExport(t, h, fileDownloadPath+"b.yml", "owner1")
|
||||
beginExport(t, h, worldPath, "owner1") // file downloads do not hold an export off
|
||||
busy(t, do(h, "POST", fileDownloadPath+"c.yml", "", as("owner1")),
|
||||
"you already have 2 file downloads in progress; let one finish first", "90")
|
||||
if len(ex.reqs) != 3 {
|
||||
t.Fatalf("exporter started %d Jobs, want 3", len(ex.reqs))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("four across the install", func(t *testing.T) {
|
||||
a, _, _, ex := fileDownloadFixture()
|
||||
h := a.ExternalHandler()
|
||||
for _, u := range []string{"owner1", "owner1", "owner3", "owner3"} {
|
||||
server := map[string]string{"owner1": "survival", "owner3": "gamma"}[u]
|
||||
beginExport(t, h, "/api/v1/servers/"+server+"/files/download?path=a.yml", u)
|
||||
}
|
||||
busy(t, do(h, "POST", fileDownloadPath+"a.yml", "", as("admin1")),
|
||||
"4 file downloads are already in progress; retry in a minute", "60")
|
||||
if len(ex.reqs) != 4 {
|
||||
t.Fatalf("exporter started %d Jobs, want 4", len(ex.reqs))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("thirty per user per hour", func(t *testing.T) {
|
||||
defer func(old time.Duration) { exportPendingTTL = old }(exportPendingTTL)
|
||||
exportPendingTTL = time.Minute
|
||||
a, _, _, _ := fileDownloadFixture()
|
||||
var clock atomic.Int64
|
||||
a.Now = func() time.Time { return time.Unix(clock.Load(), 0) }
|
||||
h := a.ExternalHandler()
|
||||
for i := range fileExportPerHour {
|
||||
clock.Store(1_700_000_000 + int64(i)*100) // each start outlives the last one's pending TTL
|
||||
beginExport(t, h, fileDownloadPath+"a.yml", "owner1")
|
||||
}
|
||||
clock.Store(1_700_000_000 + 2950)
|
||||
busy(t, do(h, "POST", fileDownloadPath+"a.yml", "", as("owner1")),
|
||||
"you have started 30 file downloads in the last hour; retry later", "650")
|
||||
beginExport(t, h, worldPath, "owner1") // exports keep their own hour
|
||||
clock.Store(1_700_000_000 + 3600)
|
||||
beginExport(t, h, fileDownloadPath+"a.yml", "owner1")
|
||||
})
|
||||
}
|
||||
|
||||
// TestFileDownloadServed: a file goes out as its raw bytes under its own name,
|
||||
// with its length; a folder as a zip, streamed without one.
|
||||
func TestFileDownloadServed(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, query, contentType, disposition string
|
||||
size int64
|
||||
}{
|
||||
{"a file", url.QueryEscape("plugins/配置 file.yml"), fileedit.DownloadFileType,
|
||||
"attachment; filename*=utf-8''%E9%85%8D%E7%BD%AE%20file.yml", 64 << 10},
|
||||
{"a folder", "plugins&dir=true", fileedit.DownloadZipType, "attachment; filename=plugins.zip", -1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a, _, _, ex := fileDownloadFixture()
|
||||
ext, in := exportServers(t, a)
|
||||
v := beginExport(t, a.ExternalHandler(), fileDownloadPath+tc.query, "owner1")
|
||||
// Past what the server would buffer and measure itself when the
|
||||
// handler sets no length.
|
||||
body := randomBytes(64 << 10)
|
||||
up := realUpload(t, in, ex.reqs[0], bytes.NewReader(body), tc.size, contentDigestOf(string(body)))
|
||||
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
|
||||
resp, got, err := realDownload(ext, v.Ticket)
|
||||
if resp == nil || err != nil || !bytes.Equal(got, body) {
|
||||
t.Fatalf("download: read %d bytes, %v", len(got), err)
|
||||
}
|
||||
wantLength := ""
|
||||
if tc.size >= 0 {
|
||||
wantLength = strconv.FormatInt(tc.size, 10)
|
||||
}
|
||||
h := resp.Header
|
||||
if h.Get("Content-Type") != tc.contentType || h.Get("Content-Disposition") != tc.disposition ||
|
||||
h.Get("Content-Length") != wantLength || h.Get("X-Content-Type-Options") != "nosniff" {
|
||||
t.Fatalf("download headers = %v", h)
|
||||
}
|
||||
if upResp := awaitResponse(t, up); upResp.StatusCode != http.StatusNoContent {
|
||||
if upResp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("upload answered %d, want 204", upResp.StatusCode)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The tail is withheld from the response writer itself, not only from
|
||||
// whatever the connection had yet to send: with every write captured, a
|
||||
// mismatch still ends short of the archive.
|
||||
t.Run("the tail waits for the digest", func(t *testing.T) {
|
||||
a, repo, _, ex := exportFixture()
|
||||
repo.backups[0].sha256 = strings.Repeat("ab", 32)
|
||||
ext := a.ExternalHandler()
|
||||
v := beginExport(t, ext, backupPath, "owner1")
|
||||
up := uploadExport(a.InternalHandler(), ex.reqs[0].ID, ex.reqs[0].Token, bytes.NewReader(archive))
|
||||
waitExportReady(t, ext, v.Ticket, "owner1")
|
||||
w := httptest.NewRecorder()
|
||||
func() {
|
||||
defer func() {
|
||||
if p := recover(); p != http.ErrAbortHandler {
|
||||
t.Errorf("the download ended with %v, want the abort", p)
|
||||
}
|
||||
}()
|
||||
r := httptest.NewRequest("GET", "/api/v1/exports/"+v.Ticket+"/download", nil)
|
||||
r.Header.Set("X-Test-User", "owner1")
|
||||
ext.ServeHTTP(w, r)
|
||||
}()
|
||||
if got := w.Body.Bytes(); len(got) != 3*exportCopyBuffer || !bytes.Equal(got, archive[:len(got)]) {
|
||||
t.Fatalf("wrote %d of %d bytes, want all but the last read (%d)", len(got), len(archive), 3*exportCopyBuffer)
|
||||
}
|
||||
if w := awaitUpload(t, up); w.Code != http.StatusConflict || decodeErr(t, w) != "backup_corrupt" {
|
||||
t.Fatalf("upload answered %d %s, want 409 backup_corrupt", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// zeros reads as an endless run of zero bytes.
|
||||
@@ -1111,6 +767,15 @@ func (zeros) Read(p []byte) (int, error) {
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func mustRead(t *testing.T, r io.Reader) []byte {
|
||||
t.Helper()
|
||||
b, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// TestExportUploadPace: the Job's upload waits for the browser longer than the
|
||||
// body-deadline grace, and then moves at the browser's pace, without being cut
|
||||
// off; a body that stops moving for exportStall is.
|
||||
@@ -1130,7 +795,7 @@ func TestExportUploadPace(t *testing.T) {
|
||||
_, _ = pw.Write(archive[1000:])
|
||||
pw.Close()
|
||||
}()
|
||||
up := realUpload(t, in, ex.reqs[0], pr, -1, contentDigestOf(string(archive)))
|
||||
up := realUpload(t, in, ex.reqs[0], pr, -1)
|
||||
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
|
||||
time.Sleep(3 * bodyGrace)
|
||||
_, got, err := realDownload(ext, v.Ticket)
|
||||
@@ -1149,7 +814,7 @@ func TestExportUploadPace(t *testing.T) {
|
||||
pr, pw := io.Pipe()
|
||||
defer pw.Close()
|
||||
go func() { _, _ = pw.Write(make([]byte, 1000)) }() // then nothing, ever
|
||||
up := realUpload(t, in, ex.reqs[0], pr, -1, "")
|
||||
up := realUpload(t, in, ex.reqs[0], pr, -1)
|
||||
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
|
||||
done := make(chan error, 1)
|
||||
go func() { _, _, err := realDownload(ext, v.Ticket); done <- err }()
|
||||
@@ -1169,7 +834,7 @@ func TestExportUploadPace(t *testing.T) {
|
||||
a, _, _, ex := exportFixture()
|
||||
ext, in := exportServers(t, a)
|
||||
v := beginExport(t, a.ExternalHandler(), worldPath, "owner1")
|
||||
realUpload(t, in, ex.reqs[0], io.LimitReader(zeros{}, 1<<30), -1, "")
|
||||
realUpload(t, in, ex.reqs[0], io.LimitReader(zeros{}, 1<<30), -1)
|
||||
waitExportReady(t, a.ExternalHandler(), v.Ticket, "owner1")
|
||||
req, _ := http.NewRequest("GET", ext.URL+"/api/v1/exports/"+v.Ticket+"/download", nil)
|
||||
req.Header.Set("X-Test-User", "owner1")
|
||||
@@ -1208,7 +873,7 @@ func TestK8sExportJobs(t *testing.T) {
|
||||
job := func(id, mode string) *batchv1.Job {
|
||||
j, err := worldexport.ExportJob(worldexport.JobParams{
|
||||
Server: "survival", ID: id, Mode: mode,
|
||||
WorldPVC: "world-survival-0", BackupPVC: "felis-backups", BackupRef: "/backups/a.tar.gz", Path: "plugins",
|
||||
WorldPVC: "world-survival-0", BackupPVC: "felis-backups", BackupRef: "/backups/a.tar.gz",
|
||||
TargetURL: exportBase + "/x", Token: "t", Namespace: "minecraft", Image: "felis:1",
|
||||
})
|
||||
if err != nil {
|
||||
@@ -1217,10 +882,9 @@ func TestK8sExportJobs(t *testing.T) {
|
||||
return j
|
||||
}
|
||||
world, backupJob := job("1111111111111111", worldexport.ModeWorld), job("2222222222222222", worldexport.ModeBackup)
|
||||
files := job("3333333333333333", worldexport.ModeFiles)
|
||||
restoring := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: "restore-survival-cc",
|
||||
Labels: map[string]string{jobServerLabel: "survival", jobManagedByLabel: jobManagedByRestore}}}
|
||||
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(world, backupJob, files, restoring).
|
||||
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(world, backupJob, restoring).
|
||||
WithStatusSubresource(&batchv1.Job{}).Build()
|
||||
k := NewK8sJobStatus(c, "minecraft")
|
||||
ctx := context.Background()
|
||||
@@ -1236,8 +900,7 @@ func TestK8sExportJobs(t *testing.T) {
|
||||
for _, j := range jobs {
|
||||
kinds[j.Name] = j.Kind + "/" + j.State
|
||||
}
|
||||
want := map[string]string{world.Name: "export_world/running", backupJob.Name: "export_backup/running",
|
||||
files.Name: "export_files/running", restoring.Name: "restore/running"}
|
||||
want := map[string]string{world.Name: "export_world/running", backupJob.Name: "export_backup/running", restoring.Name: "restore/running"}
|
||||
if len(kinds) != len(want) {
|
||||
t.Fatalf("jobs = %v, want %v", kinds, want)
|
||||
}
|
||||
@@ -1247,14 +910,11 @@ func TestK8sExportJobs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The kinds agree with maintenance.JobKind: a world export and a file
|
||||
// download hold the world, a backup export does not.
|
||||
// The kinds agree with maintenance.JobKind: a world export holds the world,
|
||||
// a backup export does not.
|
||||
if kind, holds := maintenance.JobKind(world); kind != maintenance.KindExport || !holds {
|
||||
t.Errorf("JobKind(world export) = %q, %v", kind, holds)
|
||||
}
|
||||
if kind, holds := maintenance.JobKind(files); kind != maintenance.KindExport || !holds {
|
||||
t.Errorf("JobKind(file download) = %q, %v", kind, holds)
|
||||
}
|
||||
if _, holds := maintenance.JobKind(backupJob); holds {
|
||||
t.Error("a backup export holds the world")
|
||||
}
|
||||
|
||||
@@ -1,495 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/fileedit"
|
||||
"felis.lolicon.best/internal/maintenance"
|
||||
"felis.lolicon.best/internal/naming"
|
||||
)
|
||||
|
||||
// A file too big for the one-request upload (handleUploadFile) arrives as an
|
||||
// upload session instead: POST …/files/uploads begins one for a path and a
|
||||
// size, PUT …/files/uploads/{id}?offset= sends it in parts of at most
|
||||
// fileedit.PartBytes (each part fits the Cloudflare edge's body limit), and POST
|
||||
// …/files/uploads/{id}/commit lands it. The session lives on felis-api's staging
|
||||
// disk (fileedit.Stage, session.go), bound to the account and the server it was
|
||||
// begun for; the room for the whole file is reserved when it begins, so there is
|
||||
// no product ceiling on the size, only the disk.
|
||||
//
|
||||
// Landing a file that size, like unzipping an archive, can outlast a request,
|
||||
// so both answer 202 with the op, and GET …/files/ops reports how far it has got
|
||||
// and how it ended (fileedit.Editor.StartUpload, StartUnzip, Ops). The Job holds
|
||||
// the world volume while it runs, as any file write does, and the server cannot
|
||||
// start until it ends.
|
||||
|
||||
// fileSessionView is where an upload session stands. Parts are the parts it
|
||||
// took, in order, each with the SHA-256 it arrived with: a client resuming from
|
||||
// a file on disk checks the file still holds those bytes before it sends the
|
||||
// rest.
|
||||
type fileSessionView struct {
|
||||
ID string `json:"id"`
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
Received int64 `json:"received"`
|
||||
PartMaxBytes int64 `json:"part_max_bytes"`
|
||||
Parts []filePartView `json:"parts"`
|
||||
}
|
||||
|
||||
// filePartView is one part a session took.
|
||||
type filePartView struct {
|
||||
Size int64 `json:"size"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
func sessionView(s fileedit.Session) fileSessionView {
|
||||
parts := make([]filePartView, 0, len(s.Parts))
|
||||
for _, p := range s.Parts {
|
||||
parts = append(parts, filePartView{Size: p.Size, SHA256: p.SHA256})
|
||||
}
|
||||
return fileSessionView{
|
||||
ID: s.ID, Path: s.Path, Size: s.Size, Received: s.Received,
|
||||
PartMaxBytes: fileedit.PartBytes, Parts: parts,
|
||||
}
|
||||
}
|
||||
|
||||
// namesFit reports whether every name in path fits one folder entry.
|
||||
func namesFit(path string) bool {
|
||||
for _, name := range strings.Split(path, "/") {
|
||||
if len(name) > fileedit.NameMax {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// errNameTooLong refuses a path namesFit rejects, before any byte is taken:
|
||||
// the Job would only find out once it tried to create the file.
|
||||
func errNameTooLong() *apiError {
|
||||
return newError(http.StatusBadRequest, "bad_path", "a name in the path is longer than %d bytes", fileedit.NameMax)
|
||||
}
|
||||
|
||||
// beginFileUploadRequest is the POST …/files/uploads body.
|
||||
type beginFileUploadRequest struct {
|
||||
Size *int64 `json:"size"`
|
||||
}
|
||||
|
||||
// handleBeginFileUpload serves POST /api/v1/servers/{name}/files/uploads?path=…
|
||||
// — begin an upload session for a file of body.size bytes that will land at
|
||||
// path. The gate is the file manager's, so a server that is running is refused
|
||||
// before any byte is sent; the parts that follow need only the account and the
|
||||
// server, so starting the server midway costs the upload nothing but the
|
||||
// commit's refusal until it is stopped again.
|
||||
func (a *API) handleBeginFileUpload(w http.ResponseWriter, r *http.Request) {
|
||||
name, ok := a.authorizeFileOp(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
user, ok := a.requireFileStage(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
path, ok := requirePath(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// The Job checks the path against the volume when the file lands, hours of
|
||||
// upload later for a big one; one that could never land is refused now.
|
||||
if !filepath.IsLocal(path) || filepath.Clean(path) == "." {
|
||||
writeError(w, r, newError(http.StatusBadRequest, "bad_path",
|
||||
"the path must name a file inside the world folder"))
|
||||
return
|
||||
}
|
||||
if !namesFit(path) {
|
||||
writeError(w, r, errNameTooLong())
|
||||
return
|
||||
}
|
||||
var body beginFileUploadRequest
|
||||
if err := decodeJSON(w, r, &body); err != nil {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
if body.Size == nil || *body.Size < 0 {
|
||||
writeError(w, r, newError(http.StatusBadRequest, "bad_request",
|
||||
"size must be the file's length in bytes"))
|
||||
return
|
||||
}
|
||||
s, err := a.FileStage.Begin(user, name, path, *body.Size)
|
||||
if err != nil {
|
||||
writeFileSessionError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, sessionView(s))
|
||||
}
|
||||
|
||||
// handleFileUploadStatus serves GET /api/v1/servers/{name}/files/uploads/{id}
|
||||
// — where the caller's session stands, so a client that lost a part's answer
|
||||
// resumes from received.
|
||||
func (a *API) handleFileUploadStatus(w http.ResponseWriter, r *http.Request) {
|
||||
name, user, ok := a.authorizeFileSession(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
s, err := a.FileStage.Status(user, name, r.PathValue("id"))
|
||||
if err != nil {
|
||||
writeFileSessionError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, sessionView(s))
|
||||
}
|
||||
|
||||
// handleFileUploadPart serves PUT
|
||||
// /api/v1/servers/{name}/files/uploads/{id}?offset=… — append the raw body to
|
||||
// the caller's session. offset must be where the session ends (409
|
||||
// upload_offset_mismatch otherwise; the status says where), and Content-Length
|
||||
// and Content-Digest are required, as for the one-request upload: the part is
|
||||
// taken whole or not at all, and a part that breaks midway, or whose bytes do
|
||||
// not hash to its digest (400 digest_mismatch), leaves the session where it
|
||||
// was. A part over fileedit.PartBytes is refused before a byte of it is read
|
||||
// (Append).
|
||||
func (a *API) handleFileUploadPart(w http.ResponseWriter, r *http.Request) {
|
||||
name, user, ok := a.authorizeFileSession(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
offset, err := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64)
|
||||
if err != nil {
|
||||
writeError(w, r, newError(http.StatusBadRequest, "bad_request",
|
||||
"offset must be the byte position the part starts at"))
|
||||
return
|
||||
}
|
||||
if r.ContentLength < 0 {
|
||||
writeError(w, r, newError(http.StatusLengthRequired, "length_required",
|
||||
"a part needs a Content-Length"))
|
||||
return
|
||||
}
|
||||
want, ok := contentDigest(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
s, err := a.FileStage.Append(user, name, r.PathValue("id"), offset, r.Body, r.ContentLength, want)
|
||||
if err != nil {
|
||||
writeFileSessionError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, sessionView(s))
|
||||
}
|
||||
|
||||
// handleDropFileUpload serves DELETE /api/v1/servers/{name}/files/uploads/{id}
|
||||
// — cancel the caller's session and free the room it holds.
|
||||
func (a *API) handleDropFileUpload(w http.ResponseWriter, r *http.Request) {
|
||||
name, user, ok := a.authorizeFileSession(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := a.FileStage.Drop(user, name, r.PathValue("id")); err != nil {
|
||||
writeFileSessionError(w, r, err)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// startFileOpRequest is the body of a commit or an unzip.
|
||||
type startFileOpRequest struct {
|
||||
Overwrite bool `json:"overwrite"`
|
||||
}
|
||||
|
||||
// handleCommitFileUpload serves POST
|
||||
// /api/v1/servers/{name}/files/uploads/{id}/commit — land the caller's
|
||||
// finished session at its path, replacing a file there only with
|
||||
// body.overwrite (the op ends file_exists otherwise). It answers 202 with the
|
||||
// op; GET …/files/ops reports how it ends.
|
||||
//
|
||||
// The world lock is taken BEFORE the session is sealed: a commit made while an
|
||||
// earlier commit's Job is still fetching the file is refused by that Job's hold
|
||||
// on the volume, so it never mints the fresh token that would lock the running
|
||||
// Job out. The session outlives a Job that fails for any reason, so such a
|
||||
// commit is simply made again; the Job whose file landed deletes it
|
||||
// (handleInternalFileUploadLanded).
|
||||
func (a *API) handleCommitFileUpload(w http.ResponseWriter, r *http.Request) {
|
||||
name, ok := a.authorizeFileOp(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
user, ok := a.requireFileStage(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var body startFileOpRequest
|
||||
if err := decodeJSON(w, r, &body); err != nil {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
id := r.PathValue("id")
|
||||
s, err := a.FileStage.Status(user, name, id)
|
||||
// Refused before the world lock is asked for; Seal checks again under the
|
||||
// stage's own lock, for a part that arrives in between.
|
||||
if err == nil && s.Received != s.Size {
|
||||
err = fmt.Errorf("%w: %d of %d bytes are here", fileedit.ErrUploadIncomplete, s.Received, s.Size)
|
||||
}
|
||||
if err != nil {
|
||||
writeFileSessionError(w, r, err)
|
||||
return
|
||||
}
|
||||
release, ok := a.acquireWorld(w, r, name, maintenance.KindFileWrite, "stop the server before editing its files")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
staged, err := a.FileStage.Seal(user, name, id)
|
||||
if err != nil {
|
||||
writeFileSessionError(w, r, err)
|
||||
return
|
||||
}
|
||||
op, err := a.Files.StartUpload(r.Context(), name, s.Path, fileedit.UploadSource{
|
||||
URL: a.InternalBaseURL + "/api/v1/internal/file-uploads/" + id,
|
||||
Token: staged.Token,
|
||||
Size: staged.Size,
|
||||
SHA256: staged.SHA256,
|
||||
}, body.Overwrite)
|
||||
if err != nil {
|
||||
writeFileEditError(w, r, err)
|
||||
return
|
||||
}
|
||||
a.auditFile(r, "file.upload", name, s.Path, map[string]any{
|
||||
"size_bytes": staged.Size, "sha256": staged.SHA256, "overwrite": body.Overwrite,
|
||||
})
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"op": opView(op)})
|
||||
}
|
||||
|
||||
// handleUnzipFile serves POST /api/v1/servers/{name}/files/unzip?path=… —
|
||||
// extract the .zip at path into the folder holding it. Without body.overwrite
|
||||
// an archive that would replace any file changes nothing and the op ends
|
||||
// file_exists with the list of them, for the caller to confirm and run again
|
||||
// with overwrite. It answers 202 with the op, like a commit.
|
||||
//
|
||||
// Only the name is checked here, so a caller who picked the wrong file hears
|
||||
// so at once; whether it is a zip, and whether every entry is safe to extract,
|
||||
// is the Job's to decide (fileedit/unzip.go).
|
||||
func (a *API) handleUnzipFile(w http.ResponseWriter, r *http.Request) {
|
||||
name, ok := a.authorizeFileOp(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
path, ok := requirePath(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !strings.HasSuffix(strings.ToLower(path), ".zip") {
|
||||
writeError(w, r, newError(http.StatusBadRequest, "bad_path", "only a .zip file can be extracted"))
|
||||
return
|
||||
}
|
||||
var body startFileOpRequest
|
||||
if err := decodeJSON(w, r, &body); err != nil {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
release, ok := a.acquireWorld(w, r, name, maintenance.KindFileWrite, "stop the server before editing its files")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
op, err := a.Files.StartUnzip(r.Context(), name, path, body.Overwrite)
|
||||
if err != nil {
|
||||
writeFileEditError(w, r, err)
|
||||
return
|
||||
}
|
||||
a.auditFile(r, "file.unzip", name, path, map[string]any{"overwrite": body.Overwrite})
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"op": opView(op)})
|
||||
}
|
||||
|
||||
// handleListFileOps serves GET /api/v1/servers/{name}/files/ops — the server's
|
||||
// background uploads and unzips, newest first: the one running, if any, and
|
||||
// those that ended within the last half hour. It has no stopped gate: while
|
||||
// one runs the server cannot start, and a finished one is still worth showing
|
||||
// after it has.
|
||||
func (a *API) handleListFileOps(w http.ResponseWriter, r *http.Request) {
|
||||
name, ok := a.authorizeServerFiles(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if a.Files == nil {
|
||||
writeError(w, r, newError(http.StatusServiceUnavailable, "files_unavailable",
|
||||
"the file editor is not configured"))
|
||||
return
|
||||
}
|
||||
ops, err := a.Files.Ops(r.Context(), name)
|
||||
if err != nil {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
views := make([]fileOpView, 0, len(ops))
|
||||
for _, op := range ops {
|
||||
views = append(views, opView(op))
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"ops": views})
|
||||
}
|
||||
|
||||
// fileOpView is one background file operation as the API shows it.
|
||||
type fileOpView struct {
|
||||
ID string `json:"id"`
|
||||
Op string `json:"op"`
|
||||
Path string `json:"path"`
|
||||
State string `json:"state"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
Done int64 `json:"done"`
|
||||
Total int64 `json:"total"`
|
||||
Files int `json:"files,omitempty"`
|
||||
Bytes int64 `json:"bytes,omitempty"`
|
||||
Error *fileOpError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// fileOpError is why an op failed. Code is the one the synchronous file routes
|
||||
// answer with for the same refusal (writeFileEditError), or an unzip's own
|
||||
// (archive_invalid, archive_unsafe, archive_symlink, type_conflict), or
|
||||
// job_failed for a Job that ended without saying why; the rest is what the Job
|
||||
// reported about it.
|
||||
type fileOpError struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Entry string `json:"entry,omitempty"`
|
||||
Conflicts []string `json:"conflicts,omitempty"`
|
||||
ConflictCount int `json:"conflict_count,omitempty"`
|
||||
Need int64 `json:"need,omitempty"`
|
||||
Avail int64 `json:"avail,omitempty"`
|
||||
}
|
||||
|
||||
func opView(op fileedit.OpState) fileOpView {
|
||||
v := fileOpView{
|
||||
ID: op.ID, Op: op.Op, Path: op.Path, State: op.State, StartedAt: op.Started,
|
||||
Done: op.Done, Total: op.Total,
|
||||
}
|
||||
if !op.Finished.IsZero() {
|
||||
v.FinishedAt = &op.Finished
|
||||
}
|
||||
if op.Result != nil && op.Result.Code == "" {
|
||||
v.Files, v.Bytes = op.Result.Files, op.Result.Bytes
|
||||
}
|
||||
if op.State == fileedit.OpFailed {
|
||||
v.Error = opError(op)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// opError maps a failed op onto the API's codes. A Job that printed no result
|
||||
// carries only its reason (DeadlineExceeded, BackoffLimitExceeded, OOMKilled):
|
||||
// the log it left is the world's content and the runtime's, and none of it is
|
||||
// the caller's to read.
|
||||
func opError(op fileedit.OpState) *fileOpError {
|
||||
res := op.Result
|
||||
if res == nil {
|
||||
msg := "the file operation stopped before it could report how it went (%s); run it again"
|
||||
switch op.Reason {
|
||||
case "DeadlineExceeded":
|
||||
msg = "the file operation ran out of time (%s); run it again"
|
||||
case fileedit.ReasonOOMKilled:
|
||||
msg = "the file operation ran out of memory (%s); an archive of this many files has to be split into smaller ones"
|
||||
}
|
||||
return &fileOpError{Code: "job_failed", Message: fmt.Sprintf(msg, op.Reason)}
|
||||
}
|
||||
code := res.Code
|
||||
switch res.Code {
|
||||
case fileedit.CodeExists:
|
||||
code = "file_exists"
|
||||
case fileedit.CodeNoSpace:
|
||||
code = "volume_full"
|
||||
case fileedit.CodeConflict:
|
||||
code = "file_changed"
|
||||
}
|
||||
return &fileOpError{
|
||||
Code: code, Message: res.Error, Entry: res.Entry,
|
||||
Conflicts: res.Conflicts, ConflictCount: res.ConflictCount, Need: res.Need, Avail: res.Avail,
|
||||
}
|
||||
}
|
||||
|
||||
// requireFileStage checks the caller has an account to bind an upload session
|
||||
// to and that sessions are configured, and returns the account.
|
||||
func (a *API) requireFileStage(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
p := principalFromContext(r.Context())
|
||||
if p == nil || p.UserID == "" {
|
||||
writeError(w, r, errForbidden)
|
||||
return "", false
|
||||
}
|
||||
if a.FileStage == nil || a.InternalBaseURL == "" {
|
||||
writeError(w, r, newError(http.StatusServiceUnavailable, "files_unavailable",
|
||||
"uploads are not configured"))
|
||||
return "", false
|
||||
}
|
||||
return p.UserID, true
|
||||
}
|
||||
|
||||
// authorizeServerFiles is authorizeFileOp without the stopped and world-volume
|
||||
// gates: the name is valid, the server exists, and the caller owns it or is
|
||||
// staff. It returns the server name.
|
||||
func (a *API) authorizeServerFiles(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
name := r.PathValue("name")
|
||||
if err := naming.ValidateServerName(name); err != nil {
|
||||
writeError(w, r, newError(http.StatusBadRequest, "bad_name", "invalid server name: %v", err))
|
||||
return "", false
|
||||
}
|
||||
rec, err := a.Repo.ServerByName(r.Context(), name)
|
||||
if err != nil {
|
||||
a.writeLookupError(w, r, err)
|
||||
return "", false
|
||||
}
|
||||
if !a.isOwnerOrAdmin(principalFromContext(r.Context()), rec) {
|
||||
writeError(w, r, errForbidden)
|
||||
return "", false
|
||||
}
|
||||
return name, true
|
||||
}
|
||||
|
||||
// authorizeFileSession is the gate of a session's parts, status and cancel:
|
||||
// the caller still owns the server (or is staff) and has the account the
|
||||
// session was begun under. A session answers only that account on that server
|
||||
// (fileedit.Stage), so it returns both.
|
||||
func (a *API) authorizeFileSession(w http.ResponseWriter, r *http.Request) (name, user string, ok bool) {
|
||||
name, ok = a.authorizeServerFiles(w, r)
|
||||
if !ok {
|
||||
return "", "", false
|
||||
}
|
||||
user, ok = a.requireFileStage(w, r)
|
||||
return name, user, ok
|
||||
}
|
||||
|
||||
// writeFileSessionError maps the upload session's errors onto HTTP statuses.
|
||||
func writeFileSessionError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
var offset *fileedit.OffsetError
|
||||
switch {
|
||||
case errors.Is(err, fileedit.ErrNotStaged):
|
||||
writeError(w, r, newError(http.StatusNotFound, "upload_not_found",
|
||||
"no such upload; it was cancelled, landed, or left idle too long, so start it again"))
|
||||
case errors.Is(err, fileedit.ErrStageFull):
|
||||
writeError(w, r, newError(http.StatusInsufficientStorage, "upload_staging_full",
|
||||
"felis has no room to take this upload right now; try again later or ask an admin"))
|
||||
case errors.Is(err, fileedit.ErrTooManySessions):
|
||||
writeError(w, r, newError(http.StatusTooManyRequests, "too_many_uploads",
|
||||
"you have %d uploads in progress; finish or cancel one first", fileedit.MaxSessionsPerUser))
|
||||
case errors.Is(err, fileedit.ErrUploadBusy):
|
||||
writeError(w, r, newError(http.StatusConflict, "upload_busy",
|
||||
"another request is still writing this upload; read where it stands and continue from there"))
|
||||
case errors.As(err, &offset):
|
||||
writeError(w, r, newError(http.StatusConflict, "upload_offset_mismatch",
|
||||
"the upload holds %d bytes; send the part that starts there", offset.Received))
|
||||
case errors.Is(err, fileedit.ErrPartTooLarge):
|
||||
writeError(w, r, newError(http.StatusRequestEntityTooLarge, "part_too_large",
|
||||
"the part is larger than part_max_bytes, or runs past the size the upload began with"))
|
||||
case errors.Is(err, fileedit.ErrDigestMismatch):
|
||||
writeError(w, r, errDigestMismatch())
|
||||
case errors.Is(err, fileedit.ErrShortUpload):
|
||||
writeError(w, r, newError(http.StatusBadRequest, "upload_incomplete",
|
||||
"the part ended before its Content-Length; read where the upload stands and send it again"))
|
||||
case errors.Is(err, fileedit.ErrUploadIncomplete):
|
||||
writeError(w, r, newError(http.StatusConflict, "upload_incomplete",
|
||||
"the upload has not finished arriving; read where it stands and send the rest"))
|
||||
default:
|
||||
writeError(w, r, err)
|
||||
}
|
||||
}
|
||||
@@ -1,867 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
||||
"felis.lolicon.best/internal/fileedit"
|
||||
"felis.lolicon.best/internal/maintenance"
|
||||
)
|
||||
|
||||
const (
|
||||
sessionsRoute = "/api/v1/servers/survival/files/uploads"
|
||||
unzipRoute = "/api/v1/servers/survival/files/unzip"
|
||||
opsRoute = "/api/v1/servers/survival/files/ops"
|
||||
sessionPath = "world/region/r.0.0.mca"
|
||||
)
|
||||
|
||||
var (
|
||||
fileOwner = &Principal{UserID: "owner1", Email: "[email protected]", Role: "user"}
|
||||
fileStranger = &Principal{UserID: "stranger", Email: "[email protected]", Role: "user"}
|
||||
fileAdmin = &Principal{UserID: "admin1", Email: "[email protected]", Role: "admin", ViaAdminAccess: true}
|
||||
)
|
||||
|
||||
// beginSession begins a session for size bytes at sessionPath and returns it.
|
||||
func beginSession(t *testing.T, api *API, size int) fileSessionView {
|
||||
t.Helper()
|
||||
w := do(api.ExternalHandler(), "POST", sessionsRoute+"?path="+sessionPath, `{"size":`+strconv.Itoa(size)+`}`, jsonHeader)
|
||||
if w.Code != http.StatusCreated {
|
||||
t.Fatalf("begin: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
return sessionAnswer(t, w)
|
||||
}
|
||||
|
||||
// sessionAnswer decodes a session answer, refusing a field the view lacks.
|
||||
func sessionAnswer(t *testing.T, w *httptest.ResponseRecorder) fileSessionView {
|
||||
t.Helper()
|
||||
var s fileSessionView
|
||||
dec := json.NewDecoder(strings.NewReader(w.Body.String()))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&s); err != nil {
|
||||
t.Fatalf("session answer: %v (%s)", err, w.Body.String())
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// doPart sends one part with the Content-Length given, whatever the body's own
|
||||
// length: -1 sends none, and one past the body is a part cut short. Its
|
||||
// Content-Digest is that of the body.
|
||||
func doPart(h http.Handler, target, body string, length int64) *httptest.ResponseRecorder {
|
||||
return doPartDigest(h, target, body, length, contentDigestOf(body))
|
||||
}
|
||||
|
||||
// doPartDigest is doPart with the Content-Digest given; empty sends none.
|
||||
func doPartDigest(h http.Handler, target, body string, length int64, digest string) *httptest.ResponseRecorder {
|
||||
r := httptest.NewRequest("PUT", target, strings.NewReader(body))
|
||||
r.Header.Set("Content-Type", "application/octet-stream")
|
||||
if digest != "" {
|
||||
r.Header.Set("Content-Digest", digest)
|
||||
}
|
||||
r.ContentLength = length
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
recordContract(r, body, w)
|
||||
return w
|
||||
}
|
||||
|
||||
func partAt(id string, offset int) string {
|
||||
return sessionsRoute + "/" + id + "?offset=" + strconv.Itoa(offset)
|
||||
}
|
||||
|
||||
func errMessage(t *testing.T, w *httptest.ResponseRecorder) string {
|
||||
t.Helper()
|
||||
var raw map[string]map[string]string
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &raw); err != nil {
|
||||
t.Fatalf("error body not JSON: %v (%s)", err, w.Body.String())
|
||||
}
|
||||
return raw["error"]["message"]
|
||||
}
|
||||
|
||||
// fetchStaged is the Job's fetch of what a commit handed it.
|
||||
func fetchStaged(api *API, src fileedit.UploadSource) *httptest.ResponseRecorder {
|
||||
at := strings.TrimPrefix(src.URL, api.InternalBaseURL)
|
||||
return do(api.InternalHandler(), "GET", at, "", map[string]string{"Authorization": "Bearer " + src.Token})
|
||||
}
|
||||
|
||||
var opStarted = time.Date(2026, 9, 28, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
// TestFileUploadSession drives a session from begin to the Job's fetch across
|
||||
// both faces, and each way it can go wrong on the way.
|
||||
func TestFileUploadSession(t *testing.T) {
|
||||
commit := func(api *API, id, body string) *httptest.ResponseRecorder {
|
||||
return do(api.ExternalHandler(), "POST", sessionsRoute+"/"+id+"/commit", body, jsonHeader)
|
||||
}
|
||||
status := func(api *API, id string) *httptest.ResponseRecorder {
|
||||
return do(api.ExternalHandler(), "GET", sessionsRoute+"/"+id, "", nil)
|
||||
}
|
||||
|
||||
t.Run("begin, parts, commit, and the Job fetches the whole file once", func(t *testing.T) {
|
||||
api, repo, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
// Every other internal route wants a service token; the Job has none.
|
||||
api.Internal = CallerTokens{CallerVelocity: "s3cr3t"}
|
||||
files.op = fileedit.OpState{ID: "op1", Op: fileedit.OpUpload, Path: sessionPath,
|
||||
State: fileedit.OpRunning, Started: opStarted}
|
||||
h := api.ExternalHandler()
|
||||
|
||||
s := beginSession(t, api, 10)
|
||||
// A new session lists its parts as [], which decodes non-nil; null would
|
||||
// leave a client resuming with nothing to walk.
|
||||
if len(s.ID) != 32 || s.Path != sessionPath || s.Size != 10 || s.Received != 0 || s.PartMaxBytes != fileedit.PartBytes ||
|
||||
s.Parts == nil || len(s.Parts) != 0 {
|
||||
t.Fatalf("begin = %+v", s)
|
||||
}
|
||||
|
||||
if w := doPart(h, partAt(s.ID, 0), "hello", 5); w.Code != http.StatusOK || sessionAnswer(t, w).Received != 5 {
|
||||
t.Fatalf("part 1: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
// The same part again, as a client that lost the answer might send it.
|
||||
w := doPart(h, partAt(s.ID, 0), "hello", 5)
|
||||
if w.Code != http.StatusConflict || decodeErr(t, w) != "upload_offset_mismatch" ||
|
||||
errMessage(t, w) != "the upload holds 5 bytes; send the part that starts there" {
|
||||
t.Fatalf("replayed part: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
// Too early: refused before the world lock is asked for.
|
||||
w = commit(api, s.ID, `{}`)
|
||||
if w.Code != http.StatusConflict || decodeErr(t, w) != "upload_incomplete" ||
|
||||
files.calls != 0 || len(cl.acquired) != 0 {
|
||||
t.Fatalf("early commit: code = %d calls = %d acquired %v (%s)", w.Code, files.calls, cl.acquired, w.Body.String())
|
||||
}
|
||||
|
||||
hello := sha256.Sum256([]byte("hello"))
|
||||
if w := status(api, s.ID); w.Code != http.StatusOK || !reflect.DeepEqual(sessionAnswer(t, w), fileSessionView{
|
||||
ID: s.ID, Path: sessionPath, Size: 10, Received: 5, PartMaxBytes: fileedit.PartBytes,
|
||||
Parts: []filePartView{{Size: 5, SHA256: hex.EncodeToString(hello[:])}}}) {
|
||||
t.Fatalf("status: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if w := doPart(h, partAt(s.ID, 5), "world", 5); w.Code != http.StatusOK || sessionAnswer(t, w).Received != 10 {
|
||||
t.Fatalf("part 2: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if w := doPart(h, partAt(s.ID, 10), "!", 1); w.Code != http.StatusRequestEntityTooLarge || decodeErr(t, w) != "part_too_large" {
|
||||
t.Fatalf("a part past the size: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
w = commit(api, s.ID, `{}`)
|
||||
if w.Code != http.StatusAccepted {
|
||||
t.Fatalf("commit: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
want := map[string]any{"op": map[string]any{
|
||||
"id": "op1", "op": "upload", "path": sessionPath, "state": "running",
|
||||
"started_at": "2026-09-28T10:00:00Z", "done": float64(0), "total": float64(0),
|
||||
}}
|
||||
if got := fileAnswer(t, w); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("commit answer = %v, want %v", got, want)
|
||||
}
|
||||
digest := sha256.Sum256([]byte("helloworld"))
|
||||
sum := hex.EncodeToString(digest[:])
|
||||
src := files.gotSource
|
||||
if files.calls != 1 || files.gotOp != fileedit.OpUpload || files.gotServer != "survival" || files.gotPath != sessionPath ||
|
||||
src.URL != api.InternalBaseURL+"/api/v1/internal/file-uploads/"+s.ID ||
|
||||
src.Size != 10 || src.SHA256 != sum || len(src.Token) != 64 || files.gotOverwrite {
|
||||
t.Fatalf("executor saw calls=%d op %q server %q path %q source %+v overwrite %v",
|
||||
files.calls, files.gotOp, files.gotServer, files.gotPath, src, files.gotOverwrite)
|
||||
}
|
||||
if strings.Join(cl.acquired, ",") != "survival:"+maintenance.KindFileWrite || strings.Join(cl.released, ",") != "survival" {
|
||||
t.Fatalf("lock acquired %v, released %v", cl.acquired, cl.released)
|
||||
}
|
||||
onlyAudit(t, repo, "file.upload", "survival:"+sessionPath, `{"overwrite":false,"sha256":"`+sum+`","size_bytes":10}`)
|
||||
|
||||
fetched := fetchStaged(api, src)
|
||||
if fetched.Code != http.StatusOK || fetched.Body.String() != "helloworld" || fetched.Header().Get("Content-Length") != "10" {
|
||||
t.Fatalf("fetch: code = %d %q Content-Length %q", fetched.Code, fetched.Body.String(), fetched.Header().Get("Content-Length"))
|
||||
}
|
||||
if again := fetchStaged(api, src); again.Code != http.StatusNotFound {
|
||||
t.Fatalf("second fetch: code = %d", again.Code)
|
||||
}
|
||||
// Served whole, the session stays until the Job says the file landed: a
|
||||
// Job that fails after its fetch leaves the upload to be committed again.
|
||||
if w := status(api, s.ID); w.Code != http.StatusOK || sessionAnswer(t, w).Received != 10 {
|
||||
t.Fatalf("status after the fetch: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
landed := func(token string) *httptest.ResponseRecorder {
|
||||
return do(api.InternalHandler(), "DELETE", strings.TrimPrefix(src.URL, api.InternalBaseURL), "",
|
||||
map[string]string{"Authorization": "Bearer " + token})
|
||||
}
|
||||
if w := landed(strings.Repeat("0", len(src.Token))); w.Code != http.StatusNotFound || decodeErr(t, w) != "not_found" {
|
||||
t.Fatalf("landed with the wrong token: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if w := status(api, s.ID); w.Code != http.StatusOK {
|
||||
t.Fatalf("a wrong token let go of the session: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if w := landed(src.Token); w.Code != http.StatusNoContent {
|
||||
t.Fatalf("landed: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if w := status(api, s.ID); w.Code != http.StatusNotFound || decodeErr(t, w) != "upload_not_found" {
|
||||
t.Fatalf("status after it landed: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
stageEmpty(t, api)
|
||||
if w := landed(src.Token); w.Code != http.StatusNotFound {
|
||||
t.Fatalf("landed twice: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a Job that never fetched is committed again with a fresh token", func(t *testing.T) {
|
||||
api, repo, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
s := beginSession(t, api, 3)
|
||||
doPart(api.ExternalHandler(), partAt(s.ID, 0), "abc", 3)
|
||||
if w := commit(api, s.ID, `{}`); w.Code != http.StatusAccepted {
|
||||
t.Fatalf("first commit: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
first := files.gotSource
|
||||
if w := commit(api, s.ID, `{"overwrite":true}`); w.Code != http.StatusAccepted || !files.gotOverwrite {
|
||||
t.Fatalf("second commit: code = %d overwrite %v (%s)", w.Code, files.gotOverwrite, w.Body.String())
|
||||
}
|
||||
second := files.gotSource
|
||||
if w := fetchStaged(api, first); w.Code != http.StatusNotFound {
|
||||
t.Fatalf("the first commit's token still opens it: code = %d", w.Code)
|
||||
}
|
||||
if w := fetchStaged(api, second); w.Code != http.StatusOK || w.Body.String() != "abc" {
|
||||
t.Fatalf("fetch: code = %d %q", w.Code, w.Body.String())
|
||||
}
|
||||
if len(repo.audits) != 2 || string(repo.audits[1].Payload) != `{"overwrite":true,"sha256":"`+second.SHA256+`","size_bytes":3}` {
|
||||
t.Fatalf("audits = %+v", repo.audits)
|
||||
}
|
||||
})
|
||||
|
||||
// The running Job's hold on the world refuses the commit before Seal, so the
|
||||
// token that Job carries still opens the file.
|
||||
t.Run("a commit while the world is held leaves the running Job's token alone", func(t *testing.T) {
|
||||
api, _, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
s := beginSession(t, api, 3)
|
||||
doPart(api.ExternalHandler(), partAt(s.ID, 0), "abc", 3)
|
||||
commit(api, s.ID, `{}`)
|
||||
running := files.gotSource
|
||||
cl.maintErr["survival"] = &MaintenanceBusyError{Kind: maintenance.KindFileWrite}
|
||||
w := commit(api, s.ID, `{}`)
|
||||
if w.Code != http.StatusConflict || decodeErr(t, w) != "maintenance_in_progress" || files.calls != 1 {
|
||||
t.Fatalf("code = %d calls = %d (%s)", w.Code, files.calls, w.Body.String())
|
||||
}
|
||||
if w := fetchStaged(api, running); w.Code != http.StatusOK || w.Body.String() != "abc" {
|
||||
t.Fatalf("the running Job lost its file: code = %d %q", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a Job that could not start is not audited and lets go of the world", func(t *testing.T) {
|
||||
api, repo, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
s := beginSession(t, api, 3)
|
||||
doPart(api.ExternalHandler(), partAt(s.ID, 0), "abc", 3)
|
||||
files.err = errors.New("the cluster said no")
|
||||
w := commit(api, s.ID, `{}`)
|
||||
if w.Code != http.StatusInternalServerError || len(repo.audits) != 0 || strings.Join(cl.released, ",") != "survival" {
|
||||
t.Fatalf("code = %d audits %+v released %v (%s)", w.Code, repo.audits, cl.released, w.Body.String())
|
||||
}
|
||||
if w := status(api, s.ID); w.Code != http.StatusOK || sessionAnswer(t, w).Received != 3 {
|
||||
t.Fatalf("the session went with the failed start: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a fetch cut short keeps the session for the next commit", func(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
s := beginSession(t, api, 3)
|
||||
doPart(api.ExternalHandler(), partAt(s.ID, 0), "abc", 3)
|
||||
commit(api, s.ID, `{}`)
|
||||
src := files.gotSource
|
||||
r := httptest.NewRequest("GET", strings.TrimPrefix(src.URL, api.InternalBaseURL), nil)
|
||||
r.Header.Set("Authorization", "Bearer "+src.Token)
|
||||
cut := &brokenWriter{header: http.Header{}}
|
||||
api.InternalHandler().ServeHTTP(cut, r)
|
||||
if cut.code != http.StatusOK {
|
||||
t.Fatalf("cut fetch: code = %d", cut.code)
|
||||
}
|
||||
if w := status(api, s.ID); w.Code != http.StatusOK {
|
||||
t.Fatalf("status after a cut fetch: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
commit(api, s.ID, `{}`)
|
||||
if w := fetchStaged(api, files.gotSource); w.Code != http.StatusOK || w.Body.String() != "abc" {
|
||||
t.Fatalf("refetch: code = %d %q", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a part changed on the way is refused and taken back", func(t *testing.T) {
|
||||
api, _, _, _ := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
h := api.ExternalHandler()
|
||||
s := beginSession(t, api, 10)
|
||||
doPart(h, partAt(s.ID, 0), "hello", 5)
|
||||
for _, c := range []struct {
|
||||
name, digest, errCode string
|
||||
}{
|
||||
{"another part's digest", contentDigestOf("wor1d"), "digest_mismatch"},
|
||||
{"no digest", "", "digest_required"},
|
||||
{"a malformed digest", "sha-256=:bm90IGEgc3VtCg==:", "bad_digest"},
|
||||
} {
|
||||
w := doPartDigest(h, partAt(s.ID, 5), "world", 5, c.digest)
|
||||
if w.Code != http.StatusBadRequest || decodeErr(t, w) != c.errCode {
|
||||
t.Fatalf("%s: code = %d (%s), want 400 %s", c.name, w.Code, w.Body.String(), c.errCode)
|
||||
}
|
||||
if got := sessionAnswer(t, status(api, s.ID)); got.Received != 5 || len(got.Parts) != 1 {
|
||||
t.Fatalf("%s: session after the refused part = %+v", c.name, got)
|
||||
}
|
||||
}
|
||||
if w := doPart(h, partAt(s.ID, 5), "world", 5); w.Code != http.StatusOK || sessionAnswer(t, w).Received != 10 {
|
||||
t.Fatalf("the part sent again: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a part cut short leaves the session where it was", func(t *testing.T) {
|
||||
api, _, _, _ := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
s := beginSession(t, api, 10)
|
||||
w := doPart(api.ExternalHandler(), partAt(s.ID, 0), "abc", 5)
|
||||
if w.Code != http.StatusBadRequest || decodeErr(t, w) != "upload_incomplete" {
|
||||
t.Fatalf("code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if w := status(api, s.ID); sessionAnswer(t, w).Received != 0 {
|
||||
t.Fatalf("status = %s", w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("while a part arrives, the session takes nothing else", func(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
h := api.ExternalHandler()
|
||||
s := beginSession(t, api, 6)
|
||||
pr, pw := io.Pipe()
|
||||
arriving := make(chan int)
|
||||
go func() {
|
||||
r := httptest.NewRequest("PUT", partAt(s.ID, 0), pr)
|
||||
r.Header.Set("Content-Type", "application/octet-stream")
|
||||
r.Header.Set("Content-Digest", contentDigestOf("abc"))
|
||||
r.ContentLength = 3
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
arriving <- w.Code
|
||||
}()
|
||||
// The write returns once the handler has read the byte, so the part is
|
||||
// being appended.
|
||||
if _, err := pw.Write([]byte("a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, w := range map[string]*httptest.ResponseRecorder{
|
||||
"another part": doPart(h, partAt(s.ID, 0), "xyz", 3),
|
||||
"cancel": do(h, "DELETE", sessionsRoute+"/"+s.ID, "", nil),
|
||||
} {
|
||||
if w.Code != http.StatusConflict || decodeErr(t, w) != "upload_busy" {
|
||||
t.Errorf("%s: code = %d (%s)", name, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
pw.CloseWithError(errors.New("the client went away"))
|
||||
if code := <-arriving; code != http.StatusBadRequest {
|
||||
t.Fatalf("the broken part answered %d", code)
|
||||
}
|
||||
if w := status(api, s.ID); w.Code != http.StatusOK || sessionAnswer(t, w).Received != 0 || files.calls != 0 {
|
||||
t.Fatalf("status: code = %d calls = %d (%s)", w.Code, files.calls, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("parts refused before a byte is read", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, target string
|
||||
length int64
|
||||
code int
|
||||
errCode string
|
||||
}{
|
||||
{"no offset", sessionsRoute + "/%s", 3, http.StatusBadRequest, "bad_request"},
|
||||
{"an offset that is no number", sessionsRoute + "/%s?offset=abc", 3, http.StatusBadRequest, "bad_request"},
|
||||
{"no Content-Length", sessionsRoute + "/%s?offset=0", -1, http.StatusLengthRequired, "length_required"},
|
||||
{"a part over the cap", sessionsRoute + "/%s?offset=0", fileedit.PartBytes + 1, http.StatusRequestEntityTooLarge, "part_too_large"},
|
||||
// At the cap it is taken, and the three bytes behind it end short.
|
||||
{"a part at the cap", sessionsRoute + "/%s?offset=0", fileedit.PartBytes, http.StatusBadRequest, "upload_incomplete"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
api, _, _, _ := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
s := beginSession(t, api, fileedit.PartBytes+10)
|
||||
w := doPart(api.ExternalHandler(), fmt.Sprintf(c.target, s.ID), "abc", c.length)
|
||||
if w.Code != c.code || decodeErr(t, w) != c.errCode {
|
||||
t.Fatalf("code = %d (%s), want %d %s", w.Code, w.Body.String(), c.code, c.errCode)
|
||||
}
|
||||
if w := status(api, s.ID); sessionAnswer(t, w).Received != 0 {
|
||||
t.Fatalf("status = %s", w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
// The parts need only the account and the server: starting the server midway
|
||||
// costs the upload nothing but the commit.
|
||||
t.Run("parts, status and cancel go on while the server runs", func(t *testing.T) {
|
||||
api, _, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
h := api.ExternalHandler()
|
||||
s := beginSession(t, api, 3)
|
||||
cl.byName["survival"].Ready = true
|
||||
cl.byName["survival"].DesiredState = string(v1alpha1.DesiredRunning)
|
||||
if w := doPart(h, partAt(s.ID, 0), "abc", 3); w.Code != http.StatusOK {
|
||||
t.Fatalf("part: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if w := status(api, s.ID); w.Code != http.StatusOK {
|
||||
t.Fatalf("status: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if w := commit(api, s.ID, `{}`); w.Code != http.StatusConflict || decodeErr(t, w) != "not_stopped" || files.calls != 0 {
|
||||
t.Fatalf("commit: code = %d calls = %d (%s)", w.Code, files.calls, w.Body.String())
|
||||
}
|
||||
if w := do(h, "DELETE", sessionsRoute+"/"+s.ID, "", nil); w.Code != http.StatusNoContent {
|
||||
t.Fatalf("cancel: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if w := status(api, s.ID); w.Code != http.StatusNotFound || decodeErr(t, w) != "upload_not_found" {
|
||||
t.Fatalf("status after cancel: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
stageEmpty(t, api)
|
||||
})
|
||||
|
||||
t.Run("a session answers only the account and server it was begun for", func(t *testing.T) {
|
||||
api, repo, _, files := mkFiles(t)
|
||||
repo.byName["creative"] = &ServerRecord{Name: "creative", OwnerID: "owner1"}
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
s := beginSession(t, api, 3)
|
||||
|
||||
elsewhere := strings.Replace(sessionsRoute, "survival", "creative", 1) + "/" + s.ID
|
||||
if w := do(api.ExternalHandler(), "GET", elsewhere, "", nil); w.Code != http.StatusNotFound || decodeErr(t, w) != "upload_not_found" {
|
||||
t.Fatalf("another server: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
|
||||
// Staff may reach the server, and still not someone else's session.
|
||||
api.External = staticExternal{p: fileAdmin}
|
||||
h := api.ExternalHandler()
|
||||
for name, w := range map[string]*httptest.ResponseRecorder{
|
||||
"status": status(api, s.ID),
|
||||
"part": doPart(h, partAt(s.ID, 0), "abc", 3),
|
||||
"cancel": do(h, "DELETE", sessionsRoute+"/"+s.ID, "", nil),
|
||||
"commit": commit(api, s.ID, `{}`),
|
||||
} {
|
||||
if w.Code != http.StatusNotFound || decodeErr(t, w) != "upload_not_found" {
|
||||
t.Errorf("admin %s: code = %d (%s)", name, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
if files.calls != 0 {
|
||||
t.Fatal("another account's commit reached the executor")
|
||||
}
|
||||
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
if w := status(api, s.ID); w.Code != http.StatusOK || sessionAnswer(t, w).Received != 0 {
|
||||
t.Fatalf("the owner's session was touched: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a stranger is refused on every session route", func(t *testing.T) {
|
||||
api, _, _, _ := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
s := beginSession(t, api, 3)
|
||||
api.External = staticExternal{p: fileStranger}
|
||||
h := api.ExternalHandler()
|
||||
for name, w := range map[string]*httptest.ResponseRecorder{
|
||||
"begin": do(h, "POST", sessionsRoute+"?path=a.jar", `{"size":3}`, jsonHeader),
|
||||
"status": status(api, s.ID),
|
||||
"part": doPart(h, partAt(s.ID, 0), "abc", 3),
|
||||
"cancel": do(h, "DELETE", sessionsRoute+"/"+s.ID, "", nil),
|
||||
"commit": commit(api, s.ID, `{}`),
|
||||
} {
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Errorf("%s: code = %d (%s)", name, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("begin refused", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, target, body string
|
||||
setup func(*API)
|
||||
code int
|
||||
errCode string
|
||||
}{
|
||||
{"no size", sessionsRoute + "?path=a.jar", `{}`, nil, http.StatusBadRequest, "bad_request"},
|
||||
{"a negative size", sessionsRoute + "?path=a.jar", `{"size":-1}`, nil, http.StatusBadRequest, "bad_request"},
|
||||
{"no path", sessionsRoute, `{"size":3}`, nil, http.StatusBadRequest, "bad_request"},
|
||||
{"a path leaving the world", sessionsRoute + "?path=../a.jar", `{"size":3}`, nil, http.StatusBadRequest, "bad_path"},
|
||||
{"an absolute path", sessionsRoute + "?path=/etc/a.jar", `{"size":3}`, nil, http.StatusBadRequest, "bad_path"},
|
||||
{"the world folder itself", sessionsRoute + "?path=plugins/..", `{"size":3}`, nil, http.StatusBadRequest, "bad_path"},
|
||||
{"a name longer than a folder entry holds", sessionsRoute + "?path=plugins/" + strings.Repeat("n", fileedit.NameMax-3) + ".jar", `{"size":3}`,
|
||||
nil, http.StatusBadRequest, "bad_path"},
|
||||
{"a staging disk at its floor", sessionsRoute + "?path=a.jar", `{"size":3}`,
|
||||
func(a *API) { a.FileStage.MinFree = 1 }, http.StatusInsufficientStorage, "upload_staging_full"},
|
||||
{"no stage", sessionsRoute + "?path=a.jar", `{"size":3}`,
|
||||
func(a *API) { a.FileStage = nil }, http.StatusServiceUnavailable, "files_unavailable"},
|
||||
{"no internal URL", sessionsRoute + "?path=a.jar", `{"size":3}`,
|
||||
func(a *API) { a.InternalBaseURL = "" }, http.StatusServiceUnavailable, "files_unavailable"},
|
||||
{"a caller with no account", sessionsRoute + "?path=a.jar", `{"size":3}`,
|
||||
func(a *API) { a.External = staticExternal{p: &Principal{Role: "admin", ViaAdminAccess: true}} },
|
||||
http.StatusForbidden, "forbidden"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
api, _, _, _ := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
if c.setup != nil {
|
||||
c.setup(api)
|
||||
}
|
||||
w := do(api.ExternalHandler(), "POST", c.target, c.body, jsonHeader)
|
||||
if w.Code != c.code || decodeErr(t, w) != c.errCode {
|
||||
t.Fatalf("code = %d (%s), want %d %s", w.Code, w.Body.String(), c.code, c.errCode)
|
||||
}
|
||||
if api.FileStage != nil {
|
||||
stageEmpty(t, api)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a name as long as a folder entry holds is taken", func(t *testing.T) {
|
||||
api, _, _, _ := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
long := strings.Repeat("n", fileedit.NameMax-4) + ".jar"
|
||||
w := do(api.ExternalHandler(), "POST", sessionsRoute+"?path=plugins/"+long, `{"size":3}`, jsonHeader)
|
||||
if w.Code != http.StatusCreated {
|
||||
t.Fatalf("code = %d (%s), want 201", w.Code, w.Body.String())
|
||||
}
|
||||
if s := sessionAnswer(t, w); s.Path != "plugins/"+long {
|
||||
t.Fatalf("path = %q", s.Path)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a fifth upload at once -> 429", func(t *testing.T) {
|
||||
api, _, _, _ := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
for i := 0; i < fileedit.MaxSessionsPerUser; i++ {
|
||||
beginSession(t, api, 1)
|
||||
}
|
||||
w := do(api.ExternalHandler(), "POST", sessionsRoute+"?path=a.jar", `{"size":1}`, jsonHeader)
|
||||
if w.Code != http.StatusTooManyRequests || decodeErr(t, w) != "too_many_uploads" {
|
||||
t.Fatalf("code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a commit or cancel of no session -> 404", func(t *testing.T) {
|
||||
api, _, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
const id = "00112233445566778899aabbccddeeff"
|
||||
for name, w := range map[string]*httptest.ResponseRecorder{
|
||||
"commit": commit(api, id, `{}`),
|
||||
"cancel": do(api.ExternalHandler(), "DELETE", sessionsRoute+"/"+id, "", nil),
|
||||
"part": doPart(api.ExternalHandler(), partAt(id, 0), "abc", 3),
|
||||
} {
|
||||
if w.Code != http.StatusNotFound || decodeErr(t, w) != "upload_not_found" {
|
||||
t.Errorf("%s: code = %d (%s)", name, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
if files.calls != 0 || len(cl.acquired) != 0 {
|
||||
t.Fatalf("calls = %d acquired %v", files.calls, cl.acquired)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a commit needs a body", func(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
s := beginSession(t, api, 0)
|
||||
if w := commit(api, s.ID, ""); w.Code != http.StatusBadRequest || files.calls != 0 {
|
||||
t.Fatalf("code = %d calls = %d (%s)", w.Code, files.calls, w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// brokenWriter takes the headers and fails every write, as a connection that
|
||||
// dropped once the answer began does.
|
||||
type brokenWriter struct {
|
||||
header http.Header
|
||||
code int
|
||||
}
|
||||
|
||||
func (b *brokenWriter) Header() http.Header { return b.header }
|
||||
func (b *brokenWriter) WriteHeader(code int) { b.code = code }
|
||||
func (b *brokenWriter) Write(p []byte) (int, error) { return 0, errors.New("connection reset") }
|
||||
|
||||
// TestFileOpsWorldGates pins the stopped and world-volume gates on the routes
|
||||
// that begin or start a background op: each refuses before a byte is staged, a
|
||||
// lock is asked for, or a Job is created.
|
||||
func TestFileOpsWorldGates(t *testing.T) {
|
||||
routes := []struct{ name, target, body string }{
|
||||
{"begin", sessionsRoute + "?path=a.jar", `{"size":3}`},
|
||||
{"commit", sessionsRoute + "/00112233445566778899aabbccddeeff/commit", `{}`},
|
||||
{"unzip", unzipRoute + "?path=maps/a.zip", `{}`},
|
||||
}
|
||||
gates := []struct {
|
||||
name, code string
|
||||
set func(*fakeCluster)
|
||||
}{
|
||||
{"running", "not_stopped", func(c *fakeCluster) {
|
||||
c.byName["survival"].Ready = true
|
||||
c.byName["survival"].DesiredState = string(v1alpha1.DesiredRunning)
|
||||
}},
|
||||
{"starting", "not_stopped", func(c *fakeCluster) {
|
||||
c.byName["survival"].DesiredState = string(v1alpha1.DesiredRunning)
|
||||
}},
|
||||
{"no world volume", "no_world_volume", func(c *fakeCluster) { c.noWorld["survival"] = true }},
|
||||
}
|
||||
for _, rt := range routes {
|
||||
for _, g := range gates {
|
||||
t.Run(rt.name+" on a "+g.name+" server", func(t *testing.T) {
|
||||
api, _, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
g.set(cl)
|
||||
w := do(api.ExternalHandler(), "POST", rt.target, rt.body, jsonHeader)
|
||||
if w.Code != http.StatusConflict || decodeErr(t, w) != g.code {
|
||||
t.Fatalf("code = %d (%s), want 409 %s", w.Code, w.Body.String(), g.code)
|
||||
}
|
||||
if files.calls != 0 || len(cl.acquired) != 0 {
|
||||
t.Fatalf("calls = %d acquired %v", files.calls, cl.acquired)
|
||||
}
|
||||
stageEmpty(t, api)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileUnzip(t *testing.T) {
|
||||
unzip := func(api *API, path, body string) *httptest.ResponseRecorder {
|
||||
return do(api.ExternalHandler(), "POST", unzipRoute+"?path="+path, body, jsonHeader)
|
||||
}
|
||||
|
||||
t.Run("starts the Job under the world lock and audits it", func(t *testing.T) {
|
||||
api, repo, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
files.op = fileedit.OpState{ID: "op2", Op: fileedit.OpUnzip, Path: "maps/a.zip",
|
||||
State: fileedit.OpRunning, Started: opStarted}
|
||||
w := unzip(api, "maps/a.zip", `{}`)
|
||||
if w.Code != http.StatusAccepted {
|
||||
t.Fatalf("code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
want := map[string]any{"op": map[string]any{
|
||||
"id": "op2", "op": "unzip", "path": "maps/a.zip", "state": "running",
|
||||
"started_at": "2026-09-28T10:00:00Z", "done": float64(0), "total": float64(0),
|
||||
}}
|
||||
if got := fileAnswer(t, w); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("answer = %v, want %v", got, want)
|
||||
}
|
||||
if files.calls != 1 || files.gotOp != fileedit.OpUnzip || files.gotServer != "survival" ||
|
||||
files.gotPath != "maps/a.zip" || files.gotOverwrite {
|
||||
t.Fatalf("executor saw calls=%d op %q server %q path %q overwrite %v",
|
||||
files.calls, files.gotOp, files.gotServer, files.gotPath, files.gotOverwrite)
|
||||
}
|
||||
if strings.Join(cl.acquired, ",") != "survival:"+maintenance.KindFileWrite || strings.Join(cl.released, ",") != "survival" {
|
||||
t.Fatalf("lock acquired %v, released %v", cl.acquired, cl.released)
|
||||
}
|
||||
onlyAudit(t, repo, "file.unzip", "survival:maps/a.zip", `{"overwrite":false}`)
|
||||
})
|
||||
|
||||
t.Run("overwrite reaches the executor, and the suffix is any case", func(t *testing.T) {
|
||||
api, repo, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
if w := unzip(api, "maps/A.ZIP", `{"overwrite":true}`); w.Code != http.StatusAccepted || !files.gotOverwrite || files.gotPath != "maps/A.ZIP" {
|
||||
t.Fatalf("code = %d overwrite %v path %q (%s)", w.Code, files.gotOverwrite, files.gotPath, w.Body.String())
|
||||
}
|
||||
onlyAudit(t, repo, "file.unzip", "survival:maps/A.ZIP", `{"overwrite":true}`)
|
||||
})
|
||||
|
||||
t.Run("refused before the lock", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, path, body string
|
||||
code int
|
||||
errCode string
|
||||
}{
|
||||
{"not a zip", "maps/a.tar.gz", `{}`, http.StatusBadRequest, "bad_path"},
|
||||
{"zip only inside the name", "maps/a.zip.bak", `{}`, http.StatusBadRequest, "bad_path"},
|
||||
{"no path", "", `{}`, http.StatusBadRequest, "bad_request"},
|
||||
{"no body", "maps/a.zip", "", http.StatusBadRequest, "bad_request"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
api, repo, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
w := unzip(api, c.path, c.body)
|
||||
if w.Code != c.code || decodeErr(t, w) != c.errCode {
|
||||
t.Fatalf("code = %d (%s), want %d %s", w.Code, w.Body.String(), c.code, c.errCode)
|
||||
}
|
||||
if files.calls != 0 || len(cl.acquired) != 0 || len(repo.audits) != 0 {
|
||||
t.Fatalf("calls = %d acquired %v audits %+v", files.calls, cl.acquired, repo.audits)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a held world -> 409, no Job", func(t *testing.T) {
|
||||
api, _, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
cl.maintErr["survival"] = &MaintenanceBusyError{Kind: maintenance.KindBackup}
|
||||
if w := unzip(api, "maps/a.zip", `{}`); w.Code != http.StatusConflict || decodeErr(t, w) != "maintenance_in_progress" || files.calls != 0 {
|
||||
t.Fatalf("code = %d calls = %d (%s)", w.Code, files.calls, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a Job that could not start is not audited and lets go of the world", func(t *testing.T) {
|
||||
api, repo, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
files.err = errors.New("the cluster said no")
|
||||
w := unzip(api, "maps/a.zip", `{}`)
|
||||
if w.Code != http.StatusInternalServerError || len(repo.audits) != 0 || strings.Join(cl.released, ",") != "survival" {
|
||||
t.Fatalf("code = %d audits %+v released %v (%s)", w.Code, repo.audits, cl.released, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a stranger -> 403, an admin may", func(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileStranger}
|
||||
if w := unzip(api, "maps/a.zip", `{}`); w.Code != http.StatusForbidden || files.calls != 0 {
|
||||
t.Fatalf("stranger: code = %d calls = %d", w.Code, files.calls)
|
||||
}
|
||||
api.External = staticExternal{p: fileAdmin}
|
||||
if w := unzip(api, "maps/a.zip", `{}`); w.Code != http.StatusAccepted || files.calls != 1 {
|
||||
t.Fatalf("admin: code = %d calls = %d (%s)", w.Code, files.calls, w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFileOps(t *testing.T) {
|
||||
ended := opStarted.Add(3 * time.Minute)
|
||||
list := func(api *API) *httptest.ResponseRecorder {
|
||||
return do(api.ExternalHandler(), "GET", opsRoute, "", nil)
|
||||
}
|
||||
|
||||
t.Run("each state and failure as the API shows it", func(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
base := func(id, op, state string) fileedit.OpState {
|
||||
s := fileedit.OpState{ID: id, Op: op, Path: "maps/a.zip", State: state, Started: opStarted}
|
||||
if state != fileedit.OpRunning {
|
||||
s.Finished = ended
|
||||
}
|
||||
return s
|
||||
}
|
||||
running := base("a", fileedit.OpUnzip, fileedit.OpRunning)
|
||||
running.Done, running.Total = 40, 100
|
||||
unzipped := base("b", fileedit.OpUnzip, fileedit.OpSucceeded)
|
||||
unzipped.Done, unzipped.Total = 100, 100
|
||||
unzipped.Result = &fileedit.Result{Files: 7, Bytes: 100}
|
||||
uploaded := base("c", fileedit.OpUpload, fileedit.OpSucceeded)
|
||||
uploaded.Result = &fileedit.Result{SHA256: testSum}
|
||||
exists := base("d", fileedit.OpUnzip, fileedit.OpFailed)
|
||||
exists.Result = &fileedit.Result{Code: fileedit.CodeExists, Error: "2 files are already there",
|
||||
Conflicts: []string{"maps/level.dat", "maps/r.0.0.mca"}, ConflictCount: 2}
|
||||
full := base("e", fileedit.OpUpload, fileedit.OpFailed)
|
||||
full.Result = &fileedit.Result{Code: fileedit.CodeNoSpace, Error: "no room", Need: 900, Avail: 100}
|
||||
changed := base("f", fileedit.OpUpload, fileedit.OpFailed)
|
||||
changed.Result = &fileedit.Result{Code: fileedit.CodeConflict, Error: "the bytes changed"}
|
||||
unsafe := base("g", fileedit.OpUnzip, fileedit.OpFailed)
|
||||
unsafe.Result = &fileedit.Result{Code: fileedit.CodeArchiveUnsafe, Error: "leaves the folder", Entry: "../x",
|
||||
Files: 3, Bytes: 9}
|
||||
deadline := base("h", fileedit.OpUnzip, fileedit.OpFailed)
|
||||
deadline.Reason = "DeadlineExceeded"
|
||||
killed := base("i", fileedit.OpUpload, fileedit.OpFailed)
|
||||
killed.Reason = "BackoffLimitExceeded"
|
||||
oom := base("j", fileedit.OpUnzip, fileedit.OpFailed)
|
||||
oom.Reason = "OOMKilled"
|
||||
files.ops = []fileedit.OpState{running, unzipped, uploaded, exists, full, changed, unsafe, deadline, killed, oom}
|
||||
|
||||
w := list(api)
|
||||
if w.Code != http.StatusOK || files.gotServer != "survival" {
|
||||
t.Fatalf("code = %d server %q (%s)", w.Code, files.gotServer, w.Body.String())
|
||||
}
|
||||
op := func(id, kind, state string, extra map[string]any) map[string]any {
|
||||
m := map[string]any{"id": id, "op": kind, "path": "maps/a.zip", "state": state,
|
||||
"started_at": "2026-09-28T10:00:00Z", "done": float64(0), "total": float64(0)}
|
||||
if state != "running" {
|
||||
m["finished_at"] = "2026-09-28T10:03:00Z"
|
||||
}
|
||||
for k, v := range extra {
|
||||
m[k] = v
|
||||
}
|
||||
return m
|
||||
}
|
||||
failure := func(code, msg string, extra map[string]any) map[string]any {
|
||||
m := map[string]any{"code": code, "message": msg}
|
||||
for k, v := range extra {
|
||||
m[k] = v
|
||||
}
|
||||
return map[string]any{"error": m}
|
||||
}
|
||||
want := map[string]any{"ops": []any{
|
||||
op("a", "unzip", "running", map[string]any{"done": float64(40), "total": float64(100)}),
|
||||
op("b", "unzip", "succeeded", map[string]any{"done": float64(100), "total": float64(100),
|
||||
"files": float64(7), "bytes": float64(100)}),
|
||||
op("c", "upload", "succeeded", nil),
|
||||
op("d", "unzip", "failed", failure("file_exists", "2 files are already there", map[string]any{
|
||||
"conflicts": []any{"maps/level.dat", "maps/r.0.0.mca"}, "conflict_count": float64(2)})),
|
||||
op("e", "upload", "failed", failure("volume_full", "no room", map[string]any{
|
||||
"need": float64(900), "avail": float64(100)})),
|
||||
op("f", "upload", "failed", failure("file_changed", "the bytes changed", nil)),
|
||||
op("g", "unzip", "failed", failure("archive_unsafe", "leaves the folder", map[string]any{"entry": "../x"})),
|
||||
op("h", "unzip", "failed", failure("job_failed",
|
||||
"the file operation ran out of time (DeadlineExceeded); run it again", nil)),
|
||||
op("i", "upload", "failed", failure("job_failed",
|
||||
"the file operation stopped before it could report how it went (BackoffLimitExceeded); run it again", nil)),
|
||||
op("j", "unzip", "failed", failure("job_failed",
|
||||
"the file operation ran out of memory (OOMKilled); an archive of this many files has to be split into smaller ones", nil)),
|
||||
}}
|
||||
if got := fileAnswer(t, w); !reflect.DeepEqual(got, want) {
|
||||
gotJSON, _ := json.MarshalIndent(got, "", " ")
|
||||
wantJSON, _ := json.MarshalIndent(want, "", " ")
|
||||
t.Fatalf("ops =\n%s\nwant\n%s", gotJSON, wantJSON)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("none is an empty list", func(t *testing.T) {
|
||||
api, _, _, _ := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
if w := list(api); w.Code != http.StatusOK || strings.TrimSpace(w.Body.String()) != `{"ops":[]}` {
|
||||
t.Fatalf("code = %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("answers while the server runs", func(t *testing.T) {
|
||||
api, _, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
cl.byName["survival"].Ready = true
|
||||
cl.byName["survival"].DesiredState = string(v1alpha1.DesiredRunning)
|
||||
cl.noWorld["survival"] = true
|
||||
if w := list(api); w.Code != http.StatusOK || files.calls != 1 {
|
||||
t.Fatalf("code = %d calls = %d (%s)", w.Code, files.calls, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("who may look", func(t *testing.T) {
|
||||
api, repo, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileStranger}
|
||||
if w := list(api); w.Code != http.StatusForbidden || files.calls != 0 {
|
||||
t.Fatalf("stranger: code = %d calls = %d", w.Code, files.calls)
|
||||
}
|
||||
repo.byName["survival"].OwnerID = "someone-else"
|
||||
api.External = staticExternal{p: fileAdmin}
|
||||
if w := list(api); w.Code != http.StatusOK || files.calls != 1 {
|
||||
t.Fatalf("admin: code = %d calls = %d (%s)", w.Code, files.calls, w.Body.String())
|
||||
}
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
if w := do(api.ExternalHandler(), "GET", strings.Replace(opsRoute, "survival", "missing", 1), "", nil); w.Code != http.StatusNotFound {
|
||||
t.Fatalf("unknown server: code = %d", w.Code)
|
||||
}
|
||||
if w := do(api.ExternalHandler(), "GET", strings.Replace(opsRoute, "survival", "X", 1), "", nil); w.Code != http.StatusBadRequest || decodeErr(t, w) != "bad_name" {
|
||||
t.Fatalf("bad name: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no executor -> 503, a failing one -> 500", func(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: fileOwner}
|
||||
files.err = errors.New("the cluster said no")
|
||||
if w := list(api); w.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("failing: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
api.Files = nil
|
||||
if w := list(api); w.Code != http.StatusServiceUnavailable || decodeErr(t, w) != "files_unavailable" {
|
||||
t.Fatalf("nil: code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
+29
-156
@@ -2,9 +2,6 @@ package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -15,6 +12,7 @@ import (
|
||||
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
||||
"felis.lolicon.best/internal/fileedit"
|
||||
"felis.lolicon.best/internal/maintenance"
|
||||
"felis.lolicon.best/internal/naming"
|
||||
)
|
||||
|
||||
// FileEditor is the server-file-editor surface the API depends on: list a
|
||||
@@ -41,23 +39,18 @@ import (
|
||||
// 507 / 409.
|
||||
//
|
||||
// Read and Write return the file's SHA-256 (hex); Upload lands exactly the bytes
|
||||
// src describes or fails. StartUpload and StartUnzip are the two that can run
|
||||
// longer than a request: they start their Job and return, and Ops reports it
|
||||
// (handlers_fileops.go). List also reports the room left on the volume. Write's expect is the
|
||||
// src describes or fails. Write's expect is the
|
||||
// hash a client read the file at; when set, a file that changed since is refused
|
||||
// with ErrConflict instead of being overwritten. createOnly and a false overwrite
|
||||
// refuse an existing path with ErrExists.
|
||||
type FileEditor interface {
|
||||
List(ctx context.Context, server, path string) (fileedit.Listing, error)
|
||||
List(ctx context.Context, server, path string) (entries []fileedit.Entry, truncated bool, err error)
|
||||
Read(ctx context.Context, server, path string) (content []byte, sha256 string, err error)
|
||||
Write(ctx context.Context, server, path string, content []byte, expect string, createOnly bool) (sha256 string, err error)
|
||||
Mkdir(ctx context.Context, server, path string) error
|
||||
Delete(ctx context.Context, server, path string) error
|
||||
Rename(ctx context.Context, server, path, to string) error
|
||||
Upload(ctx context.Context, server, path string, src fileedit.UploadSource, overwrite bool) error
|
||||
StartUpload(ctx context.Context, server, path string, src fileedit.UploadSource, overwrite bool) (fileedit.OpState, error)
|
||||
StartUnzip(ctx context.Context, server, path string, overwrite bool) (fileedit.OpState, error)
|
||||
Ops(ctx context.Context, server string) ([]fileedit.OpState, error)
|
||||
}
|
||||
|
||||
// writeFileRequest is the PUT /servers/{name}/file body. Content is []byte, so
|
||||
@@ -79,14 +72,8 @@ type FileEditor interface {
|
||||
// nothing is at the path yet (409 file_exists otherwise), so it can never
|
||||
// truncate a file the caller did not know was there. The two cannot be combined —
|
||||
// one says the file exists, the other that it must not.
|
||||
//
|
||||
// ContentSHA256 is required: the SHA-256 (hex) of the decoded content, which
|
||||
// the caller computes over the bytes it means to write. Content that hashes
|
||||
// otherwise changed on the way and is refused (400 digest_mismatch) before a Job
|
||||
// starts; the Job checks the bytes it received the same way before it writes.
|
||||
type writeFileRequest struct {
|
||||
Content *[]byte `json:"content"`
|
||||
ContentSHA256 string `json:"content_sha256"`
|
||||
ExpectSHA256 string `json:"expect_sha256,omitempty"`
|
||||
CreateOnly bool `json:"create_only,omitempty"`
|
||||
}
|
||||
@@ -109,23 +96,16 @@ func (a *API) handleListFiles(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
path := r.URL.Query().Get("path")
|
||||
|
||||
ls, err := a.Files.List(r.Context(), name, path)
|
||||
entries, truncated, err := a.Files.List(r.Context(), name, path)
|
||||
if err != nil {
|
||||
writeFileEditError(w, r, err)
|
||||
return
|
||||
}
|
||||
if ls.Entries == nil {
|
||||
ls.Entries = []fileedit.Entry{} // an empty directory is [], never null
|
||||
}
|
||||
// free_bytes lets the panel refuse an upload the volume cannot take before
|
||||
// sending any of it; the Job that lands it checks again. It is null when the
|
||||
// Job could not read it, so a full volume (0) is never mistaken for that.
|
||||
var free any = ls.Free
|
||||
if ls.Free < 0 {
|
||||
free = nil
|
||||
if entries == nil {
|
||||
entries = []fileedit.Entry{} // an empty directory is [], never null
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"path": path, "entries": ls.Entries, "truncated": ls.Truncated, "free_bytes": free,
|
||||
"path": path, "entries": entries, "truncated": truncated,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -152,12 +132,7 @@ func (a *API) handleReadFile(w http.ResponseWriter, r *http.Request) {
|
||||
if content == nil {
|
||||
content = []byte{} // an empty file is "", never null
|
||||
}
|
||||
// content_sha256 is of the bytes as sent (sha256 is of the file on disk,
|
||||
// before any redaction), so the panel can tell a damaged read from the file.
|
||||
got := sha256.Sum256(content)
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"path": path, "content": content, "sha256": sum, "content_sha256": hex.EncodeToString(got[:]),
|
||||
})
|
||||
writeJSON(w, http.StatusOK, map[string]any{"path": path, "content": content, "sha256": sum})
|
||||
}
|
||||
|
||||
// handleWriteFile serves PUT /api/v1/servers/{name}/file?path=… — replace a file's
|
||||
@@ -213,26 +188,10 @@ func (a *API) handleWriteFile(w http.ResponseWriter, r *http.Request) {
|
||||
"create_only and expect_sha256 cannot be combined"))
|
||||
return
|
||||
}
|
||||
switch sum := sha256.Sum256(*body.Content); {
|
||||
case body.ContentSHA256 == "":
|
||||
writeError(w, r, newError(http.StatusBadRequest, "digest_required",
|
||||
"send the SHA-256 of the content as content_sha256"))
|
||||
return
|
||||
case !sha256Hex.MatchString(body.ContentSHA256):
|
||||
writeError(w, r, newError(http.StatusBadRequest, "bad_digest",
|
||||
"content_sha256 must be the 64-digit lowercase hex SHA-256 of the content"))
|
||||
return
|
||||
case hex.EncodeToString(sum[:]) != body.ContentSHA256:
|
||||
writeError(w, r, newError(http.StatusBadRequest, "digest_mismatch",
|
||||
"the content that arrived does not hash to content_sha256, so it was changed on the way; send it again"))
|
||||
return
|
||||
}
|
||||
|
||||
// A write holds the world volume for its Job's lifetime (internal/maintenance);
|
||||
// reads and listings do not. A read-only mount cannot hurt a server starting
|
||||
// beside it, and a read that overlaps a restore or another change can at worst
|
||||
// show a file mid-change: the sha256 it returned then no longer matches, so a
|
||||
// save built on it is refused with file_changed.
|
||||
// reads and listings do not, since a read-only mount cannot hurt a server
|
||||
// starting beside it.
|
||||
release, ok := a.acquireWorld(w, r, name, maintenance.KindFileWrite, "stop the server before editing its files")
|
||||
if !ok {
|
||||
return
|
||||
@@ -370,10 +329,7 @@ func (a *API) handleRenameFile(w http.ResponseWriter, r *http.Request) {
|
||||
//
|
||||
// Content-Length is required (411 length_required): the stage reserves room for
|
||||
// the declared size before a byte is written, and a size promised up front is
|
||||
// what lets a short body be told from a whole one. So is Content-Digest, the
|
||||
// SHA-256 the client computed over the body (contentDigest): bytes that arrive
|
||||
// hashing to anything else are refused with 400 digest_mismatch, and the client
|
||||
// sends them again.
|
||||
// what lets a short body be told from a whole one.
|
||||
func (a *API) handleUploadFile(w http.ResponseWriter, r *http.Request) {
|
||||
name, ok := a.authorizeFileOp(w, r)
|
||||
if !ok {
|
||||
@@ -383,10 +339,6 @@ func (a *API) handleUploadFile(w http.ResponseWriter, r *http.Request) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !namesFit(path) {
|
||||
writeError(w, r, errNameTooLong())
|
||||
return
|
||||
}
|
||||
if a.FileStage == nil || a.InternalBaseURL == "" {
|
||||
writeError(w, r, newError(http.StatusServiceUnavailable, "files_unavailable",
|
||||
"uploads are not configured"))
|
||||
@@ -399,21 +351,13 @@ func (a *API) handleUploadFile(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
if r.ContentLength > fileedit.MaxUploadBytes {
|
||||
writeError(w, r, newError(http.StatusRequestEntityTooLarge, "too_large",
|
||||
"the file is %d bytes; one request carries at most %d, so send it as an upload session (POST files/uploads)",
|
||||
r.ContentLength, fileedit.MaxUploadBytes))
|
||||
return
|
||||
}
|
||||
want, ok := contentDigest(w, r)
|
||||
if !ok {
|
||||
"the file is %d bytes; uploads are at most %d", r.ContentLength, fileedit.MaxUploadBytes))
|
||||
return
|
||||
}
|
||||
overwrite := r.URL.Query().Get("overwrite") == "true"
|
||||
|
||||
staged, drop, err := a.FileStage.Put(r.Body, r.ContentLength, want)
|
||||
staged, drop, err := a.FileStage.Put(r.Body, r.ContentLength)
|
||||
switch {
|
||||
case errors.Is(err, fileedit.ErrDigestMismatch):
|
||||
writeError(w, r, errDigestMismatch())
|
||||
return
|
||||
case errors.Is(err, fileedit.ErrStageFull):
|
||||
writeError(w, r, newError(http.StatusInsufficientStorage, "upload_staging_full",
|
||||
"felis has no room to take this upload right now; try again later or ask an admin"))
|
||||
@@ -453,8 +397,7 @@ func (a *API) handleUploadFile(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// handleInternalFileUpload serves GET /api/v1/internal/file-uploads/{id} — the
|
||||
// staged bytes of one upload, to the one Job created to land them. A file sent
|
||||
// in parts (handlers_fileops.go) is served the same way. It is Public on
|
||||
// staged bytes of one upload, to the one Job created to land them. It is Public on
|
||||
// the internal face: the Job holds no service token (it holds no credential at
|
||||
// all), so the bearer token minted with the upload is the whole check, and it
|
||||
// opens that upload once. An unknown id, a wrong token and a spent one are the
|
||||
@@ -481,82 +424,6 @@ func (a *API) handleInternalFileUpload(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(w, f)
|
||||
}
|
||||
|
||||
// handleInternalFileUploadLanded serves DELETE
|
||||
// /api/v1/internal/file-uploads/{id} — the Job that fetched a file sent in
|
||||
// parts reports it landed, with the token it fetched it by, and the staged copy
|
||||
// is deleted. Until then the copy stays: a Job that failed after its fetch (the
|
||||
// digest did not match, the volume filled, a file was in the way) is started
|
||||
// again by committing again, without the file being sent again. Public on the
|
||||
// internal face like the fetch, with the same token as the check; a token that
|
||||
// does not open the upload, or an unknown id, is 404.
|
||||
func (a *API) handleInternalFileUploadLanded(w http.ResponseWriter, r *http.Request) {
|
||||
token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
|
||||
if a.FileStage == nil || !ok || a.FileStage.Landed(r.PathValue("id"), token) != nil {
|
||||
writeError(w, r, newError(http.StatusNotFound, "not_found", "no such upload"))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// contentDigest reads the SHA-256 a client computed over the body it sends, from
|
||||
// its Content-Digest header (parseContentDigest). An upload without one is
|
||||
// refused (400 digest_required): felis-api could not otherwise tell bytes
|
||||
// changed on the way from the bytes that were sent. It writes the refusal itself
|
||||
// and reports false.
|
||||
func contentDigest(w http.ResponseWriter, r *http.Request) ([]byte, bool) {
|
||||
sum, err := parseContentDigest(r.Header.Values("Content-Digest"))
|
||||
switch {
|
||||
case errors.Is(err, errNoContentDigest):
|
||||
writeError(w, r, newError(http.StatusBadRequest, "digest_required",
|
||||
"send the SHA-256 of the body as Content-Digest: sha-256=:<base64>:"))
|
||||
return nil, false
|
||||
case err != nil:
|
||||
writeError(w, r, newError(http.StatusBadRequest, "bad_digest",
|
||||
"Content-Digest must carry sha-256=:<base64 of the 32-byte SHA-256>:"))
|
||||
return nil, false
|
||||
}
|
||||
return sum, true
|
||||
}
|
||||
|
||||
var (
|
||||
errNoContentDigest = errors.New("no sha-256 Content-Digest")
|
||||
errBadContentDigest = errors.New("a malformed sha-256 Content-Digest")
|
||||
)
|
||||
|
||||
// parseContentDigest finds the SHA-256 in the values of a Content-Digest field
|
||||
// (RFC 9530): sha-256=:<base64>:, among any other algorithms it lists, which
|
||||
// are ignored. errNoContentDigest when there is none, errBadContentDigest when
|
||||
// it is not 32 bytes of base64 between colons.
|
||||
func parseContentDigest(values []string) ([]byte, error) {
|
||||
var found []byte
|
||||
for _, v := range values {
|
||||
for _, member := range strings.Split(v, ",") {
|
||||
key, value, _ := strings.Cut(member, "=")
|
||||
if !strings.EqualFold(strings.TrimSpace(key), "sha-256") {
|
||||
continue
|
||||
}
|
||||
value, _, _ = strings.Cut(value, ";") // parameters
|
||||
value = strings.TrimSpace(value)
|
||||
inner, pre := strings.CutPrefix(value, ":")
|
||||
inner, post := strings.CutSuffix(inner, ":")
|
||||
sum, err := base64.StdEncoding.DecodeString(inner)
|
||||
if !pre || !post || err != nil || len(sum) != sha256.Size {
|
||||
return nil, errBadContentDigest
|
||||
}
|
||||
found = sum
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
return nil, errNoContentDigest
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
func errDigestMismatch() error {
|
||||
return newError(http.StatusBadRequest, "digest_mismatch",
|
||||
"the bytes that arrived do not match their Content-Digest, so they were changed on the way; send them again")
|
||||
}
|
||||
|
||||
// auditFile records a file change. The target is "<server>:<path>", as file.write
|
||||
// has always recorded it; extra, when set, is the payload.
|
||||
func (a *API) auditFile(r *http.Request, action, server, path string, extra map[string]any) {
|
||||
@@ -596,8 +463,20 @@ var sha256Hex = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
// It returns the validated server name and false if it has already written a
|
||||
// response.
|
||||
func (a *API) authorizeFileOp(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
name, ok := a.authorizeServerFiles(w, r) // ① ② ③
|
||||
if !ok {
|
||||
name := r.PathValue("name")
|
||||
if err := naming.ValidateServerName(name); err != nil {
|
||||
writeError(w, r, newError(http.StatusBadRequest, "bad_name", "invalid server name: %v", err))
|
||||
return "", false
|
||||
}
|
||||
|
||||
p := principalFromContext(r.Context())
|
||||
rec, err := a.Repo.ServerByName(r.Context(), name)
|
||||
if err != nil {
|
||||
a.writeLookupError(w, r, err)
|
||||
return "", false
|
||||
}
|
||||
if !a.isOwnerOrAdmin(p, rec) {
|
||||
writeError(w, r, errForbidden)
|
||||
return "", false
|
||||
}
|
||||
|
||||
@@ -608,7 +487,7 @@ func (a *API) authorizeFileOp(w http.ResponseWriter, r *http.Request) (string, b
|
||||
}
|
||||
if info.Ready || info.DesiredState != string(v1alpha1.DesiredStopped) {
|
||||
writeError(w, r, newError(http.StatusConflict, "not_stopped",
|
||||
"stop the server before working with its files"))
|
||||
"stop the server before editing its files"))
|
||||
return "", false
|
||||
}
|
||||
|
||||
@@ -659,12 +538,6 @@ func writeFileEditError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
writeError(w, r, newError(http.StatusInsufficientStorage, "volume_full", "%s", err.Error()))
|
||||
case errors.Is(err, fileedit.ErrExists):
|
||||
writeError(w, r, newError(http.StatusConflict, "file_exists", "%s", err.Error()))
|
||||
case errors.Is(err, fileedit.ErrDigestMismatch):
|
||||
// A write's content reached its Job changed; nothing was written.
|
||||
writeError(w, r, newError(http.StatusBadRequest, "digest_mismatch", "%s", err.Error()))
|
||||
case errors.Is(err, fileedit.ErrReadDamaged):
|
||||
writeError(w, r, newError(http.StatusBadGateway, "read_damaged",
|
||||
"the file's bytes changed on their way from the file Job; read it again"))
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
writeError(w, r, newError(http.StatusGatewayTimeout, "files_timeout",
|
||||
"the file operation did not finish in time; retry shortly"))
|
||||
|
||||
@@ -3,7 +3,6 @@ package api
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -14,7 +13,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
||||
"felis.lolicon.best/internal/fileedit"
|
||||
@@ -44,19 +42,14 @@ type fakeFileEditor struct {
|
||||
|
||||
entries []fileedit.Entry
|
||||
truncated bool
|
||||
free int64
|
||||
content []byte
|
||||
sum string
|
||||
|
||||
// op is what StartUpload and StartUnzip answer (started), ops what Ops does.
|
||||
op fileedit.OpState
|
||||
ops []fileedit.OpState
|
||||
}
|
||||
|
||||
func (f *fakeFileEditor) List(_ context.Context, server, path string) (fileedit.Listing, error) {
|
||||
func (f *fakeFileEditor) List(_ context.Context, server, path string) ([]fileedit.Entry, bool, error) {
|
||||
f.calls++
|
||||
f.gotServer, f.gotPath = server, path
|
||||
return fileedit.Listing{Entries: f.entries, Truncated: f.truncated, Free: f.free}, f.err
|
||||
return f.entries, f.truncated, f.err
|
||||
}
|
||||
|
||||
func (f *fakeFileEditor) Read(_ context.Context, server, path string) ([]byte, string, error) {
|
||||
@@ -99,39 +92,12 @@ func (f *fakeFileEditor) Upload(_ context.Context, server, path string, src file
|
||||
return f.err
|
||||
}
|
||||
|
||||
func (f *fakeFileEditor) StartUpload(_ context.Context, server, path string, src fileedit.UploadSource, overwrite bool) (fileedit.OpState, error) {
|
||||
f.calls++
|
||||
f.gotOp, f.gotServer, f.gotPath, f.gotSource, f.gotOverwrite = fileedit.OpUpload, server, path, src, overwrite
|
||||
return f.started(fileedit.OpUpload, path), f.err
|
||||
}
|
||||
|
||||
func (f *fakeFileEditor) StartUnzip(_ context.Context, server, path string, overwrite bool) (fileedit.OpState, error) {
|
||||
f.calls++
|
||||
f.gotOp, f.gotServer, f.gotPath, f.gotOverwrite = fileedit.OpUnzip, server, path, overwrite
|
||||
return f.started(fileedit.OpUnzip, path), f.err
|
||||
}
|
||||
|
||||
// started is the op StartUpload and StartUnzip answer: f.op when a test set
|
||||
// one, else a running op as the real Editor answers it.
|
||||
func (f *fakeFileEditor) started(op, path string) fileedit.OpState {
|
||||
if f.op.ID != "" {
|
||||
return f.op
|
||||
}
|
||||
return fileedit.OpState{ID: "op" + strconv.Itoa(f.calls), Op: op, Path: path, State: fileedit.OpRunning, Started: time.Now()}
|
||||
}
|
||||
|
||||
func (f *fakeFileEditor) Ops(_ context.Context, server string) ([]fileedit.OpState, error) {
|
||||
f.calls++
|
||||
f.gotServer = server
|
||||
return f.ops, f.err
|
||||
}
|
||||
|
||||
// fileRouteHeader is the headers a file route's body goes with: raw bytes and
|
||||
// their Content-Digest for an upload, JSON for any other body.
|
||||
// fileRouteHeader is the Content-Type a file route's body goes with: raw bytes
|
||||
// for an upload, JSON for any other body.
|
||||
func fileRouteHeader(name, body string) map[string]string {
|
||||
switch {
|
||||
case name == "upload":
|
||||
return map[string]string{"Content-Type": "application/octet-stream", "Content-Digest": contentDigestOf(body)}
|
||||
return ctHeader("application/octet-stream")
|
||||
case body != "":
|
||||
return jsonHeader
|
||||
}
|
||||
@@ -180,7 +146,7 @@ func TestFileEditorStoppedGate(t *testing.T) {
|
||||
}{
|
||||
{"list", "GET", "/api/v1/servers/survival/files?path=config", ""},
|
||||
{"read", "GET", "/api/v1/servers/survival/file?path=server.properties", ""},
|
||||
{"write", "PUT", "/api/v1/servers/survival/file?path=server.properties", `{"content":"aGk=","content_sha256":"` + hiSum + `"}`},
|
||||
{"write", "PUT", "/api/v1/servers/survival/file?path=server.properties", `{"content":"aGk="}`},
|
||||
{"mkdir", "POST", "/api/v1/servers/survival/files/mkdir?path=plugins", ""},
|
||||
{"delete", "DELETE", "/api/v1/servers/survival/file?path=old.jar", ""},
|
||||
{"rename", "POST", "/api/v1/servers/survival/files/rename?path=a.txt", `{"to":"b.txt"}`},
|
||||
@@ -237,7 +203,7 @@ func TestFileEditorWorldVolumeGate(t *testing.T) {
|
||||
}{
|
||||
{"list", "GET", "/api/v1/servers/survival/files?path=config", ""},
|
||||
{"read", "GET", "/api/v1/servers/survival/file?path=server.properties", ""},
|
||||
{"write", "PUT", "/api/v1/servers/survival/file?path=server.properties", `{"content":"aGk=","content_sha256":"` + hiSum + `"}`},
|
||||
{"write", "PUT", "/api/v1/servers/survival/file?path=server.properties", `{"content":"aGk="}`},
|
||||
{"mkdir", "POST", "/api/v1/servers/survival/files/mkdir?path=plugins", ""},
|
||||
{"delete", "DELETE", "/api/v1/servers/survival/file?path=old.jar", ""},
|
||||
{"rename", "POST", "/api/v1/servers/survival/files/rename?path=a.txt", `{"to":"b.txt"}`},
|
||||
@@ -278,7 +244,7 @@ func TestFileEditorAuthorization(t *testing.T) {
|
||||
}{
|
||||
{"list", "GET", "/api/v1/servers/survival/files", ""},
|
||||
{"read", "GET", "/api/v1/servers/survival/file?path=server.properties", ""},
|
||||
{"write", "PUT", "/api/v1/servers/survival/file?path=server.properties", `{"content":"aGk=","content_sha256":"` + hiSum + `"}`},
|
||||
{"write", "PUT", "/api/v1/servers/survival/file?path=server.properties", `{"content":"aGk="}`},
|
||||
{"mkdir", "POST", "/api/v1/servers/survival/files/mkdir?path=plugins", ""},
|
||||
{"delete", "DELETE", "/api/v1/servers/survival/file?path=old.jar", ""},
|
||||
{"rename", "POST", "/api/v1/servers/survival/files/rename?path=a.txt", `{"to":"b.txt"}`},
|
||||
@@ -374,7 +340,6 @@ func TestFileEditorHandlers(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
files.entries = []fileedit.Entry{{Name: "paper.yml", Size: 12}, {Name: "sub", IsDir: true}}
|
||||
files.truncated = true
|
||||
files.free = 5 << 30
|
||||
api.External = staticExternal{p: owner}
|
||||
|
||||
w := do(api.ExternalHandler(), "GET", "/api/v1/servers/survival/files?path=config", "", nil)
|
||||
@@ -385,7 +350,6 @@ func TestFileEditorHandlers(t *testing.T) {
|
||||
Path string `json:"path"`
|
||||
Entries []fileedit.Entry `json:"entries"`
|
||||
Truncated bool `json:"truncated"`
|
||||
FreeBytes int64 `json:"free_bytes"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("body not JSON: %v (%s)", err, w.Body.String())
|
||||
@@ -393,26 +357,11 @@ func TestFileEditorHandlers(t *testing.T) {
|
||||
if files.gotPath != "config" {
|
||||
t.Fatalf("executor saw path %q, want the query value verbatim", files.gotPath)
|
||||
}
|
||||
if resp.Path != "config" || len(resp.Entries) != 2 || !resp.Truncated || resp.FreeBytes != 5<<30 {
|
||||
if resp.Path != "config" || len(resp.Entries) != 2 || !resp.Truncated {
|
||||
t.Fatalf("unexpected response %+v", resp)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a full volume lists 0 free and one the Job could not measure null", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
free int64
|
||||
want string
|
||||
}{{0, `"free_bytes":0`}, {-1, `"free_bytes":null`}} {
|
||||
api, _, _, files := mkFiles(t)
|
||||
files.free = c.free
|
||||
api.External = staticExternal{p: owner}
|
||||
w := do(api.ExternalHandler(), "GET", "/api/v1/servers/survival/files?path=config", "", nil)
|
||||
if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), c.want) {
|
||||
t.Fatalf("free %d: code = %d body %s, want %s", c.free, w.Code, w.Body.String(), c.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("list without a path lists the world root", func(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: owner}
|
||||
@@ -439,14 +388,11 @@ func TestFileEditorHandlers(t *testing.T) {
|
||||
Path string `json:"path"`
|
||||
Content []byte `json:"content"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Content256 string `json:"content_sha256"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("body not JSON: %v", err)
|
||||
}
|
||||
// content_sha256 is of the bytes sent, so the panel can check what arrived.
|
||||
if resp.Path != "server.properties" || string(resp.Content) != "motd=hello\n" || resp.SHA256 != testSum ||
|
||||
resp.Content256 != hexSum([]byte("motd=hello\n")) {
|
||||
if resp.Path != "server.properties" || string(resp.Content) != "motd=hello\n" || resp.SHA256 != testSum {
|
||||
t.Fatalf("unexpected response %+v (%q)", resp, resp.Content)
|
||||
}
|
||||
})
|
||||
@@ -468,7 +414,7 @@ func TestFileEditorHandlers(t *testing.T) {
|
||||
api.External = staticExternal{p: owner}
|
||||
|
||||
w := do(api.ExternalHandler(), "PUT", "/api/v1/servers/survival/file?path=server.properties",
|
||||
`{"content":"bW90ZD1jaGFuZ2VkCg==","content_sha256":"`+hexSum([]byte("motd=changed\n"))+`"}`, jsonHeader)
|
||||
`{"content":"bW90ZD1jaGFuZ2VkCg=="}`, jsonHeader)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -486,7 +432,7 @@ func TestFileEditorHandlers(t *testing.T) {
|
||||
files.sum = strings.Repeat("b", 64)
|
||||
api.External = staticExternal{p: owner}
|
||||
w := do(api.ExternalHandler(), "PUT", "/api/v1/servers/survival/file?path=server.properties",
|
||||
`{"content":"aGk=","content_sha256":"`+hiSum+`","expect_sha256":"`+testSum+`"}`, jsonHeader)
|
||||
`{"content":"aGk=","expect_sha256":"`+testSum+`"}`, jsonHeader)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -519,7 +465,7 @@ func TestFileEditorHandlers(t *testing.T) {
|
||||
files.err = fmt.Errorf("%w: server.properties has changed", fileedit.ErrConflict)
|
||||
api.External = staticExternal{p: owner}
|
||||
w := do(api.ExternalHandler(), "PUT", "/api/v1/servers/survival/file?path=server.properties",
|
||||
`{"content":"aGk=","content_sha256":"`+hiSum+`","expect_sha256":"`+testSum+`"}`, jsonHeader)
|
||||
`{"content":"aGk=","expect_sha256":"`+testSum+`"}`, jsonHeader)
|
||||
if w.Code != http.StatusConflict || decodeErr(t, w) != "file_changed" {
|
||||
t.Fatalf("code = %d body %s, want 409 file_changed", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -580,7 +526,7 @@ func TestFileEditorHandlers(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: owner}
|
||||
w := do(api.ExternalHandler(), "PUT", "/api/v1/servers/survival/file?path=server.properties",
|
||||
`{"content":"","content_sha256":"`+hexSum(nil)+`"}`, jsonHeader)
|
||||
`{"content":""}`, jsonHeader)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("code = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -593,8 +539,7 @@ func TestFileEditorHandlers(t *testing.T) {
|
||||
t.Run("a write at exactly the limit is allowed", func(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: owner}
|
||||
content := make([]byte, fileedit.MaxWriteBytes)
|
||||
body, err := json.Marshal(writeFileRequest{Content: &content, ContentSHA256: hexSum(content)})
|
||||
body, err := json.Marshal(writeFileRequest{Content: bytesPtr(make([]byte, fileedit.MaxWriteBytes))})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
@@ -742,12 +687,12 @@ func TestFileManagerHandlers(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: owner}
|
||||
w := do(api.ExternalHandler(), "PUT", "/api/v1/servers/survival/file?path=plugins/new.yml",
|
||||
`{"content":"","content_sha256":"`+hexSum(nil)+`","create_only":true}`, jsonHeader)
|
||||
`{"content":"","create_only":true}`, jsonHeader)
|
||||
if w.Code != http.StatusOK || !files.gotCreateOnly || files.gotExpect != "" {
|
||||
t.Fatalf("code = %d, createOnly = %v, expect = %q (%s)", w.Code, files.gotCreateOnly, files.gotExpect, w.Body.String())
|
||||
}
|
||||
w = do(api.ExternalHandler(), "PUT", "/api/v1/servers/survival/file?path=server.properties",
|
||||
`{"content":"aGk=","content_sha256":"`+hiSum+`"}`, jsonHeader)
|
||||
`{"content":"aGk="}`, jsonHeader)
|
||||
if w.Code != http.StatusOK || files.gotCreateOnly {
|
||||
t.Fatalf("a plain save: code = %d, createOnly = %v", w.Code, files.gotCreateOnly)
|
||||
}
|
||||
@@ -764,71 +709,11 @@ func TestFileManagerHandlers(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// contentDigestOf is the Content-Digest a client sends with body.
|
||||
func contentDigestOf(body string) string {
|
||||
sum := sha256.Sum256([]byte(body))
|
||||
return "sha-256=:" + base64.StdEncoding.EncodeToString(sum[:]) + ":"
|
||||
}
|
||||
|
||||
// hexSum is the content_sha256 a client sends with a write of b; hiSum is that
|
||||
// of "hi" (aGk=).
|
||||
func hexSum(b []byte) string {
|
||||
sum := sha256.Sum256(b)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
var hiSum = hexSum([]byte("hi"))
|
||||
|
||||
// A write carries the SHA-256 of its content: one without it, with a malformed
|
||||
// one, or whose content hashes otherwise is refused before a Job starts, and a
|
||||
// Job that found the bytes it received changed answers the same way. None of
|
||||
// them is audited.
|
||||
func TestWriteFileChecksTheContentDigest(t *testing.T) {
|
||||
owner := &Principal{UserID: "owner1", Email: "[email protected]", Role: "user"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body string
|
||||
want string
|
||||
}{
|
||||
{"without a digest", `{"content":"aGk="}`, "digest_required"},
|
||||
{"a malformed digest", `{"content":"aGk=","content_sha256":"` + strings.ToUpper(hiSum) + `"}`, "bad_digest"},
|
||||
{"changed on the way", `{"content":"aGo=","content_sha256":"` + hiSum + `"}`, "digest_mismatch"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
api, repo, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: owner}
|
||||
w := do(api.ExternalHandler(), "PUT", "/api/v1/servers/survival/file?path=server.properties", tc.body, jsonHeader)
|
||||
if w.Code != http.StatusBadRequest || decodeErr(t, w) != tc.want {
|
||||
t.Fatalf("code = %d body %s, want 400 %s", w.Code, w.Body.String(), tc.want)
|
||||
}
|
||||
if files.calls != 0 || len(repo.audits) != 0 {
|
||||
t.Fatalf("calls = %d audits = %+v, want no Job and no audit", files.calls, repo.audits)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("changed on the way to the Job", func(t *testing.T) {
|
||||
api, repo, _, files := mkFiles(t)
|
||||
files.err = fmt.Errorf("%w: the content hashes to 00 and was sent as 01", fileedit.ErrDigestMismatch)
|
||||
api.External = staticExternal{p: owner}
|
||||
w := do(api.ExternalHandler(), "PUT", "/api/v1/servers/survival/file?path=server.properties",
|
||||
`{"content":"aGk=","content_sha256":"`+hiSum+`"}`, jsonHeader)
|
||||
if w.Code != http.StatusBadRequest || decodeErr(t, w) != "digest_mismatch" {
|
||||
t.Fatalf("code = %d body %s, want 400 digest_mismatch", w.Code, w.Body.String())
|
||||
}
|
||||
if files.calls != 1 || len(repo.audits) != 0 {
|
||||
t.Fatalf("calls = %d audits = %+v, want the one Job and no audit", files.calls, repo.audits)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// doUpload sends an upload whose Content-Length is declared, not measured, the
|
||||
// way a client that streams or lies would send it. Its Content-Digest is that
|
||||
// of the body.
|
||||
// way a client that streams or lies would send it.
|
||||
func doUpload(h http.Handler, body string, length int64) *httptest.ResponseRecorder {
|
||||
r := httptest.NewRequest("PUT", "/api/v1/servers/survival/files/upload?path=plugins/x.jar", strings.NewReader(body))
|
||||
r.Header.Set("Content-Type", "application/octet-stream")
|
||||
r.Header.Set("Content-Digest", contentDigestOf(body))
|
||||
r.ContentLength = length
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, r)
|
||||
@@ -857,7 +742,7 @@ func TestFileUpload(t *testing.T) {
|
||||
digest := sha256.Sum256([]byte(body))
|
||||
sum := hex.EncodeToString(digest[:])
|
||||
const route = "/api/v1/servers/survival/files/upload?path=plugins/x.jar"
|
||||
octet := map[string]string{"Content-Type": "application/octet-stream", "Content-Digest": contentDigestOf(body)}
|
||||
octet := ctHeader("application/octet-stream")
|
||||
|
||||
t.Run("the Job fetches the body once, with its token alone", func(t *testing.T) {
|
||||
api, repo, _, files := mkFiles(t)
|
||||
@@ -865,7 +750,7 @@ func TestFileUpload(t *testing.T) {
|
||||
// Every other internal route wants a service token; the Job has none.
|
||||
api.Internal = CallerTokens{CallerVelocity: "s3cr3t"}
|
||||
prefix := api.InternalBaseURL + "/api/v1/internal/file-uploads/"
|
||||
var bare, wrong, unschemed, fetched, again, landedWrong, landed *httptest.ResponseRecorder
|
||||
var bare, wrong, unschemed, fetched, again *httptest.ResponseRecorder
|
||||
files.onUpload = func(src fileedit.UploadSource) {
|
||||
id, ok := strings.CutPrefix(src.URL, prefix)
|
||||
if !ok || len(id) != 32 {
|
||||
@@ -879,10 +764,6 @@ func TestFileUpload(t *testing.T) {
|
||||
unschemed = do(h, "GET", at, "", map[string]string{"Authorization": src.Token})
|
||||
fetched = do(h, "GET", at, "", map[string]string{"Authorization": "Bearer " + src.Token})
|
||||
again = do(h, "GET", at, "", map[string]string{"Authorization": "Bearer " + src.Token})
|
||||
// The Job reports it landed. A single upload goes when its request
|
||||
// ends, so the report takes nothing away early.
|
||||
landedWrong = do(h, "DELETE", at, "", map[string]string{"Authorization": "Bearer " + strings.Repeat("0", len(src.Token))})
|
||||
landed = do(h, "DELETE", at, "", map[string]string{"Authorization": "Bearer " + src.Token})
|
||||
}
|
||||
|
||||
w := do(api.ExternalHandler(), "PUT", route, body, octet)
|
||||
@@ -894,7 +775,6 @@ func TestFileUpload(t *testing.T) {
|
||||
}
|
||||
for name, r := range map[string]*httptest.ResponseRecorder{
|
||||
"no token": bare, "wrong token": wrong, "the token without Bearer": unschemed, "second fetch": again,
|
||||
"a landed report with the wrong token": landedWrong,
|
||||
} {
|
||||
if r.Code != http.StatusNotFound || decodeErr(t, r) != "not_found" {
|
||||
t.Errorf("%s: code = %d (%s), want 404 not_found", name, r.Code, r.Body.String())
|
||||
@@ -904,9 +784,6 @@ func TestFileUpload(t *testing.T) {
|
||||
fetched.Header().Get("Content-Length") != strconv.Itoa(len(body)) {
|
||||
t.Fatalf("fetch: code = %d, %q, Content-Length %q", fetched.Code, fetched.Body.String(), fetched.Header().Get("Content-Length"))
|
||||
}
|
||||
if landed.Code != http.StatusNoContent {
|
||||
t.Fatalf("landed: code = %d (%s)", landed.Code, landed.Body.String())
|
||||
}
|
||||
if files.gotPath != "plugins/x.jar" || files.gotSource.Size != int64(len(body)) ||
|
||||
files.gotSource.SHA256 != sum || files.gotOverwrite {
|
||||
t.Fatalf("executor saw path %q, source %+v, overwrite %v", files.gotPath, files.gotSource, files.gotOverwrite)
|
||||
@@ -956,51 +833,6 @@ func TestFileUpload(t *testing.T) {
|
||||
stageEmpty(t, api)
|
||||
})
|
||||
|
||||
t.Run("the body comes with its SHA-256, checked before anything is asked of the world", func(t *testing.T) {
|
||||
other := sha256.Sum256([]byte("PK\x03\x04 another jar"))
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
digest []string
|
||||
code int
|
||||
errCode string
|
||||
}{
|
||||
{"none", nil, http.StatusBadRequest, "digest_required"},
|
||||
{"only another algorithm", []string{"sha-512=:" + base64.StdEncoding.EncodeToString(make([]byte, 64)) + ":"}, http.StatusBadRequest, "digest_required"},
|
||||
{"hex in place of base64", []string{"sha-256=:" + sum + ":"}, http.StatusBadRequest, "bad_digest"},
|
||||
{"a bare base64 value", []string{strings.Trim(strings.TrimPrefix(contentDigestOf(body), "sha-256="), ":")}, http.StatusBadRequest, "digest_required"},
|
||||
{"no closing colon", []string{strings.TrimSuffix(contentDigestOf(body), ":")}, http.StatusBadRequest, "bad_digest"},
|
||||
{"base64 without colons", []string{"sha-256=" + strings.Trim(strings.TrimPrefix(contentDigestOf(body), "sha-256="), ":")}, http.StatusBadRequest, "bad_digest"},
|
||||
{"31 bytes", []string{"sha-256=:" + base64.StdEncoding.EncodeToString(digest[:31]) + ":"}, http.StatusBadRequest, "bad_digest"},
|
||||
{"another body's", []string{"sha-256=:" + base64.StdEncoding.EncodeToString(other[:]) + ":"}, http.StatusBadRequest, "digest_mismatch"},
|
||||
{"the right one, then a wrong one", []string{contentDigestOf(body), "sha-256=:" + base64.StdEncoding.EncodeToString(other[:]) + ":"}, http.StatusBadRequest, "digest_mismatch"},
|
||||
{"among others, upper case, with a parameter", []string{"sha-512=:" + base64.StdEncoding.EncodeToString(make([]byte, 64)) + ":, SHA-256=" + strings.TrimPrefix(contentDigestOf(body), "sha-256=") + ";p=1"}, http.StatusOK, ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
api, repo, cl, files := mkFiles(t)
|
||||
api.External = staticExternal{p: owner}
|
||||
r := httptest.NewRequest("PUT", route, strings.NewReader(body))
|
||||
r.Header.Set("Content-Type", "application/octet-stream")
|
||||
for _, v := range c.digest {
|
||||
r.Header.Add("Content-Digest", v)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
api.ExternalHandler().ServeHTTP(w, r)
|
||||
recordContract(r, body, w)
|
||||
if c.code == http.StatusOK {
|
||||
if w.Code != http.StatusOK || files.calls != 1 || files.gotSource.SHA256 != sum {
|
||||
t.Fatalf("code = %d calls = %d source %+v (%s)", w.Code, files.calls, files.gotSource, w.Body.String())
|
||||
}
|
||||
return
|
||||
}
|
||||
if w.Code != c.code || decodeErr(t, w) != c.errCode || files.calls != 0 || len(cl.acquired) != 0 || len(repo.audits) != 0 {
|
||||
t.Fatalf("code = %d calls = %d acquired %v audits %d (%s), want %d %s",
|
||||
w.Code, files.calls, cl.acquired, len(repo.audits), w.Body.String(), c.code, c.errCode)
|
||||
}
|
||||
stageEmpty(t, api)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no Content-Length -> 411", func(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: owner}
|
||||
@@ -1020,28 +852,6 @@ func TestFileUpload(t *testing.T) {
|
||||
stageEmpty(t, api)
|
||||
})
|
||||
|
||||
t.Run("a name longer than a folder entry holds -> 400 before a byte is staged", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
code int
|
||||
errCode string
|
||||
}{
|
||||
{strings.Repeat("n", fileedit.NameMax-3) + ".jar", http.StatusBadRequest, "bad_path"},
|
||||
{strings.Repeat("n", fileedit.NameMax-4) + ".jar", http.StatusOK, ""},
|
||||
} {
|
||||
api, _, _, files := mkFiles(t)
|
||||
api.External = staticExternal{p: owner}
|
||||
w := do(api.ExternalHandler(), "PUT", "/api/v1/servers/survival/files/upload?path=plugins/"+c.name, body, octet)
|
||||
if w.Code != c.code || (c.errCode != "" && decodeErr(t, w) != c.errCode) {
|
||||
t.Fatalf("%d-byte name: code = %d (%s), want %d", len(c.name), w.Code, w.Body.String(), c.code)
|
||||
}
|
||||
if wantCalls := map[bool]int{true: 1, false: 0}[c.code == http.StatusOK]; files.calls != wantCalls {
|
||||
t.Fatalf("%d-byte name: executor calls = %d, want %d", len(c.name), files.calls, wantCalls)
|
||||
}
|
||||
stageEmpty(t, api)
|
||||
}
|
||||
})
|
||||
|
||||
// Staged, and so short: the body is a few bytes of a declared 64 MiB.
|
||||
t.Run("a declared size at the cap is taken", func(t *testing.T) {
|
||||
api, _, _, files := mkFiles(t)
|
||||
@@ -1090,12 +900,10 @@ func TestFileUpload(t *testing.T) {
|
||||
t.Run("the internal route without a stage -> 404", func(t *testing.T) {
|
||||
api, _, _, _ := mkFiles(t)
|
||||
api.FileStage = nil
|
||||
for _, method := range []string{"GET", "DELETE"} {
|
||||
w := do(api.InternalHandler(), method, "/api/v1/internal/file-uploads/00112233445566778899aabbccddeeff", "",
|
||||
w := do(api.InternalHandler(), "GET", "/api/v1/internal/file-uploads/00112233445566778899aabbccddeeff", "",
|
||||
map[string]string{"Authorization": "Bearer t"})
|
||||
if w.Code != http.StatusNotFound || decodeErr(t, w) != "not_found" {
|
||||
t.Fatalf("%s: code = %d (%s)", method, w.Code, w.Body.String())
|
||||
}
|
||||
t.Fatalf("code = %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1119,7 +927,6 @@ func TestFileEditorErrorMapping(t *testing.T) {
|
||||
{"changed since read", fmt.Errorf("%w: nope", fileedit.ErrConflict), http.StatusConflict, "file_changed"},
|
||||
{"volume full", fmt.Errorf("%w: nope", fileedit.ErrNoSpace), http.StatusInsufficientStorage, "volume_full"},
|
||||
{"already there", fmt.Errorf("%w: nope", fileedit.ErrExists), http.StatusConflict, "file_exists"},
|
||||
{"changed on the way from the Job", fmt.Errorf("%w: nope", fileedit.ErrReadDamaged), http.StatusBadGateway, "read_damaged"},
|
||||
{"timeout", fmt.Errorf("waiting: %w", context.DeadlineExceeded), http.StatusGatewayTimeout, "files_timeout"},
|
||||
}
|
||||
|
||||
|
||||
@@ -107,7 +107,7 @@ func maintenanceOps(t *testing.T) (*API, *fakeCluster, []maintenanceOp) {
|
||||
{"backup", maintenance.KindBackup, "POST", "/api/v1/servers/survival/backup", "",
|
||||
func() int { return backuper.calls }},
|
||||
{"file write", maintenance.KindFileWrite, "PUT", "/api/v1/servers/survival/file?path=server.properties",
|
||||
`{"content":"aGk=","content_sha256":"` + hiSum + `"}`, func() int { return files.calls }},
|
||||
`{"content":"aGk="}`, func() int { return files.calls }},
|
||||
{"file mkdir", maintenance.KindFileWrite, "POST", "/api/v1/servers/survival/files/mkdir?path=plugins",
|
||||
"", func() int { return files.calls }},
|
||||
{"file delete", maintenance.KindFileWrite, "DELETE", "/api/v1/servers/survival/file?path=old.jar",
|
||||
|
||||
@@ -175,11 +175,6 @@ func (a *API) handlePatchUser(w http.ResponseWriter, r *http.Request) {
|
||||
"role must be 'admin' or 'user', got %q", *body.Role))
|
||||
return
|
||||
}
|
||||
// A new address unverifies the old one, and with it the email factor that
|
||||
// guards adding a passkey: your own takes the same reauth as /account/email.
|
||||
if body.Email != nil && id == p.UserID && !a.requireReauth(w, r, p) {
|
||||
return
|
||||
}
|
||||
|
||||
u, err := a.Repo.UpdateUser(r.Context(), id, UpdateUserInput(body), p.Email)
|
||||
if err != nil {
|
||||
@@ -436,11 +431,6 @@ func (a *API) handleUnbindUserPasskeys(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, errBadRequest)
|
||||
return
|
||||
}
|
||||
// Your own passkeys are a factor that guards adding one: severing them takes
|
||||
// the same reauth as removing one under /account/passkey.
|
||||
if p := principalFromContext(r.Context()); id == p.UserID && !a.requireReauth(w, r, p) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := a.Repo.DeleteAllPasskeyCredentialsForUser(r.Context(), id); err != nil {
|
||||
writeError(w, r, err)
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -85,77 +83,6 @@ func TestOwnerAccountProtectedFromPanelMutations(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// The caller's own email and passkeys are ways into the caller's account, so
|
||||
// changing them through /users/{id} takes the same recent reauth as through
|
||||
// /account. Without it a stolen owner session could strip both here, and with no
|
||||
// factor left to guard, /account/passkey/register/begin would let it plant its own.
|
||||
func TestOwnSignInFactorsNeedReauthUnderUsers(t *testing.T) {
|
||||
owner := &Principal{UserID: "usr-root", Role: "owner", Email: "[email protected]",
|
||||
ViaAdminAccess: true, ViaSession: true, EmailVerified: true}
|
||||
repo := newFakeRepo()
|
||||
repo.seedUser(UserView{ID: "usr-root", Username: "root", Email: "[email protected]", Role: "owner", EmailVerified: true})
|
||||
repo.seedUser(UserView{ID: "u2", Username: "alice", Email: "[email protected]", Role: "user"})
|
||||
repo.passkeyCreds["pk-root"] = PasskeyCredential{ID: "pk-root", UserID: "usr-root", CredentialID: "c-root", UserVerified: true, CreatedAt: frozenNow}
|
||||
api := newTestAPI(repo, newFakeCluster())
|
||||
api.Mailer = &captureMailer{}
|
||||
api.External = staticExternal{p: owner}
|
||||
eh := api.ExternalHandler()
|
||||
send := func(method, path, body string) *httptest.ResponseRecorder {
|
||||
if body == "" {
|
||||
return do(eh, method, path, "", nil)
|
||||
}
|
||||
return do(eh, method, path, body, jsonHeader)
|
||||
}
|
||||
|
||||
own := []struct {
|
||||
name, method, path, body string
|
||||
untouched func() bool
|
||||
}{
|
||||
{"own email", "PATCH", "/api/v1/users/usr-root", `{"email":"[email protected]"}`, func() bool {
|
||||
d, err := repo.UserDetail(context.Background(), "usr-root")
|
||||
return err == nil && d.Email == "[email protected]"
|
||||
}},
|
||||
{"own passkeys", "DELETE", "/api/v1/users/usr-root/passkeys", "", func() bool {
|
||||
_, ok := repo.passkeyCreds["pk-root"]
|
||||
return ok
|
||||
}},
|
||||
}
|
||||
for _, tc := range own {
|
||||
t.Run(tc.name+" refused without a recent reauth", func(t *testing.T) {
|
||||
w := send(tc.method, tc.path, tc.body)
|
||||
if w.Code != http.StatusForbidden || decodeErr(t, w) != "reauth_required" {
|
||||
t.Fatalf("code = %d body %s, want 403 reauth_required", w.Code, w.Body.String())
|
||||
}
|
||||
if !tc.untouched() {
|
||||
t.Fatal("the change went through despite the refusal")
|
||||
}
|
||||
})
|
||||
}
|
||||
// Another account's factors are the owner's to manage, and a username is no way in.
|
||||
for _, tc := range []struct{ name, method, path, body string }{
|
||||
{"another user's email", "PATCH", "/api/v1/users/u2", `{"email":"[email protected]"}`},
|
||||
{"another user's passkeys", "DELETE", "/api/v1/users/u2/passkeys", ""},
|
||||
{"own username", "PATCH", "/api/v1/users/usr-root", `{"username":"root2"}`},
|
||||
} {
|
||||
t.Run("control: "+tc.name+" needs no reauth", func(t *testing.T) {
|
||||
if w := send(tc.method, tc.path, tc.body); w.Code != http.StatusOK {
|
||||
t.Fatalf("code = %d body %s, want 200", w.Code, w.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
owner.ReauthAt = api.now()
|
||||
for _, tc := range own {
|
||||
t.Run(tc.name+" allowed after a reauth", func(t *testing.T) {
|
||||
if w := send(tc.method, tc.path, tc.body); w.Code != http.StatusOK {
|
||||
t.Fatalf("code = %d body %s, want 200", w.Code, w.Body.String())
|
||||
}
|
||||
if tc.untouched() {
|
||||
t.Fatal("the change did not go through")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The user-scoped admin sub-resources (quotas, account links) must answer 404
|
||||
// for an unknown user id. Before the requireLiveUser guard the quota upsert and
|
||||
// the link insert reached the users(id) foreign key and surfaced as an opaque
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
// object an operator with kubectl could read. The route is the API-side outlet.
|
||||
type AsyncJob struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"` // "backup" | "restore" | "export_world" | "export_backup" | "export_files"
|
||||
Kind string `json:"kind"` // "backup" | "restore" | "export_world" | "export_backup"
|
||||
State string `json:"state"` // "running" | "succeeded" | "failed"
|
||||
Message string `json:"message,omitempty"`
|
||||
StartedAt time.Time `json:"started_at,omitzero"`
|
||||
|
||||
@@ -179,9 +179,6 @@ func TestLatestJobsExplainsFailures(t *testing.T) {
|
||||
}}},
|
||||
}
|
||||
}
|
||||
// Killed for memory mid-walk: the last line it printed says nothing of that.
|
||||
oom := pod("c-1", "backup-survival-c", 4, 137, "archiving survival\n")
|
||||
oom.Status.ContainerStatuses[0].State.Terminated.Reason = "OOMKilled"
|
||||
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
|
||||
failedJob("backup-survival-a", 1),
|
||||
failedJob("backup-survival-b", 2),
|
||||
@@ -189,7 +186,6 @@ func TestLatestJobsExplainsFailures(t *testing.T) {
|
||||
pod("a-2", "backup-survival-a", 3, 1,
|
||||
"archiving survival\nfelis backup: backup: not enough free disk for the archive: the world is 2.0 GiB\n"),
|
||||
pod("b-1", "backup-survival-b", 2, 0, "done"),
|
||||
failedJob("backup-survival-c", 4), oom,
|
||||
).Build()
|
||||
|
||||
jobs, err := NewK8sJobStatus(c, "minecraft").LatestJobs(context.Background(), "survival")
|
||||
@@ -206,7 +202,4 @@ func TestLatestJobsExplainsFailures(t *testing.T) {
|
||||
if want := "Job has reached the specified backoff limit"; got["backup-survival-b"] != want {
|
||||
t.Errorf("b: message = %q, want the condition text", got["backup-survival-b"])
|
||||
}
|
||||
if want := "the job ran out of memory and the system stopped it (OOMKilled)"; got["backup-survival-c"] != want {
|
||||
t.Errorf("c: message = %q, want %q", got["backup-survival-c"], want)
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/fileedit"
|
||||
"felis.lolicon.best/internal/maintenance"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
@@ -108,23 +107,14 @@ func (k *K8sJobStatus) explainFailures(ctx context.Context, serverName string, j
|
||||
}
|
||||
}
|
||||
|
||||
// outOfMemoryLine is a failed Job's message when the kernel killed it for
|
||||
// memory. The reason in brackets is what the panel knows it by.
|
||||
const outOfMemoryLine = "the job ran out of memory and the system stopped it (OOMKilled)"
|
||||
|
||||
// lastTerminationLine returns the last non-empty line of the pod's terminated
|
||||
// container message, capped for display. A container the kernel killed for
|
||||
// going over its memory limit printed nothing about it, so that is said instead
|
||||
// of whatever line it happened to print last.
|
||||
// container message, capped for display.
|
||||
func lastTerminationLine(pod *corev1.Pod) string {
|
||||
for _, cs := range pod.Status.ContainerStatuses {
|
||||
t := cs.State.Terminated
|
||||
if t == nil || t.ExitCode == 0 {
|
||||
continue
|
||||
}
|
||||
if t.Reason == fileedit.ReasonOOMKilled {
|
||||
return outOfMemoryLine
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(t.Message), "\n")
|
||||
line := strings.TrimSpace(lines[len(lines)-1])
|
||||
if len(line) > 400 {
|
||||
@@ -240,15 +230,11 @@ func jobOutcome(j *batchv1.Job) (AsyncJob, bool) {
|
||||
case jobManagedByRestore:
|
||||
kind = "restore"
|
||||
case jobManagedByExport:
|
||||
// As maintenance.JobKind reads it: a Job that names no mode this build
|
||||
// knows reads as a world export, the kind that holds the world.
|
||||
switch j.Labels[maintenance.LabelExportMode] {
|
||||
case maintenance.ExportModeBackup:
|
||||
kind = "export_backup"
|
||||
case maintenance.ExportModeFiles:
|
||||
kind = "export_files"
|
||||
default:
|
||||
// As maintenance.JobKind reads it: only a Job that says it reads a
|
||||
// backup is not a world export.
|
||||
kind = "export_world"
|
||||
if j.Labels[maintenance.LabelExportMode] == maintenance.ExportModeBackup {
|
||||
kind = "export_backup"
|
||||
}
|
||||
default:
|
||||
return AsyncJob{}, false
|
||||
|
||||
@@ -38,7 +38,7 @@ func maintenanceLabel(kind string) string {
|
||||
case maintenance.KindFileWrite:
|
||||
return "a file write"
|
||||
case maintenance.KindExport:
|
||||
return "a world export or file download"
|
||||
return "a world export"
|
||||
case maintenance.KindReap:
|
||||
return "the idle-world reaper"
|
||||
}
|
||||
|
||||
@@ -42,9 +42,6 @@ func TestOpenAPISchemasMatchWireStructs(t *testing.T) {
|
||||
"BackupView": BackupView{},
|
||||
"ExportTicket": exportTicketView{},
|
||||
"ExportStatus": exportStatusView{},
|
||||
"FileUploadSession": fileSessionView{},
|
||||
"FileOp": fileOpView{},
|
||||
"FileOpError": fileOpError{},
|
||||
"Schedule": Schedule{},
|
||||
"Build": build.Build{},
|
||||
"Image": build.Image{},
|
||||
|
||||
@@ -166,10 +166,6 @@ func (a *API) handleCreateSubmission(w http.ResponseWriter, r *http.Request) {
|
||||
// caller does not own is reported as 404, so this endpoint cannot upload to — or
|
||||
// probe the existence of — another user's submission.
|
||||
//
|
||||
// The body carries its SHA-256 as Content-Digest (contentDigest); bytes that hash
|
||||
// to anything else were changed on the way, and none of them are kept
|
||||
// (submit.VerifyDigest).
|
||||
//
|
||||
// Uploading does not change the submission row (there is no "uploaded" column):
|
||||
// the blob store is the presence source of truth, which admin approval consults.
|
||||
func (a *API) handleUploadSubmissionContext(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -177,10 +173,6 @@ func (a *API) handleUploadSubmissionContext(w http.ResponseWriter, r *http.Reque
|
||||
writeError(w, r, errSubmissionsUnavailable)
|
||||
return
|
||||
}
|
||||
want, ok := contentDigest(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
p := principalFromContext(r.Context())
|
||||
// Reserve the per-user upload cooldown BEFORE streaming. The body is the
|
||||
// expensive part (up to the 1 GiB blob cap), so without a reservation the
|
||||
@@ -196,7 +188,7 @@ func (a *API) handleUploadSubmissionContext(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
defer release()
|
||||
id := r.PathValue("id")
|
||||
sub, err := a.Submissions.UploadContext(r.Context(), id, p.UserID, submit.VerifyDigest(r.Body, want))
|
||||
sub, err := a.Submissions.UploadContext(r.Context(), id, p.UserID, r.Body)
|
||||
if err != nil {
|
||||
writeSubmitError(w, r, err)
|
||||
return
|
||||
@@ -248,8 +240,7 @@ func (a *API) handleContextUploadStatus(w http.ResponseWriter, r *http.Request)
|
||||
// else must equal the staged length (409 upload_offset_mismatch otherwise). A
|
||||
// part is small enough for any edge, so no cooldown applies here: the staged
|
||||
// total is bounded by the context cap and the storage budget, and completion
|
||||
// holds the cooldown. Each part carries its own Content-Digest, and one that
|
||||
// does not hash to it is cut back off, as a part that breaks off is.
|
||||
// holds the cooldown.
|
||||
func (a *API) handleContextUploadPart(w http.ResponseWriter, r *http.Request) {
|
||||
if a.Submissions == nil {
|
||||
writeError(w, r, errSubmissionsUnavailable)
|
||||
@@ -261,12 +252,8 @@ func (a *API) handleContextUploadPart(w http.ResponseWriter, r *http.Request) {
|
||||
"offset must be the byte position the part starts at"))
|
||||
return
|
||||
}
|
||||
want, ok := contentDigest(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
p := principalFromContext(r.Context())
|
||||
prog, err := a.Submissions.UploadPart(r.Context(), r.PathValue("id"), p.UserID, offset, submit.VerifyDigest(r.Body, want))
|
||||
prog, err := a.Submissions.UploadPart(r.Context(), r.PathValue("id"), p.UserID, offset, r.Body)
|
||||
if err != nil {
|
||||
writeSubmitError(w, r, err)
|
||||
return
|
||||
@@ -570,8 +557,6 @@ func writeSubmitError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
case errors.Is(err, submit.ErrUploadBusy):
|
||||
writeError(w, r, newError(http.StatusConflict, "upload_busy",
|
||||
"another request is still writing this upload; read where it stands and continue from there"))
|
||||
case errors.Is(err, submit.ErrDigestMismatch):
|
||||
writeError(w, r, errDigestMismatch())
|
||||
case errors.As(err, &mismatch):
|
||||
writeError(w, r, newError(http.StatusConflict, "upload_offset_mismatch",
|
||||
"the upload holds %d bytes; send the part that starts there", mismatch.Received))
|
||||
|
||||
@@ -14,7 +14,7 @@ func TestContextUploadPartForwardsOffsetBodyAndPrincipal(t *testing.T) {
|
||||
fs := &fakeSubmissions{progress: submit.UploadProgress{Received: 8, PartMaxBytes: 33554432, MaxContextBytes: 1073741824}}
|
||||
api := appSubAPI(fs)
|
||||
w := do(api.ExternalHandler(), "PUT", "/api/v1/me/submissions/sub-9/context/upload?offset=4", "abcd",
|
||||
map[string]string{"Content-Type": "application/octet-stream", "Content-Digest": contentDigestOf("abcd")})
|
||||
ctHeader("application/octet-stream"))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("code = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -72,8 +72,7 @@ func TestContextUploadErrors(t *testing.T) {
|
||||
503, "uploads_store_unavailable", "send the request again", "5"},
|
||||
} {
|
||||
fs := &fakeSubmissions{chunkErr: tc.err}
|
||||
w := do(appSubAPI(fs).ExternalHandler(), "PUT", "/api/v1/me/submissions/sub-9/context/upload?offset=12", "abcd",
|
||||
map[string]string{"Content-Digest": contentDigestOf("abcd")})
|
||||
w := do(appSubAPI(fs).ExternalHandler(), "PUT", "/api/v1/me/submissions/sub-9/context/upload?offset=12", "abcd", nil)
|
||||
if w.Code != tc.code || decodeErr(t, w) != tc.want {
|
||||
t.Errorf("%s: %d %s, want %d %s", tc.name, w.Code, w.Body.String(), tc.code, tc.want)
|
||||
}
|
||||
@@ -123,8 +122,7 @@ func TestContextUploadCompleteHoldsTheCooldownAndAudits(t *testing.T) {
|
||||
t.Fatalf("second completion in the window: %d %s, want 429 submission_cooldown", w.Code, w.Body.String())
|
||||
}
|
||||
// A part never waits on the cooldown.
|
||||
if w := do(eh, "PUT", "/api/v1/me/submissions/sub-9/context/upload?offset=0", "\x1f\x8b",
|
||||
map[string]string{"Content-Digest": contentDigestOf("\x1f\x8b")}); w.Code != http.StatusOK {
|
||||
if w := do(eh, "PUT", "/api/v1/me/submissions/sub-9/context/upload?offset=0", "\x1f\x8b", nil); w.Code != http.StatusOK {
|
||||
t.Fatalf("part inside the cooldown: code = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -143,42 +141,3 @@ func TestContextUploadWithoutServiceIs503(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A context upload, whole or in parts, carries the SHA-256 of its body: one
|
||||
// without it is refused before the lane sees it, and bytes that do not hash to
|
||||
// it are refused as changed on the way (the lane keeps none of them), so the
|
||||
// client sends them again.
|
||||
func TestContextUploadsCheckTheBodyDigest(t *testing.T) {
|
||||
body := "\x1f\x8b\x08\x00 the modpack bytes"
|
||||
for _, rq := range []struct{ name, method, target string }{
|
||||
{"a part", "PUT", "/api/v1/me/submissions/sub-9/context/upload?offset=0"},
|
||||
{"a whole context", "POST", "/api/v1/me/submissions/sub-9/context"},
|
||||
} {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
sent string
|
||||
headers map[string]string
|
||||
code int
|
||||
want string
|
||||
}{
|
||||
{"without a digest", body, nil, http.StatusBadRequest, "digest_required"},
|
||||
{"changed on the way", body[:len(body)-1] + "X", map[string]string{"Content-Digest": contentDigestOf(body)}, http.StatusBadRequest, "digest_mismatch"},
|
||||
{"as sent", body, map[string]string{"Content-Digest": contentDigestOf(body)}, http.StatusOK, ""},
|
||||
} {
|
||||
t.Run(rq.name+" "+tc.name, func(t *testing.T) {
|
||||
fs := &fakeSubmissions{}
|
||||
w := do(appSubAPI(fs).ExternalHandler(), rq.method, rq.target, tc.sent, tc.headers)
|
||||
if w.Code != tc.code {
|
||||
t.Fatalf("code = %d (%s), want %d", w.Code, w.Body.String(), tc.code)
|
||||
}
|
||||
if tc.want != "" && decodeErr(t, w) != tc.want {
|
||||
t.Fatalf("error = %s, want %s", w.Body.String(), tc.want)
|
||||
}
|
||||
reached := fs.chunkID != "" || fs.uploadedID != ""
|
||||
if reached != (tc.headers != nil) {
|
||||
t.Fatalf("the body reached the lane: %v, want %v", reached, tc.headers != nil)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -73,13 +73,8 @@ func (f *fakeSubmissions) UploadStatus(_ context.Context, id, submittedBy string
|
||||
|
||||
func (f *fakeSubmissions) UploadPart(_ context.Context, id, submittedBy string, offset int64, r io.Reader) (submit.UploadProgress, error) {
|
||||
f.chunkID, f.chunkBy, f.partOffset = id, submittedBy, offset
|
||||
b, err := io.ReadAll(r)
|
||||
b, _ := io.ReadAll(r)
|
||||
f.partBody = string(b)
|
||||
if err != nil {
|
||||
// A read that fails (a body changed on the way) keeps nothing, as in
|
||||
// PartStore.
|
||||
return submit.UploadProgress{}, err
|
||||
}
|
||||
return f.progress, f.chunkErr
|
||||
}
|
||||
|
||||
@@ -106,10 +101,7 @@ func (f *fakeSubmissions) UploadContext(_ context.Context, id, submittedBy strin
|
||||
if f.uploadErr != nil {
|
||||
return nil, f.uploadErr
|
||||
}
|
||||
n, err := io.Copy(io.Discard, r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n, _ := io.Copy(io.Discard, r)
|
||||
f.uploadedN = n
|
||||
return &submit.Submission{ID: id, SubmittedBy: submittedBy, Status: submit.StatusPendingReview}, nil
|
||||
}
|
||||
@@ -243,7 +235,7 @@ func TestUploadSubmissionContextStreamsBody(t *testing.T) {
|
||||
// A tiny gzip-magic-prefixed body stands in for a real context.tar.gz.
|
||||
body := "\x1f\x8b\x08\x00 the modpack bytes"
|
||||
w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", body,
|
||||
map[string]string{"Content-Type": "application/gzip", "Content-Digest": contentDigestOf(body)})
|
||||
map[string]string{"Content-Type": "application/gzip"})
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("code = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -263,7 +255,7 @@ func TestUploadSubmissionContextStreamsBody(t *testing.T) {
|
||||
func TestUploadSubmissionContextNotOwnedIs404(t *testing.T) {
|
||||
fs := &fakeSubmissions{uploadErr: submit.ErrNotFound}
|
||||
api := appSubAPI(fs)
|
||||
w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-x/context", "\x1f\x8bdata", map[string]string{"Content-Digest": contentDigestOf("\x1f\x8bdata")})
|
||||
w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-x/context", "\x1f\x8bdata", nil)
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Fatalf("code = %d, want 404 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -273,7 +265,7 @@ func TestUploadSubmissionContextNotOwnedIs404(t *testing.T) {
|
||||
func TestUploadSubmissionContextAlreadyReviewedIs409(t *testing.T) {
|
||||
fs := &fakeSubmissions{uploadErr: submit.ErrAlreadyReviewed}
|
||||
api := appSubAPI(fs)
|
||||
w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", map[string]string{"Content-Digest": contentDigestOf("\x1f\x8bdata")})
|
||||
w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", nil)
|
||||
if w.Code != http.StatusConflict {
|
||||
t.Fatalf("code = %d, want 409 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -283,7 +275,7 @@ func TestUploadSubmissionContextAlreadyReviewedIs409(t *testing.T) {
|
||||
func TestUploadSubmissionContextBadFormatIs400(t *testing.T) {
|
||||
fs := &fakeSubmissions{uploadErr: fmt.Errorf("%w: build context must be a gzip-compressed tarball (.tar.gz)", submit.ErrInvalid)}
|
||||
api := appSubAPI(fs)
|
||||
w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "not gzip", map[string]string{"Content-Digest": contentDigestOf("not gzip")})
|
||||
w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "not gzip", nil)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("code = %d, want 400 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -297,7 +289,7 @@ func TestUploadSubmissionContextBadFormatIs400(t *testing.T) {
|
||||
func TestUploadSubmissionContextNoTransportIs503(t *testing.T) {
|
||||
fs := &fakeSubmissions{uploadErr: submit.ErrUploadsUnavailable}
|
||||
api := appSubAPI(fs)
|
||||
w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", map[string]string{"Content-Digest": contentDigestOf("\x1f\x8bdata")})
|
||||
w := do(api.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", nil)
|
||||
if w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("code = %d, want 503 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -311,7 +303,7 @@ func TestUploadSubmissionContextNoTransportIs503(t *testing.T) {
|
||||
func TestUploadSubmissionContextWithoutServiceIs503(t *testing.T) {
|
||||
app := appSubAPI(nil)
|
||||
app.Submissions = nil
|
||||
w := do(app.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", map[string]string{"Content-Digest": contentDigestOf("\x1f\x8bdata")})
|
||||
w := do(app.ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", nil)
|
||||
if w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("code = %d, want 503 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -879,7 +871,7 @@ func TestSubmissionQuotaIs403(t *testing.T) {
|
||||
})
|
||||
t.Run("upload", func(t *testing.T) {
|
||||
fs := &fakeSubmissions{uploadErr: fmt.Errorf("%w: exceeds your remaining storage allowance", submit.ErrQuotaExceeded)}
|
||||
w := do(appSubAPI(fs).ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", map[string]string{"Content-Digest": contentDigestOf("\x1f\x8bdata")})
|
||||
w := do(appSubAPI(fs).ExternalHandler(), "POST", "/api/v1/me/submissions/sub-9/context", "\x1f\x8bdata", nil)
|
||||
if w.Code != http.StatusForbidden {
|
||||
t.Fatalf("code = %d, want 403 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -947,12 +939,11 @@ func TestUploadSubmissionContextRateLimited(t *testing.T) {
|
||||
api.SubmitUploadCooldown = time.Minute
|
||||
eh := api.ExternalHandler()
|
||||
body := "\x1f\x8b\x08\x00 the modpack bytes"
|
||||
sent := map[string]string{"Content-Digest": contentDigestOf(body)}
|
||||
|
||||
if w := do(eh, "POST", "/api/v1/me/submissions/sub-9/context", body, sent); w.Code != http.StatusOK {
|
||||
if w := do(eh, "POST", "/api/v1/me/submissions/sub-9/context", body, nil); w.Code != http.StatusOK {
|
||||
t.Fatalf("first upload: code = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
w := do(eh, "POST", "/api/v1/me/submissions/sub-9/context", body, sent)
|
||||
w := do(eh, "POST", "/api/v1/me/submissions/sub-9/context", body, nil)
|
||||
if w.Code != http.StatusTooManyRequests || decodeErr(t, w) != "submission_cooldown" {
|
||||
t.Fatalf("immediate second upload: code = %d body %s, want 429 submission_cooldown", w.Code, w.Body.String())
|
||||
}
|
||||
@@ -964,11 +955,11 @@ func TestUploadSubmissionContextRateLimited(t *testing.T) {
|
||||
api2.Now = func() time.Time { return clock }
|
||||
api2.SubmitUploadCooldown = time.Minute
|
||||
eh2 := api2.ExternalHandler()
|
||||
if w := do(eh2, "POST", "/api/v1/me/submissions/sub-9/context", body, sent); w.Code != http.StatusServiceUnavailable {
|
||||
if w := do(eh2, "POST", "/api/v1/me/submissions/sub-9/context", body, nil); w.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("failed upload: code = %d, want 503", w.Code)
|
||||
}
|
||||
fs2.uploadErr = nil
|
||||
if w := do(eh2, "POST", "/api/v1/me/submissions/sub-9/context", body, sent); w.Code != http.StatusOK {
|
||||
if w := do(eh2, "POST", "/api/v1/me/submissions/sub-9/context", body, nil); w.Code != http.StatusOK {
|
||||
t.Fatalf("retry at the same instant after failure: code = %d, want 200 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Filter decides, for one regular file on its way out of a world into an
|
||||
// export, whether it is left out (withhold) or has its bytes replaced (rewrite
|
||||
// non-nil). name is the archive path; info is the file's, or nil when the file
|
||||
// is an entry of a stored archive, which has only names. A backup is not
|
||||
// filtered: it stays on the platform, and a restore must bring the world back
|
||||
// whole.
|
||||
type Filter func(name string, info fs.FileInfo) (withhold bool, rewrite func([]byte) []byte)
|
||||
|
||||
// maxRewrite bounds a file a Filter rewrites, which is read whole into memory.
|
||||
// The files it rewrites are small configs; one larger than this is withheld
|
||||
// rather than sent unrewritten.
|
||||
const maxRewrite = 1 << 20
|
||||
|
||||
// WriteTarGz archives srcDir into w laid out exactly as Archive lays out a
|
||||
// backup, so an exported world restores like any other archive. It returns the
|
||||
// entries a tar cannot hold and the files filter withheld. The world export Job
|
||||
// streams it straight into its upload.
|
||||
func WriteTarGz(ctx context.Context, w io.Writer, srcDir string, filter Filter) (skipped, withheld []string, err error) {
|
||||
st, err := writeTarGz(ctx, w, srcDir, filter)
|
||||
return st.skipped, st.withheld, err
|
||||
}
|
||||
|
||||
// FilterTarGz copies the gzip+tar archive read from r into w, passing every
|
||||
// regular file through filter by name. It is how a stored backup leaves the
|
||||
// platform: the archive is the world as it was, secrets included, so it is
|
||||
// re-written on the way out rather than handed over as stored.
|
||||
//
|
||||
// r is read to its very end, past the tar trailer, before w's archive is
|
||||
// closed. A reader that checks a digest when it reaches EOF therefore fails the
|
||||
// copy while w still lacks the end of its archive, and a receiver never holds a
|
||||
// complete-looking copy of a corrupt backup.
|
||||
func FilterTarGz(ctx context.Context, w io.Writer, r io.Reader, filter Filter) (withheld []string, err error) {
|
||||
zr, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: open archive: %w", err)
|
||||
}
|
||||
tr := tar.NewReader(zr)
|
||||
zw := gzip.NewWriter(w)
|
||||
tw := tar.NewWriter(zw)
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return withheld, err
|
||||
}
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return withheld, fmt.Errorf("backup: read archive: %w", err)
|
||||
}
|
||||
if !hdr.FileInfo().Mode().IsRegular() {
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return withheld, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
withhold, rewrite := filter(hdr.Name, nil)
|
||||
if withhold {
|
||||
withheld = append(withheld, hdr.Name)
|
||||
continue
|
||||
}
|
||||
if rewrite == nil {
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return withheld, err
|
||||
}
|
||||
if _, err := io.Copy(tw, tr); err != nil {
|
||||
return withheld, fmt.Errorf("backup: copy %s: %w", hdr.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
content, err := io.ReadAll(io.LimitReader(tr, maxRewrite+1))
|
||||
if err != nil {
|
||||
return withheld, fmt.Errorf("backup: read %s: %w", hdr.Name, err)
|
||||
}
|
||||
if len(content) > maxRewrite {
|
||||
withheld = append(withheld, hdr.Name)
|
||||
continue
|
||||
}
|
||||
content = rewrite(content)
|
||||
hdr.Size = int64(len(content))
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return withheld, err
|
||||
}
|
||||
if _, err := tw.Write(content); err != nil {
|
||||
return withheld, err
|
||||
}
|
||||
}
|
||||
// Through the gzip trailer to r's EOF (see above).
|
||||
if _, err := io.Copy(io.Discard, zr); err != nil {
|
||||
return withheld, fmt.Errorf("backup: read archive: %w", err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return withheld, fmt.Errorf("backup: close tar: %w", err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return withheld, fmt.Errorf("backup: close gzip: %w", err)
|
||||
}
|
||||
return withheld, nil
|
||||
}
|
||||
|
||||
// readRewritable reads a file a Filter rewrites, reporting false when it is
|
||||
// larger than maxRewrite.
|
||||
func readRewritable(path string) ([]byte, bool, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer f.Close()
|
||||
b, err := io.ReadAll(io.LimitReader(f, maxRewrite+1))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return b, len(b) <= maxRewrite, nil
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// testFilter withholds secret.yml and rewrites props.txt, by name.
|
||||
func testFilter(name string, _ fs.FileInfo) (bool, func([]byte) []byte) {
|
||||
switch filepath.Base(name) {
|
||||
case "secret.yml":
|
||||
return true, nil
|
||||
case "props.txt":
|
||||
return false, summarize
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// summarize is a rewrite that changes the length, so a header left with the
|
||||
// old size shows: the first bytes upper-cased, then how many there were.
|
||||
func summarize(b []byte) []byte {
|
||||
return fmt.Appendf(nil, "%s (%d bytes)", bytes.ToUpper(b[:min(len(b), 8)]), len(b))
|
||||
}
|
||||
|
||||
// untar reads a gzip+tar stream into name → content ("<dir>" for folders,
|
||||
// "-> target" for symbolic links).
|
||||
func untar(t *testing.T, b []byte) map[string]string {
|
||||
t.Helper()
|
||||
zr, err := gzip.NewReader(bytes.NewReader(b))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tr := tar.NewReader(zr)
|
||||
got := map[string]string{}
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return got
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
got[hdr.Name] = "<dir>"
|
||||
continue
|
||||
case tar.TypeSymlink:
|
||||
got[hdr.Name] = "-> " + hdr.Linkname
|
||||
continue
|
||||
}
|
||||
body, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if int64(len(body)) != hdr.Size {
|
||||
t.Fatalf("%s: header says %d bytes, body has %d", hdr.Name, hdr.Size, len(body))
|
||||
}
|
||||
got[hdr.Name] = string(body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteTarGzFilter(t *testing.T) {
|
||||
src := t.TempDir()
|
||||
for name, body := range map[string]string{
|
||||
"keep.txt": "kept",
|
||||
"props.txt": "rcon=x",
|
||||
"conf/secret.yml": "key",
|
||||
"conf/big/props.txt": strings.Repeat("a", maxRewrite+1),
|
||||
"edge/props.txt": strings.Repeat("a", maxRewrite),
|
||||
} {
|
||||
p := filepath.Join(src, name)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.Symlink("keep.txt", filepath.Join(src, "link")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
skipped, withheld, err := WriteTarGz(context.Background(), &out, src, testFilter)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{
|
||||
"keep.txt": "kept", "props.txt": "RCON=X (6 bytes)",
|
||||
"conf/": "<dir>", "conf/big/": "<dir>",
|
||||
// Exactly maxRewrite bytes still fits.
|
||||
"edge/": "<dir>", "edge/props.txt": "AAAAAAAA (1048576 bytes)",
|
||||
}
|
||||
if got := untar(t, out.Bytes()); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("archive = %v\nwant %v", got, want)
|
||||
}
|
||||
if !reflect.DeepEqual(skipped, []string{"link"}) {
|
||||
t.Errorf("skipped = %v, want [link]", skipped)
|
||||
}
|
||||
// An oversized file the filter would rewrite goes out withheld, never as is.
|
||||
if !reflect.DeepEqual(withheld, []string{"conf/big/props.txt", "conf/secret.yml"}) {
|
||||
t.Errorf("withheld = %v", withheld)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, _, err := WriteTarGz(ctx, io.Discard, src, testFilter); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancelled: err = %v, want context.Canceled", err)
|
||||
}
|
||||
|
||||
// No filter keeps everything: the backup path.
|
||||
out.Reset()
|
||||
if _, withheld, err := WriteTarGz(context.Background(), &out, src, nil); err != nil || withheld != nil {
|
||||
t.Fatalf("unfiltered: withheld %v, err %v", withheld, err)
|
||||
}
|
||||
if got := untar(t, out.Bytes()); got["props.txt"] != "rcon=x" || got["conf/secret.yml"] != "key" {
|
||||
t.Fatalf("unfiltered archive changed files: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// storedArchive is a gzip+tar like one Archive writes.
|
||||
func storedArchive(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
add := func(hdr *tar.Header, body string) {
|
||||
hdr.Size = int64(len(body))
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write([]byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
add(&tar.Header{Name: "conf/", Typeflag: tar.TypeDir, Mode: 0o755}, "")
|
||||
add(&tar.Header{Name: "conf/secret.yml", Typeflag: tar.TypeReg, Mode: 0o600}, "key")
|
||||
add(&tar.Header{Name: "props.txt", Typeflag: tar.TypeReg, Mode: 0o644}, "rcon=x")
|
||||
add(&tar.Header{Name: "world/level.dat", Typeflag: tar.TypeReg, Mode: 0o600}, "level")
|
||||
add(&tar.Header{Name: "big/props.txt", Typeflag: tar.TypeReg, Mode: 0o644}, strings.Repeat("a", maxRewrite+1))
|
||||
add(&tar.Header{Name: "edge/props.txt", Typeflag: tar.TypeReg, Mode: 0o644}, strings.Repeat("a", maxRewrite))
|
||||
// A link holds no bytes, so it passes whatever its name.
|
||||
add(&tar.Header{Name: "old/secret.yml", Typeflag: tar.TypeSymlink, Linkname: "../world/level.dat", Mode: 0o777}, "")
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// failAtEOF passes r through and turns its EOF into err, as the export Job's
|
||||
// digest check does on a mismatch.
|
||||
type failAtEOF struct {
|
||||
r io.Reader
|
||||
err error
|
||||
read int
|
||||
}
|
||||
|
||||
func (f *failAtEOF) Read(p []byte) (int, error) {
|
||||
n, err := f.r.Read(p)
|
||||
f.read += n
|
||||
if err == io.EOF {
|
||||
return n, f.err
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func TestFilterTarGz(t *testing.T) {
|
||||
stored := storedArchive(t)
|
||||
|
||||
t.Run("withholds and rewrites by name, keeps the rest", func(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
withheld, err := FilterTarGz(context.Background(), &out, bytes.NewReader(stored), testFilter)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{
|
||||
"conf/": "<dir>", "props.txt": "RCON=X (6 bytes)", "world/level.dat": "level",
|
||||
"edge/props.txt": "AAAAAAAA (1048576 bytes)", "old/secret.yml": "-> ../world/level.dat",
|
||||
}
|
||||
if got := untar(t, out.Bytes()); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("archive = %v\nwant %v", got, want)
|
||||
}
|
||||
if !reflect.DeepEqual(withheld, []string{"conf/secret.yml", "big/props.txt"}) {
|
||||
t.Errorf("withheld = %v", withheld)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an error at the end of the input leaves the output unfinished", func(t *testing.T) {
|
||||
bad := errors.New("digest mismatch")
|
||||
src := &failAtEOF{r: bytes.NewReader(stored), err: bad}
|
||||
var out bytes.Buffer
|
||||
if _, err := FilterTarGz(context.Background(), &out, src, testFilter); !errors.Is(err, bad) {
|
||||
t.Fatalf("err = %v, want the end-of-input error", err)
|
||||
}
|
||||
if src.read != len(stored) {
|
||||
t.Fatalf("read %d of %d input bytes", src.read, len(stored))
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(out.Bytes()))
|
||||
if err == nil {
|
||||
_, err = io.ReadAll(zr)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("the output is a complete gzip stream; it must lack its end")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a cancelled copy stops", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := FilterTarGz(ctx, io.Discard, bytes.NewReader(stored), testFilter); err != context.Canceled {
|
||||
t.Fatalf("err = %v, want context.Canceled", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("not an archive", func(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if _, err := FilterTarGz(context.Background(), &out, strings.NewReader("plain text"), testFilter); err == nil {
|
||||
t.Fatal("a non-gzip input was accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
+12
-35
@@ -79,7 +79,7 @@ func (t *TarLocal) Archive(ctx context.Context, server, pvc string) (Archived, e
|
||||
return Archived{}, fmt.Errorf("backup: create archive: %w", err)
|
||||
}
|
||||
h := sha256.New()
|
||||
st, err := writeTarGz(ctx, io.MultiWriter(f, h), srcDir, nil)
|
||||
st, err := writeTarGz(ctx, io.MultiWriter(f, h), srcDir)
|
||||
if err == nil {
|
||||
if err = f.Sync(); err != nil {
|
||||
err = fmt.Errorf("backup: sync archive: %w", err)
|
||||
@@ -349,20 +349,17 @@ func (t *TarLocal) Delete(_ context.Context, ref ArchiveRef) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// tarStats is what writeTarGz put in the archive and what it left out: skipped
|
||||
// are entries a tar cannot hold, withheld the files the filter kept back.
|
||||
// tarStats is what writeTarGz put in the archive and what it left out.
|
||||
type tarStats struct {
|
||||
entries int
|
||||
skipped []string
|
||||
withheld []string
|
||||
}
|
||||
|
||||
// writeTarGz archives srcDir (not the root entry itself) as gzip+tar. Each entry
|
||||
// keeps its permission bits, without setuid, setgid and sticky, and its
|
||||
// modification time. Entries other than regular files and directories are left
|
||||
// out and listed in the stats. A non-nil filter sees every regular file (see
|
||||
// Filter); a backup passes nil and keeps everything.
|
||||
func writeTarGz(ctx context.Context, w io.Writer, srcDir string, filter Filter) (tarStats, error) {
|
||||
// out and listed in the stats.
|
||||
func writeTarGz(ctx context.Context, w io.Writer, srcDir string) (tarStats, error) {
|
||||
var st tarStats
|
||||
gz := gzip.NewWriter(w)
|
||||
tw := tar.NewWriter(gz)
|
||||
@@ -395,34 +392,6 @@ func writeTarGz(ctx context.Context, w io.Writer, srcDir string, filter Filter)
|
||||
st.entries++
|
||||
return nil
|
||||
case info.Mode().IsRegular():
|
||||
var rewrite func([]byte) []byte
|
||||
if filter != nil {
|
||||
var withhold bool
|
||||
if withhold, rewrite = filter(name, info); withhold {
|
||||
st.withheld = append(st.withheld, name)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if rewrite != nil {
|
||||
content, ok, err := readRewritable(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
st.withheld = append(st.withheld, name)
|
||||
return nil
|
||||
}
|
||||
content = rewrite(content)
|
||||
hdr := &tar.Header{Name: name, Mode: mode, ModTime: info.ModTime(), Size: int64(len(content)), Typeflag: tar.TypeReg}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tw.Write(content); err != nil {
|
||||
return err
|
||||
}
|
||||
st.entries++
|
||||
return nil
|
||||
}
|
||||
hdr := &tar.Header{Name: name, Mode: mode, ModTime: info.ModTime(), Size: info.Size(), Typeflag: tar.TypeReg}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
@@ -457,6 +426,14 @@ func writeTarGz(ctx context.Context, w io.Writer, srcDir string, filter Filter)
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// WriteTarGz archives srcDir into w laid out exactly as Archive lays out a
|
||||
// backup, so an exported world restores like any other archive, and returns the
|
||||
// entries it left out. The world export Job streams it straight into its upload.
|
||||
func WriteTarGz(ctx context.Context, w io.Writer, srcDir string) ([]string, error) {
|
||||
st, err := writeTarGz(ctx, w, srcDir)
|
||||
return st.skipped, err
|
||||
}
|
||||
|
||||
// dirMeta is a directory's recorded permission bits and modification time,
|
||||
// applied once nothing more is written into it.
|
||||
type dirMeta struct {
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
package fileedit
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/flate"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
)
|
||||
|
||||
// Content types of a download: a file goes out as its bytes, a folder as a zip.
|
||||
const (
|
||||
DownloadFileType = "application/octet-stream"
|
||||
DownloadZipType = "application/zip"
|
||||
)
|
||||
|
||||
// Download is one file or folder of a world on its way to the owner's browser,
|
||||
// run by the export Job (cmd/felis export --mode files). It passes the same
|
||||
// guards a read does (Guard): the forwarding-secret file never leaves, and
|
||||
// server.properties leaves with its RCON password redacted.
|
||||
type Download struct {
|
||||
// Size is a file download's exact length, or -1 for a folder, whose zip is
|
||||
// written as it streams.
|
||||
Size int64
|
||||
ContentType string
|
||||
// Skipped and Withheld count, once WriteTo has run, the entries a folder
|
||||
// download left out: links, devices and sockets, and guarded files.
|
||||
Skipped, Withheld int
|
||||
|
||||
root *os.Root
|
||||
name string
|
||||
file *os.File // a file download
|
||||
body []byte // a redacted file download
|
||||
guard Guard
|
||||
}
|
||||
|
||||
// OpenDownload opens name under rootPath for download. dir is what the caller
|
||||
// saw at name when it asked (the panel's listing): a download of a file that has
|
||||
// since become a folder, or the reverse, is refused rather than sent as the
|
||||
// other thing. The world root itself is refused; the world export sends that.
|
||||
func OpenDownload(rootPath, name string, dir bool) (*Download, error) {
|
||||
name = path.Clean(name)
|
||||
if name == "." || name == "/" || !fs.ValidPath(name) {
|
||||
return nil, fmt.Errorf("%s is not a file or folder inside the world", name)
|
||||
}
|
||||
r, err := os.OpenRoot(rootPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open the world root: %w", err)
|
||||
}
|
||||
d := &Download{root: r, name: name, guard: NewGuard(r)}
|
||||
if err := d.open(dir); err != nil {
|
||||
r.Close()
|
||||
return nil, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (d *Download) open(dir bool) error {
|
||||
info, err := d.root.Stat(d.name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() != dir {
|
||||
if info.IsDir() {
|
||||
return fmt.Errorf("%s is a folder now; reload the file list and download it again", d.name)
|
||||
}
|
||||
return fmt.Errorf("%s is not a folder now; reload the file list and download it again", d.name)
|
||||
}
|
||||
if dir {
|
||||
d.Size, d.ContentType = -1, DownloadZipType
|
||||
return nil
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("%s is not a regular file", d.name)
|
||||
}
|
||||
withhold, redact := d.guard.Rule(info)
|
||||
if withhold {
|
||||
return fmt.Errorf("%s is the file holding the proxy forwarding secret, which is shared cluster-wide, and cannot be downloaded", d.name)
|
||||
}
|
||||
f, err := d.root.Open(d.name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.ContentType = DownloadFileType
|
||||
if !redact {
|
||||
d.file, d.Size = f, info.Size()
|
||||
return nil
|
||||
}
|
||||
defer f.Close()
|
||||
b, err := io.ReadAll(io.LimitReader(f, MaxReadBytes+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(b) > MaxReadBytes {
|
||||
return fmt.Errorf("%s is over %d bytes and cannot be redacted for download", d.name, MaxReadBytes)
|
||||
}
|
||||
d.body = RedactProps(b)
|
||||
d.Size = int64(len(d.body))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close releases what OpenDownload opened.
|
||||
func (d *Download) Close() error {
|
||||
if d.file != nil {
|
||||
d.file.Close()
|
||||
}
|
||||
return d.root.Close()
|
||||
}
|
||||
|
||||
// WriteTo writes the download to w: exactly Size bytes of a file, or a zip of a
|
||||
// folder whose entries sit under the folder's own name, so unpacking it makes
|
||||
// that one folder. A file that shrank since it was opened is an error, never a
|
||||
// short download passed off as whole.
|
||||
func (d *Download) WriteTo(ctx context.Context, w io.Writer) error {
|
||||
switch {
|
||||
case d.body != nil:
|
||||
_, err := w.Write(d.body)
|
||||
return err
|
||||
case d.file != nil:
|
||||
_, err := io.CopyN(w, ctxReader{ctx, d.file}, d.Size)
|
||||
if errors.Is(err, io.EOF) {
|
||||
return fmt.Errorf("%s shrank while it was being downloaded", d.name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return d.writeZip(ctx, w)
|
||||
}
|
||||
|
||||
// writeZip streams the folder as a zip. Everything is deflated at the fastest
|
||||
// level: the Job has one CPU and the owner's connection is the slower end, and
|
||||
// already-compressed files (jars, region files) come out as stored blocks
|
||||
// without costing much. Links, devices and sockets are left out, like a world
|
||||
// export leaves them out.
|
||||
func (d *Download) writeZip(ctx context.Context, w io.Writer) error {
|
||||
zw := zip.NewWriter(w)
|
||||
zw.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
|
||||
return flate.NewWriter(out, flate.BestSpeed)
|
||||
})
|
||||
base := path.Base(d.name)
|
||||
err := fs.WalkDir(d.root.FS(), d.name, func(p string, de fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
entry := base + p[len(d.name):]
|
||||
info, err := de.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case de.IsDir():
|
||||
hdr := &zip.FileHeader{Name: entry + "/", Modified: info.ModTime()}
|
||||
hdr.SetMode(info.Mode().Perm() | fs.ModeDir)
|
||||
_, err := zw.CreateHeader(hdr)
|
||||
return err
|
||||
case !de.Type().IsRegular():
|
||||
d.Skipped++
|
||||
return nil
|
||||
}
|
||||
withhold, redact := d.guard.Rule(info)
|
||||
if withhold {
|
||||
d.Withheld++
|
||||
return nil
|
||||
}
|
||||
f, err := d.root.Open(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
var src io.Reader = ctxReader{ctx, f}
|
||||
if redact {
|
||||
b, err := io.ReadAll(io.LimitReader(f, MaxReadBytes+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(b) > MaxReadBytes {
|
||||
d.Withheld++
|
||||
return nil
|
||||
}
|
||||
src = bytes.NewReader(RedactProps(b))
|
||||
}
|
||||
hdr := &zip.FileHeader{Name: entry, Method: zip.Deflate, Modified: info.ModTime()}
|
||||
hdr.SetMode(info.Mode().Perm())
|
||||
fw, err := zw.CreateHeader(hdr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(fw, src)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return zw.Close()
|
||||
}
|
||||
|
||||
// ctxReader stops a long copy once ctx is done.
|
||||
type ctxReader struct {
|
||||
ctx context.Context
|
||||
r io.Reader
|
||||
}
|
||||
|
||||
func (c ctxReader) Read(p []byte) (int, error) {
|
||||
if err := c.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return c.r.Read(p)
|
||||
}
|
||||
@@ -1,304 +0,0 @@
|
||||
package fileedit
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// downloadWorld is worldRoot plus the guarded files with real secrets in them.
|
||||
func downloadWorld(t *testing.T) string {
|
||||
t.Helper()
|
||||
root, _ := worldRoot(t)
|
||||
for name, body := range map[string]string{
|
||||
"server.properties": "motd=hi\nrcon.password=hunter2\n",
|
||||
"config/paper-global.yml": "secret: aVeryRealForwardingKey\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
// download runs a whole download into memory.
|
||||
func download(t *testing.T, root, name string, dir bool) (*Download, []byte) {
|
||||
t.Helper()
|
||||
d, err := OpenDownload(root, name, dir)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenDownload(%s): %v", name, err)
|
||||
}
|
||||
defer d.Close()
|
||||
var out bytes.Buffer
|
||||
if err := d.WriteTo(context.Background(), &out); err != nil {
|
||||
t.Fatalf("WriteTo(%s): %v", name, err)
|
||||
}
|
||||
return d, out.Bytes()
|
||||
}
|
||||
|
||||
// unzipped reads a zip into name → content ("<dir> <mode>" for folders).
|
||||
func unzipped(t *testing.T, b []byte) map[string]string {
|
||||
t.Helper()
|
||||
zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, f := range zr.File {
|
||||
if f.FileInfo().IsDir() {
|
||||
// The Unix mode, S_IFDIR included, which is what unzip tools
|
||||
// restore a folder's permissions from.
|
||||
got[f.Name] = fmt.Sprintf("<dir %o>", f.ExternalAttrs>>16)
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got[f.Name] = f.Mode().Perm().String() + " " + string(body)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func TestDownload(t *testing.T) {
|
||||
const redacted = "motd=hi\nrcon.password=" + redactedValue + "\n"
|
||||
|
||||
t.Run("a file goes out as its exact bytes", func(t *testing.T) {
|
||||
root := downloadWorld(t)
|
||||
d, got := download(t, root, "config/paper.yml", false)
|
||||
if string(got) != "verbose: false\n" || d.Size != int64(len(got)) || d.ContentType != DownloadFileType {
|
||||
t.Fatalf("download = %q, size %d, type %s", got, d.Size, d.ContentType)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("server.properties, under any name, goes out redacted", func(t *testing.T) {
|
||||
root := downloadWorld(t)
|
||||
if err := os.Link(filepath.Join(root, "server.properties"), filepath.Join(root, "copy.txt")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
symlink(t, "server.properties", filepath.Join(root, "sym.txt"))
|
||||
for _, name := range []string{"server.properties", "./server.properties", "copy.txt", "sym.txt"} {
|
||||
d, got := download(t, root, name, false)
|
||||
if string(got) != redacted || d.Size != int64(len(redacted)) {
|
||||
t.Errorf("%s: download = %q, size %d; want %q", name, got, d.Size, redacted)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the forwarding secret, under any name, is refused", func(t *testing.T) {
|
||||
root := downloadWorld(t)
|
||||
if err := os.Link(filepath.Join(root, "config/paper-global.yml"), filepath.Join(root, "hard.yml")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
symlink(t, "config", filepath.Join(root, "cfg"))
|
||||
for _, name := range []string{"config/paper-global.yml", "hard.yml", "cfg/paper-global.yml"} {
|
||||
d, err := OpenDownload(root, name, false)
|
||||
if err == nil {
|
||||
d.Close()
|
||||
t.Errorf("%s: opened for download", name)
|
||||
} else if !strings.Contains(err.Error(), "forwarding secret") {
|
||||
t.Errorf("%s: err = %v", name, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// A guarded name that is itself a link guards what it points at: that file
|
||||
// is what the server reads, under whatever name it is reached.
|
||||
t.Run("a guarded name that is a link guards its target", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
for name, body := range map[string]string{
|
||||
"config/real.yml": "secret: aVeryRealForwardingKey\n",
|
||||
"real.properties": "motd=hi\nrcon.password=hunter2\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.Remove(filepath.Join(root, "server.properties")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
symlink(t, "real.yml", filepath.Join(root, "config/paper-global.yml"))
|
||||
symlink(t, "real.properties", filepath.Join(root, "server.properties"))
|
||||
if d, err := OpenDownload(root, "config/real.yml", false); err == nil {
|
||||
d.Close()
|
||||
t.Error("the forwarding secret opened under its link target's name")
|
||||
}
|
||||
if _, got := download(t, root, "real.properties", false); string(got) != redacted {
|
||||
t.Errorf("real.properties = %q, want %q", got, redacted)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("what is not there as the listing said is refused", func(t *testing.T) {
|
||||
root := downloadWorld(t)
|
||||
// Opening a FIFO for reading would wait for a writer that never comes.
|
||||
if err := syscall.Mkfifo(filepath.Join(root, "pipe"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
dir bool
|
||||
want string
|
||||
}{
|
||||
{"config", false, "is a folder now"},
|
||||
{"server.properties", true, "is not a folder now"},
|
||||
{"missing.txt", false, "no such file"},
|
||||
{"pipe", false, "not a regular file"},
|
||||
{".", true, "not a file or folder inside"},
|
||||
{"", true, "not a file or folder inside"},
|
||||
{"../outside", true, "not a file or folder inside"},
|
||||
{"/etc", true, "not a file or folder inside"},
|
||||
} {
|
||||
d, err := OpenDownload(root, c.name, c.dir)
|
||||
if err == nil {
|
||||
d.Close()
|
||||
t.Errorf("%q: opened", c.name)
|
||||
} else if !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("%q: err = %v, want %q", c.name, err, c.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a folder goes out as a zip under its own name, guarded", func(t *testing.T) {
|
||||
root := downloadWorld(t)
|
||||
plugins := filepath.Join(root, "plugins")
|
||||
for _, d := range []string{"plugins/Essentials/empty", "plugins/Essentials/data"} {
|
||||
if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for name, body := range map[string]string{
|
||||
"plugins/a.jar": "jar",
|
||||
"plugins/Essentials/config.yml": "x: 1",
|
||||
"plugins/Essentials/data/server.properties": "rcon.password=notthereal\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for name, mode := range map[string]os.FileMode{"a.jar": 0o755, "Essentials/empty": 0o700} {
|
||||
if err := os.Chmod(filepath.Join(plugins, name), mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.Link(filepath.Join(root, "config/paper-global.yml"), filepath.Join(plugins, "stolen.yml")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Link(filepath.Join(root, "server.properties"), filepath.Join(plugins, "props.txt")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
symlink(t, "../config/paper-global.yml", filepath.Join(plugins, "sym.yml"))
|
||||
|
||||
d, b := download(t, root, "plugins", true)
|
||||
if d.Size != -1 || d.ContentType != DownloadZipType {
|
||||
t.Fatalf("size %d, type %s", d.Size, d.ContentType)
|
||||
}
|
||||
want := map[string]string{
|
||||
"plugins/": "<dir 40755>",
|
||||
"plugins/Essentials/": "<dir 40755>",
|
||||
"plugins/Essentials/empty/": "<dir 40700>",
|
||||
"plugins/Essentials/data/": "<dir 40755>",
|
||||
"plugins/a.jar": "-rwxr-xr-x jar",
|
||||
"plugins/Essentials/config.yml": "-rw-r--r-- x: 1",
|
||||
// Only the world root's server.properties is the server's.
|
||||
"plugins/Essentials/data/server.properties": "-rw-r--r-- rcon.password=notthereal\n",
|
||||
"plugins/props.txt": "-rw-r--r-- " + redacted,
|
||||
}
|
||||
if got := unzipped(t, b); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("zip = %v\nwant %v", got, want)
|
||||
}
|
||||
if d.Skipped != 1 || d.Withheld != 1 {
|
||||
t.Errorf("skipped %d, withheld %d; want 1 and 1", d.Skipped, d.Withheld)
|
||||
}
|
||||
if bytes.Contains(b, []byte("aVeryReal")) || bytes.Contains(b, []byte("hunter2")) {
|
||||
t.Fatal("a secret is in the zip")
|
||||
}
|
||||
|
||||
// A nested folder unpacks as itself, not under its parents.
|
||||
_, b = download(t, root, "plugins/Essentials/data", true)
|
||||
want = map[string]string{
|
||||
"data/": "<dir 40755>",
|
||||
"data/server.properties": "-rw-r--r-- rcon.password=notthereal\n",
|
||||
}
|
||||
if got := unzipped(t, b); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("nested zip = %v\nwant %v", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
// Redaction reads the file whole; one too big for that is refused, or left
|
||||
// out of a folder, never sent as it is.
|
||||
t.Run("a server.properties too big to redact is refused", func(t *testing.T) {
|
||||
root := downloadWorld(t)
|
||||
big := append([]byte("rcon.password=hunter2\n"), bytes.Repeat([]byte("#"), MaxReadBytes)...)
|
||||
if err := os.WriteFile(filepath.Join(root, "server.properties"), big, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := OpenDownload(root, "server.properties", false)
|
||||
if err == nil {
|
||||
d.Close()
|
||||
t.Fatal("an oversized server.properties opened for download")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "cannot be redacted") {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
|
||||
if err := os.Link(filepath.Join(root, "server.properties"), filepath.Join(root, "config/props.txt")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, b := download(t, root, "config", true)
|
||||
want := map[string]string{"config/": "<dir 40755>", "config/paper.yml": "-rw-r--r-- verbose: false\n"}
|
||||
if got := unzipped(t, b); !reflect.DeepEqual(got, want) || d.Withheld != 2 {
|
||||
t.Fatalf("zip = %v, withheld %d; want %v and 2", got, d.Withheld, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a file that shrinks mid-download fails it", func(t *testing.T) {
|
||||
root := downloadWorld(t)
|
||||
d, err := OpenDownload(root, "config/paper.yml", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer d.Close()
|
||||
if err := os.Truncate(filepath.Join(root, "config/paper.yml"), 3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.WriteTo(context.Background(), io.Discard); err == nil || !strings.Contains(err.Error(), "shrank") {
|
||||
t.Fatalf("err = %v, want the shrank error", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a cancelled download stops", func(t *testing.T) {
|
||||
root := downloadWorld(t)
|
||||
// Folders only: no file copy is there to notice the cancel.
|
||||
if err := os.MkdirAll(filepath.Join(root, "empty/a/b"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
dir bool
|
||||
}{{"config/paper.yml", false}, {"config", true}, {"empty", true}} {
|
||||
d, err := OpenDownload(root, c.name, c.dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.WriteTo(ctx, io.Discard); err != context.Canceled {
|
||||
t.Errorf("%s: err = %v, want context.Canceled", c.name, err)
|
||||
}
|
||||
d.Close()
|
||||
}
|
||||
})
|
||||
}
|
||||
+25
-146
@@ -77,17 +77,15 @@ var (
|
||||
// ErrExists is a create, mkdir, rename or upload whose target is already
|
||||
// there.
|
||||
ErrExists = errors.New("fileedit: the target already exists")
|
||||
// ErrReadDamaged is a read whose bytes do not hash to the digest the Job
|
||||
// sent with them: they changed on the way to felis-api.
|
||||
ErrReadDamaged = errors.New("fileedit: the file's bytes changed on their way from the file Job")
|
||||
)
|
||||
|
||||
// Runner is the cluster-side half of a file operation. Run renders and creates
|
||||
// the Job, waits for its Pod to reach a terminal phase, and returns the marked
|
||||
// JSON payload the Pod printed: from the API's point of view an operation is one
|
||||
// call. An upload too big for one request and an unzip can outlast any request,
|
||||
// so those two are started instead (Start) and read back later (Ops), from the
|
||||
// Jobs felis-api lists and the progress lines their Pods print.
|
||||
// Runner is the cluster-side half of one file operation: render and create the
|
||||
// Job, wait for its Pod to reach a terminal phase, and return the marked JSON
|
||||
// payload the Pod printed. It is one method rather than a create/poll/read trio
|
||||
// because felis-api cannot poll a Job at all (no jobs:get — see FilesJobName), so
|
||||
// there is no intermediate state a caller could usefully observe; the operation is
|
||||
// synchronous from the API's point of view whether or not the seam pretends
|
||||
// otherwise.
|
||||
//
|
||||
// It is an interface so the Editor's orchestration and error mapping are tested
|
||||
// against a fake; the client-go implementation (K8sRunner) is integration-only.
|
||||
@@ -95,11 +93,6 @@ type Runner interface {
|
||||
// Run creates the Job for p and returns the raw JSON payload from the
|
||||
// ResultPrefix line of its Pod's log.
|
||||
Run(ctx context.Context, p JobParams) ([]byte, error)
|
||||
// Start creates the Job for p and returns once it exists.
|
||||
Start(ctx context.Context, p JobParams) error
|
||||
// Ops reports the background operations (JobParams.Async) of one server
|
||||
// whose Jobs the cluster still holds, newest first.
|
||||
Ops(ctx context.Context, namespace, server string) ([]OpState, error)
|
||||
}
|
||||
|
||||
// Config parameterises the file editor. Image has no default on purpose: it is
|
||||
@@ -146,15 +139,6 @@ type Config struct {
|
||||
// it must stay comfortably longer than the moment felis-api needs to read the
|
||||
// Pod's log, because the TTL takes the Pod (and its log) with the Job.
|
||||
TTLAfterFinished time.Duration
|
||||
|
||||
// AsyncDeadline, AsyncTTL and AsyncCPULimit stand in for Deadline,
|
||||
// TTLAfterFinished and CPULimit on an upload or unzip felis-api starts and
|
||||
// does not wait on. Such a Job moves a whole archive or a file of gigabytes,
|
||||
// so it gets hours; it stays after finishing long enough for the panel to
|
||||
// show how it ended; and it gets a whole core, since inflating is CPU-bound.
|
||||
AsyncDeadline time.Duration
|
||||
AsyncTTL time.Duration
|
||||
AsyncCPULimit string
|
||||
}
|
||||
|
||||
// defaults applied when a Config field is left zero. They are sized for what a
|
||||
@@ -169,9 +153,6 @@ const (
|
||||
defaultCPULimit = "500m"
|
||||
defaultMemLimit = "256Mi"
|
||||
defaultTTL = 2 * time.Minute
|
||||
defaultAsyncDeadline = 2 * time.Hour
|
||||
defaultAsyncTTL = 30 * time.Minute
|
||||
defaultAsyncCPULimit = "1"
|
||||
)
|
||||
|
||||
// withDefaults returns a copy of c with zero fields filled, so a partially
|
||||
@@ -201,15 +182,6 @@ func (c Config) withDefaults() Config {
|
||||
if c.TTLAfterFinished <= 0 {
|
||||
c.TTLAfterFinished = defaultTTL
|
||||
}
|
||||
if c.AsyncDeadline <= 0 {
|
||||
c.AsyncDeadline = defaultAsyncDeadline
|
||||
}
|
||||
if c.AsyncTTL <= 0 {
|
||||
c.AsyncTTL = defaultAsyncTTL
|
||||
}
|
||||
if c.AsyncCPULimit == "" {
|
||||
c.AsyncCPULimit = defaultAsyncCPULimit
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -219,44 +191,28 @@ type Editor struct {
|
||||
Config Config
|
||||
}
|
||||
|
||||
// Listing is one directory as List returns it.
|
||||
type Listing struct {
|
||||
Entries []Entry
|
||||
// Truncated reports that the directory holds more than MaxEntries.
|
||||
Truncated bool
|
||||
// Free is the bytes free on the server's volume, negative when the Job could
|
||||
// not tell.
|
||||
Free int64
|
||||
}
|
||||
|
||||
// List returns one directory's entries, resolved under the server's world root.
|
||||
// An empty path lists the world root itself.
|
||||
func (e *Editor) List(ctx context.Context, server, path string) (Listing, error) {
|
||||
func (e *Editor) List(ctx context.Context, server, path string) ([]Entry, bool, error) {
|
||||
res, err := e.run(ctx, server, JobParams{Op: OpList, Path: path})
|
||||
if err != nil {
|
||||
return Listing{}, err
|
||||
return nil, false, err
|
||||
}
|
||||
// A genuinely empty directory unmarshals Entries as nil; normalise it so the
|
||||
// handler serialises [] rather than null.
|
||||
if res.Entries == nil {
|
||||
res.Entries = []Entry{}
|
||||
}
|
||||
return Listing{Entries: res.Entries, Truncated: res.Truncated, Free: res.Avail}, nil
|
||||
return res.Entries, res.Truncated, nil
|
||||
}
|
||||
|
||||
// Read returns a file's bytes, resolved under the server's world root, and the
|
||||
// SHA-256 of the file as it is on disk — the value to hand back as Write's expect.
|
||||
// Bytes that do not hash to the digest the Job computed over what it sent
|
||||
// (Result.ContentSHA256) are ErrReadDamaged: an editor that saves back a
|
||||
// damaged read would write the damage.
|
||||
func (e *Editor) Read(ctx context.Context, server, path string) ([]byte, string, error) {
|
||||
res, err := e.run(ctx, server, JobParams{Op: OpRead, Path: path})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if got := digest(res.Content); got != res.ContentSHA256 {
|
||||
return nil, "", fmt.Errorf("%w: they hash to %s, sent as %q", ErrReadDamaged, got, res.ContentSHA256)
|
||||
}
|
||||
// A zero-length file unmarshals Content as nil, which is a legitimate result,
|
||||
// not an error — normalise so the caller never has to distinguish nil from empty.
|
||||
if res.Content == nil {
|
||||
@@ -333,7 +289,7 @@ func (e *Editor) run(ctx context.Context, server string, p JobParams) (Result, e
|
||||
}
|
||||
|
||||
cfg := e.Config.withDefaults()
|
||||
p, err := cfg.params(server, p)
|
||||
opID, err := newOpID()
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
@@ -345,6 +301,20 @@ func (e *Editor) run(ctx context.Context, server string, p JobParams) (Result, e
|
||||
ctx, cancel := context.WithTimeout(ctx, cfg.Timeout)
|
||||
defer cancel()
|
||||
|
||||
p.Server = server
|
||||
p.OpID = opID
|
||||
p.WorldPVC = naming.WorldPVCName(server)
|
||||
p.Namespace = cfg.Namespace
|
||||
p.ServiceAccount = cfg.ServiceAccount
|
||||
p.Image = cfg.Image
|
||||
p.WorldsRoot = cfg.WorldsRoot
|
||||
p.Deadline = cfg.Deadline
|
||||
p.CPULimit = cfg.CPULimit
|
||||
p.MemLimit = cfg.MemLimit
|
||||
p.RunAsUser = cfg.RunAsUser
|
||||
p.RunAsGroup = cfg.RunAsGroup
|
||||
p.FSGroup = cfg.FSGroup
|
||||
p.TTLAfterFinished = cfg.TTLAfterFinished
|
||||
payload, err := e.Runner.Run(ctx, p)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
@@ -357,95 +327,6 @@ func (e *Editor) run(ctx context.Context, server string, p JobParams) (Result, e
|
||||
return res, resultError(res)
|
||||
}
|
||||
|
||||
// params fills in what every Job of server takes from the Config, and a fresh op
|
||||
// id; p carries the op and its own fields.
|
||||
func (c Config) params(server string, p JobParams) (JobParams, error) {
|
||||
opID, err := newOpID()
|
||||
if err != nil {
|
||||
return JobParams{}, err
|
||||
}
|
||||
p.Server = server
|
||||
p.OpID = opID
|
||||
p.WorldPVC = naming.WorldPVCName(server)
|
||||
p.Namespace = c.Namespace
|
||||
p.ServiceAccount = c.ServiceAccount
|
||||
p.Image = c.Image
|
||||
p.WorldsRoot = c.WorldsRoot
|
||||
p.Deadline = c.Deadline
|
||||
p.CPULimit = c.CPULimit
|
||||
p.MemLimit = c.MemLimit
|
||||
p.RunAsUser = c.RunAsUser
|
||||
p.RunAsGroup = c.RunAsGroup
|
||||
p.FSGroup = c.FSGroup
|
||||
p.TTLAfterFinished = c.TTLAfterFinished
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// The states of an OpState.
|
||||
const (
|
||||
OpRunning = "running"
|
||||
OpSucceeded = "succeeded"
|
||||
OpFailed = "failed"
|
||||
)
|
||||
|
||||
// OpState is where one background file operation stands.
|
||||
type OpState struct {
|
||||
ID string
|
||||
Op string
|
||||
Path string
|
||||
State string
|
||||
// Started is when the Job was created; Finished when it ended, zero while it
|
||||
// runs.
|
||||
Started time.Time
|
||||
Finished time.Time
|
||||
// Done and Total are the bytes of the latest progress line, zero before the
|
||||
// first.
|
||||
Done, Total int64
|
||||
// Result is what the Job printed once it finished. It is nil while the Job
|
||||
// runs, and for a Job that ended without printing one (killed at its
|
||||
// deadline, out of memory, its bytes unfetchable), whose Reason says why
|
||||
// (ReasonOOMKilled for memory).
|
||||
Result *Result
|
||||
Reason string
|
||||
}
|
||||
|
||||
// StartUpload starts landing the staged bytes src describes at path and returns
|
||||
// without waiting, for a file too big to land inside one request (Upload). The
|
||||
// Job checks what Upload's does; Ops reports how it ends.
|
||||
func (e *Editor) StartUpload(ctx context.Context, server, path string, src UploadSource, overwrite bool) (OpState, error) {
|
||||
return e.start(ctx, server, JobParams{
|
||||
Op: OpUpload, Path: path, Overwrite: overwrite,
|
||||
SourceURL: src.URL, UploadToken: src.Token, UploadSize: src.Size, UploadSHA256: src.SHA256,
|
||||
})
|
||||
}
|
||||
|
||||
// StartUnzip starts extracting the .zip at path into the folder holding it and
|
||||
// returns without waiting. Without overwrite an archive that would replace a
|
||||
// file changes nothing and ends with CodeExists and the list (Result.Conflicts).
|
||||
func (e *Editor) StartUnzip(ctx context.Context, server, path string, overwrite bool) (OpState, error) {
|
||||
return e.start(ctx, server, JobParams{Op: OpUnzip, Path: path, Overwrite: overwrite})
|
||||
}
|
||||
|
||||
// Ops reports the server's background operations the cluster still holds: the
|
||||
// one running, if any, and those finished within AsyncTTL.
|
||||
func (e *Editor) Ops(ctx context.Context, server string) ([]OpState, error) {
|
||||
return e.Runner.Ops(ctx, e.Config.withDefaults().Namespace, server)
|
||||
}
|
||||
|
||||
func (e *Editor) start(ctx context.Context, server string, p JobParams) (OpState, error) {
|
||||
cfg := e.Config.withDefaults()
|
||||
p, err := cfg.params(server, p)
|
||||
if err != nil {
|
||||
return OpState{}, err
|
||||
}
|
||||
p.Async = true
|
||||
p.Deadline, p.TTLAfterFinished, p.CPULimit = cfg.AsyncDeadline, cfg.AsyncTTL, cfg.AsyncCPULimit
|
||||
if err := e.Runner.Start(ctx, p); err != nil {
|
||||
return OpState{}, err
|
||||
}
|
||||
return OpState{ID: p.OpID, Op: p.Op, Path: p.Path, State: OpRunning, Started: time.Now()}, nil
|
||||
}
|
||||
|
||||
// resultError translates a Result's code into the sentinel the API maps. An
|
||||
// unrecognised code is deliberately NOT swallowed as success: a Job reporting a
|
||||
// failure this build does not know about must still fail the request, or a future
|
||||
@@ -466,8 +347,6 @@ func resultError(res Result) error {
|
||||
return fmt.Errorf("%w: %s", ErrNoSpace, res.Error)
|
||||
case CodeExists:
|
||||
return fmt.Errorf("%w: %s", ErrExists, res.Error)
|
||||
case CodeDigestMismatch:
|
||||
return fmt.Errorf("%w: %s", ErrDigestMismatch, res.Error)
|
||||
default:
|
||||
return fmt.Errorf("fileedit: file operation failed (%s): %s", res.Code, res.Error)
|
||||
}
|
||||
|
||||
@@ -2,11 +2,9 @@ package fileedit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fakeRunner stands in for the cluster: it records the JobParams the Editor
|
||||
@@ -16,21 +14,6 @@ type fakeRunner struct {
|
||||
got []JobParams
|
||||
payload []byte
|
||||
err error
|
||||
|
||||
started []JobParams
|
||||
startErr error
|
||||
ops []OpState
|
||||
opsArgs [][2]string
|
||||
}
|
||||
|
||||
func (f *fakeRunner) Start(_ context.Context, p JobParams) error {
|
||||
f.started = append(f.started, p)
|
||||
return f.startErr
|
||||
}
|
||||
|
||||
func (f *fakeRunner) Ops(_ context.Context, namespace, server string) ([]OpState, error) {
|
||||
f.opsArgs = append(f.opsArgs, [2]string{namespace, server})
|
||||
return f.ops, f.err
|
||||
}
|
||||
|
||||
func (f *fakeRunner) Run(_ context.Context, p JobParams) ([]byte, error) {
|
||||
@@ -54,15 +37,15 @@ func mustPayload(t *testing.T, res Result) []byte {
|
||||
// the editor pointed at the same volume the operator created and the reaper deletes.
|
||||
func TestEditorRendersParams(t *testing.T) {
|
||||
t.Run("list", func(t *testing.T) {
|
||||
r := &fakeRunner{payload: mustPayload(t, Result{Entries: []Entry{{Name: "a"}}, Avail: 7 << 30})}
|
||||
r := &fakeRunner{payload: mustPayload(t, Result{Entries: []Entry{{Name: "a"}}})}
|
||||
e := &Editor{Runner: r, Config: Config{Image: "img"}}
|
||||
|
||||
ls, err := e.List(context.Background(), "survival", "config")
|
||||
entries, truncated, err := e.List(context.Background(), "survival", "config")
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
if len(ls.Entries) != 1 || ls.Truncated || ls.Free != 7<<30 {
|
||||
t.Fatalf("listing = %+v", ls)
|
||||
if len(entries) != 1 || truncated {
|
||||
t.Fatalf("entries=%+v truncated=%v", entries, truncated)
|
||||
}
|
||||
p := r.got[0]
|
||||
if p.Op != OpList || p.Path != "config" || p.Server != "survival" {
|
||||
@@ -80,7 +63,7 @@ func TestEditorRendersParams(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("read", func(t *testing.T) {
|
||||
r := &fakeRunner{payload: mustPayload(t, Result{Content: []byte("motd=hi\n"), SHA256: "abc", ContentSHA256: hex.EncodeToString(sumOf("motd=hi\n"))})}
|
||||
r := &fakeRunner{payload: mustPayload(t, Result{Content: []byte("motd=hi\n"), SHA256: "abc"})}
|
||||
e := &Editor{Runner: r, Config: Config{Image: "img"}}
|
||||
|
||||
got, sum, err := e.Read(context.Background(), "survival", "server.properties")
|
||||
@@ -163,7 +146,7 @@ func TestEditorRendersParams(t *testing.T) {
|
||||
// every time. If it ever cached one, two operations would collide on a name
|
||||
// felis-api has no permission to delete.
|
||||
func TestEditorMintsAFreshOpID(t *testing.T) {
|
||||
r := &fakeRunner{payload: mustPayload(t, Result{ContentSHA256: hex.EncodeToString(sumOf(""))})}
|
||||
r := &fakeRunner{payload: mustPayload(t, Result{})}
|
||||
e := &Editor{Runner: r, Config: Config{Image: "img"}}
|
||||
|
||||
for range 3 {
|
||||
@@ -199,7 +182,6 @@ func TestEditorMapsResultCodes(t *testing.T) {
|
||||
{"changed since read", CodeConflict, ErrConflict},
|
||||
{"volume full", CodeNoSpace, ErrNoSpace},
|
||||
{"already there", CodeExists, ErrExists},
|
||||
{"changed on the way", CodeDigestMismatch, ErrDigestMismatch},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -253,40 +235,18 @@ func TestEditorRefusesOversizedWriteBeforeTheCluster(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A read whose bytes do not hash to the digest the Job sent with them changed
|
||||
// on the way, and none of them is handed on: an editor saving a damaged read
|
||||
// would write the damage back.
|
||||
func TestEditorReadRefusesBytesChangedOnTheWay(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
sent string
|
||||
}{
|
||||
{"hashed otherwise", hex.EncodeToString(sumOf("motd=hi\n"))},
|
||||
{"with no digest", ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r := &fakeRunner{payload: mustPayload(t, Result{Content: []byte("motd=ho\n"), SHA256: "abc", ContentSHA256: tc.sent})}
|
||||
e := &Editor{Runner: r, Config: Config{Image: "img"}}
|
||||
got, sum, err := e.Read(context.Background(), "survival", "server.properties")
|
||||
if !errors.Is(err, ErrReadDamaged) || got != nil || sum != "" {
|
||||
t.Fatalf("Read = %q, %q, %v; want nothing and ErrReadDamaged", got, sum, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEditorNormalisesEmptyResults pins that "nothing there" is a success, not a
|
||||
// nil surprise: an empty directory lists as [] and a zero-length file reads as
|
||||
// empty bytes, so no caller has to distinguish nil from empty.
|
||||
func TestEditorNormalisesEmptyResults(t *testing.T) {
|
||||
r := &fakeRunner{payload: mustPayload(t, Result{ContentSHA256: hex.EncodeToString(sumOf(""))})}
|
||||
r := &fakeRunner{payload: mustPayload(t, Result{})}
|
||||
e := &Editor{Runner: r, Config: Config{Image: "img"}}
|
||||
|
||||
ls, err := e.List(context.Background(), "survival", "empty")
|
||||
entries, _, err := e.List(context.Background(), "survival", "empty")
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
if ls.Entries == nil {
|
||||
if entries == nil {
|
||||
t.Fatal("an empty directory must list as [], not nil")
|
||||
}
|
||||
|
||||
@@ -337,85 +297,3 @@ func TestExtractResult(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestEditorStartsBackgroundOps checks an upload or unzip too long to wait on is
|
||||
// started, not run: its Job carries the async label and the longer deadline,
|
||||
// the longer TTL Ops reads it back within, and the larger CPU share, and what
|
||||
// comes back names the Job Ops will report on.
|
||||
func TestEditorStartsBackgroundOps(t *testing.T) {
|
||||
src := UploadSource{URL: "http://api/big", Token: "tok", Size: 5 << 30, SHA256: "sum"}
|
||||
|
||||
t.Run("upload", func(t *testing.T) {
|
||||
r := &fakeRunner{}
|
||||
e := &Editor{Runner: r, Config: Config{Image: "img", Namespace: "mc"}}
|
||||
before := time.Now()
|
||||
st, err := e.StartUpload(context.Background(), "survival", "maps/world.zip", src, true)
|
||||
if err != nil {
|
||||
t.Fatalf("StartUpload: %v", err)
|
||||
}
|
||||
if r.calls != 0 || len(r.started) != 1 {
|
||||
t.Fatalf("ran %d, started %d; want the one Job started and none waited on", r.calls, len(r.started))
|
||||
}
|
||||
p := r.started[0]
|
||||
if !p.Async || p.Op != OpUpload || p.Path != "maps/world.zip" || !p.Overwrite ||
|
||||
p.SourceURL != src.URL || p.UploadToken != "tok" || p.UploadSize != 5<<30 || p.UploadSHA256 != "sum" {
|
||||
t.Fatalf("params = %+v", p)
|
||||
}
|
||||
if p.Deadline != 2*time.Hour || p.TTLAfterFinished != 30*time.Minute || p.CPULimit != "1" || p.MemLimit != "256Mi" {
|
||||
t.Fatalf("deadline %v ttl %v cpu %q mem %q, want 2h 30m 1 256Mi",
|
||||
p.Deadline, p.TTLAfterFinished, p.CPULimit, p.MemLimit)
|
||||
}
|
||||
if p.Namespace != "mc" || p.Server != "survival" || p.WorldPVC != "world-survival-0" || p.Image != "img" || p.OpID == "" {
|
||||
t.Fatalf("params = %+v", p)
|
||||
}
|
||||
if st.ID != p.OpID || st.Op != OpUpload || st.Path != "maps/world.zip" || st.State != OpRunning || st.Started.Before(before) {
|
||||
t.Fatalf("state = %+v, want the started Job %s running", st, p.OpID)
|
||||
}
|
||||
|
||||
if _, err := e.StartUpload(context.Background(), "survival", "maps/world.zip", src, false); err != nil || r.started[1].Overwrite {
|
||||
t.Fatalf("an upload that must not replace a file started with %+v (%v)", r.started[1], err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unzip, with its own limits", func(t *testing.T) {
|
||||
r := &fakeRunner{}
|
||||
e := &Editor{Runner: r, Config: Config{Image: "img", AsyncDeadline: time.Hour, AsyncTTL: time.Minute, AsyncCPULimit: "2"}}
|
||||
st, err := e.StartUnzip(context.Background(), "survival", "maps/world.zip", false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartUnzip: %v", err)
|
||||
}
|
||||
p := r.started[0]
|
||||
if !p.Async || p.Op != OpUnzip || p.Path != "maps/world.zip" || p.Overwrite || p.SourceURL != "" {
|
||||
t.Fatalf("params = %+v", p)
|
||||
}
|
||||
if p.Deadline != time.Hour || p.TTLAfterFinished != time.Minute || p.CPULimit != "2" {
|
||||
t.Fatalf("deadline %v ttl %v cpu %q, want the configured 1h 1m 2", p.Deadline, p.TTLAfterFinished, p.CPULimit)
|
||||
}
|
||||
if st.ID != p.OpID || st.Op != OpUnzip {
|
||||
t.Fatalf("state = %+v", st)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a Job that could not be created", func(t *testing.T) {
|
||||
boom := errors.New("forbidden")
|
||||
r := &fakeRunner{startErr: boom}
|
||||
e := &Editor{Runner: r, Config: Config{Image: "img"}}
|
||||
st, err := e.StartUnzip(context.Background(), "survival", "a.zip", false)
|
||||
if !errors.Is(err, boom) || st != (OpState{}) {
|
||||
t.Fatalf("state %+v err %v, want nothing started and %v", st, err, boom)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ops", func(t *testing.T) {
|
||||
want := []OpState{{ID: "0a", State: OpRunning}}
|
||||
r := &fakeRunner{ops: want}
|
||||
e := &Editor{Runner: r, Config: Config{Image: "img"}}
|
||||
got, err := e.Ops(context.Background(), "survival")
|
||||
if err != nil || len(got) != 1 || got[0] != want[0] {
|
||||
t.Fatalf("Ops = %+v %v", got, err)
|
||||
}
|
||||
if r.opsArgs[0] != [2]string{"minecraft", "survival"} {
|
||||
t.Fatalf("asked %v, want the default namespace and the server", r.opsArgs[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
+36
-124
@@ -10,7 +10,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
@@ -21,12 +20,11 @@ import (
|
||||
)
|
||||
|
||||
// The operations the editor supports: list a directory, read a file, write a
|
||||
// file, make a directory, delete, rename, upload, and extract a .zip. The set is
|
||||
// closed; there is no chmod, chown, link or copy. Every op resolves every path
|
||||
// through os.Root (see Execute), and each mutating op carries its own
|
||||
// containment note below.
|
||||
// file, make a directory, delete, rename, and upload. The set is closed; there is
|
||||
// no chmod, chown, link or copy. Every op resolves every path through os.Root (see
|
||||
// Execute), and each mutating op carries its own containment note below.
|
||||
//
|
||||
// A write, upload or unzip DOES land arbitrary bytes at any path inside the mount, and
|
||||
// A write or upload DOES land arbitrary bytes at any path inside the mount, and
|
||||
// that is a real capability rather than an oversight: the root is the server's
|
||||
// whole working directory (see Config.WorldsRoot), so an owner can upload
|
||||
// plugins/<x>.jar and Paper will load it on the next boot. It is the same power a
|
||||
@@ -35,8 +33,8 @@ import (
|
||||
// image curation. Images are admin-only (POST /images, POST /images/build) and
|
||||
// modpack submissions need an admin verdict, so this is the one owner-tier route
|
||||
// that lands executable code in a backend pod. That trade was made deliberately;
|
||||
// if it is ever revisited, the guard belongs in land() below, the choke point
|
||||
// write and upload route through, and in unzip's extractOne (unzip.go).
|
||||
// if it is ever revisited, the guard belongs in land() below, which is the single
|
||||
// choke point both byte-landing ops route through.
|
||||
const (
|
||||
OpList = "list"
|
||||
OpRead = "read"
|
||||
@@ -45,7 +43,6 @@ const (
|
||||
OpDelete = "delete"
|
||||
OpRename = "rename"
|
||||
OpUpload = "upload"
|
||||
OpUnzip = "unzip"
|
||||
)
|
||||
|
||||
// mutates reports whether op changes the world, and so whether its Job gets the
|
||||
@@ -57,7 +54,7 @@ func mutates(op string) bool { return op != OpList && op != OpRead }
|
||||
// validOp reports whether op is one the Job knows.
|
||||
func validOp(op string) bool {
|
||||
switch op {
|
||||
case OpList, OpRead, OpWrite, OpMkdir, OpDelete, OpRename, OpUpload, OpUnzip:
|
||||
case OpList, OpRead, OpWrite, OpMkdir, OpDelete, OpRename, OpUpload:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -84,10 +81,6 @@ const (
|
||||
// there. None of them replaces anything unless told to (an upload's
|
||||
// Overwrite), so a name collision is reported rather than resolved.
|
||||
CodeExists = "exists"
|
||||
// CodeDigestMismatch is a write whose bytes do not hash to the SHA-256
|
||||
// felis-api computed over them (Request.ContentSHA256): they changed on the
|
||||
// way to the Job, and nothing was written.
|
||||
CodeDigestMismatch = "digest_mismatch"
|
||||
)
|
||||
|
||||
// ResultPrefix marks the single stdout line carrying the JSON Result. The Job's
|
||||
@@ -140,12 +133,12 @@ const UploadTokenEnv = "FELIS_UPLOAD_TOKEN"
|
||||
// - MaxEntries bounds a listing. A world's region/ directory legitimately holds
|
||||
// thousands of .mca files, so this truncates rather than errors (Truncated
|
||||
// says so), keeping the log line bounded while still being useful.
|
||||
// - MaxUploadBytes bounds an upload sent as ONE request body: the Cloudflare
|
||||
// edge refuses bodies over 100 MB on the Free and Pro plans, and 64 MiB
|
||||
// covers the largest plugin jars (a Geyser build is about 20 MiB) with room
|
||||
// to spare. It is felis-api's bound on that route only. A bigger file arrives
|
||||
// in parts and is bounded by nothing but the room on the server's volume,
|
||||
// which the Job checks before it fetches a byte (upload).
|
||||
// - MaxUploadBytes bounds an upload. Its bytes travel neither through the Job
|
||||
// spec nor the pod log — felis-api stages them and the Job fetches them — so
|
||||
// the bound is the request body instead: the Cloudflare edge refuses bodies
|
||||
// over 100 MB on the Free and Pro plans, and 64 MiB covers the largest plugin
|
||||
// jars (a Geyser build is about 20 MiB) with room to spare. A whole world is
|
||||
// a different operation (a restore), not an upload.
|
||||
const (
|
||||
MaxWriteBytes = 256 << 10 // 256 KiB
|
||||
MaxReadBytes = 1 << 20 // 1 MiB
|
||||
@@ -190,27 +183,6 @@ type Result struct {
|
||||
// conflict, the file as it is now. A client hands it back as the expected
|
||||
// hash of its next write (see write).
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
// ContentSHA256 is, after a read, the hex digest of Content as handed out
|
||||
// (after any redaction), so felis-api can tell the bytes it got from the
|
||||
// bytes this Job sent (Editor.Read).
|
||||
ContentSHA256 string `json:"content_sha256,omitempty"`
|
||||
|
||||
// Conflicts lists, relative to the root and sorted, the existing files an
|
||||
// unzip would replace: the first of them, up to MaxConflicts and 8 KiB of
|
||||
// names (see maxConflictBytes). ConflictCount is how many there are in all.
|
||||
Conflicts []string `json:"conflicts,omitempty"`
|
||||
ConflictCount int `json:"conflict_count,omitempty"`
|
||||
// Entry names what an unzip refused: the archive entry, or the path on the
|
||||
// server it collides with.
|
||||
Entry string `json:"entry,omitempty"`
|
||||
// Need and Avail are, on a no_space an upload or unzip saw coming, the bytes
|
||||
// it needs and the bytes the volume has free. A listing sets Avail too, to
|
||||
// -1 when it could not read it.
|
||||
Need int64 `json:"need,omitempty"`
|
||||
Avail int64 `json:"avail,omitempty"`
|
||||
// Files and Bytes are what a successful unzip extracted.
|
||||
Files int `json:"files,omitempty"`
|
||||
Bytes int64 `json:"bytes,omitempty"`
|
||||
}
|
||||
|
||||
// Request is one file operation. Op decides which of the other fields it reads.
|
||||
@@ -221,24 +193,17 @@ type Request struct {
|
||||
To string
|
||||
// Content and Expect are a write's bytes and precondition: when Expect is
|
||||
// non-empty, the write lands only if the file's current SHA-256 (hex) equals
|
||||
// it. ContentSHA256, when set, is the SHA-256 (hex) felis-api computed over
|
||||
// Content; bytes that hash otherwise are not written (CodeDigestMismatch).
|
||||
// it.
|
||||
Content []byte
|
||||
Expect string
|
||||
ContentSHA256 string
|
||||
// CreateOnly makes a write refuse a path that already exists. It is the
|
||||
// panel's "new file", which must never truncate a file it did not know was
|
||||
// there.
|
||||
CreateOnly bool
|
||||
// Upload is where an upload's bytes come from. Overwrite lets an upload
|
||||
// replace a file already at the path, and an unzip replace the files it
|
||||
// collides with.
|
||||
// Upload is where an upload's bytes come from; Overwrite lets it replace a
|
||||
// file already at the path.
|
||||
Upload *Upload
|
||||
Overwrite bool
|
||||
// Progress, when set, hears how far an upload or unzip has got: bytes landed
|
||||
// so far out of the total. It is called from the copy loop, often; the caller
|
||||
// throttles.
|
||||
Progress func(done, total int64)
|
||||
}
|
||||
|
||||
// Upload describes the bytes an upload lands. Size and SHA256 are what felis-api
|
||||
@@ -250,10 +215,6 @@ type Upload struct {
|
||||
// Open starts the transfer. It runs only once the target has passed every
|
||||
// check, so a refused upload never pulls the bytes.
|
||||
Open func() (io.ReadCloser, error)
|
||||
// Landed, when set, runs once the bytes are in place, so felis-api can let
|
||||
// go of the copy it staged (Stage.Landed). Until then felis-api keeps it,
|
||||
// and a failed landing is started again without the bytes being sent again.
|
||||
Landed func()
|
||||
}
|
||||
|
||||
// Execute performs one operation inside root and returns the Result to print.
|
||||
@@ -301,11 +262,11 @@ func Execute(root string, req Request) (Result, error) {
|
||||
|
||||
switch req.Op {
|
||||
case OpList:
|
||||
return list(r, root, path), nil
|
||||
return list(r, path), nil
|
||||
case OpRead:
|
||||
return read(r, path), nil
|
||||
case OpWrite:
|
||||
return write(r, path, req.Content, req.ContentSHA256, req.Expect, req.CreateOnly), nil
|
||||
return write(r, path, req.Content, req.Expect, req.CreateOnly), nil
|
||||
case OpMkdir:
|
||||
return mkdir(r, path), nil
|
||||
case OpDelete:
|
||||
@@ -316,9 +277,7 @@ func Execute(root string, req Request) (Result, error) {
|
||||
if req.Upload == nil {
|
||||
return Result{}, errors.New("an upload needs a source")
|
||||
}
|
||||
return upload(r, root, path, *req.Upload, req.Overwrite, req.Progress)
|
||||
case OpUnzip:
|
||||
return unzip(r, root, path, req.Overwrite, req.Progress), nil
|
||||
return upload(r, path, *req.Upload, req.Overwrite)
|
||||
default:
|
||||
return Result{}, fmt.Errorf("unknown op %q", req.Op)
|
||||
}
|
||||
@@ -326,10 +285,8 @@ func Execute(root string, req Request) (Result, error) {
|
||||
|
||||
// list reads one directory. It does not recurse: a browser asks for one level at
|
||||
// a time, and recursion would make both the result size and the traversal cost
|
||||
// unbounded in a world directory. It also reports the room left on the volume
|
||||
// (Avail), so the panel can refuse an upload the volume cannot take before
|
||||
// sending a byte of it.
|
||||
func list(r *os.Root, rootPath, path string) Result {
|
||||
// unbounded in a world directory.
|
||||
func list(r *os.Root, path string) Result {
|
||||
f, err := r.Open(path)
|
||||
if err != nil {
|
||||
return failure(err, path)
|
||||
@@ -361,13 +318,7 @@ func list(r *os.Root, rootPath, path string) Result {
|
||||
}
|
||||
entries = append(entries, e)
|
||||
}
|
||||
// A full volume is Avail 0, which the JSON leaves out; -1 is a volume whose
|
||||
// free space could not be read, so the two stay apart.
|
||||
res := Result{Entries: entries, Truncated: truncated, Avail: -1}
|
||||
if avail, _, err := statfs(rootPath); err == nil {
|
||||
res.Avail = int64(min(avail, math.MaxInt64))
|
||||
}
|
||||
return res
|
||||
return Result{Entries: entries, Truncated: truncated}
|
||||
}
|
||||
|
||||
// secretConfigPath is the one file in a world mount holding PLATFORM secret
|
||||
@@ -419,13 +370,6 @@ func read(r *os.Root, name string) Result {
|
||||
if info.IsDir() {
|
||||
return Result{Code: CodeBadPath, Error: fmt.Sprintf("%s is a directory, not a file", name)}
|
||||
}
|
||||
// The name check above answers the plain path with a clear reason; this one
|
||||
// catches the same file reached through a link (see Guard).
|
||||
withhold, redact := NewGuard(r).Rule(info)
|
||||
if withhold {
|
||||
return Result{Code: CodeBadPath, Error: fmt.Sprintf(
|
||||
"%s is the file holding the proxy forwarding secret, which is shared cluster-wide, and is not readable through the editor", name)}
|
||||
}
|
||||
if info.Size() > MaxReadBytes {
|
||||
return Result{Code: CodeTooLarge, Error: fmt.Sprintf(
|
||||
"%s is %d bytes; the editor reads at most %d", name, info.Size(), MaxReadBytes)}
|
||||
@@ -438,11 +382,7 @@ func read(r *os.Root, name string) Result {
|
||||
if err != nil {
|
||||
return failure(err, name)
|
||||
}
|
||||
content := b
|
||||
if redact {
|
||||
content = RedactProps(b)
|
||||
}
|
||||
return Result{Content: content, SHA256: digest(b), ContentSHA256: digest(content)}
|
||||
return Result{Content: redactSecretProps(name, b), SHA256: digest(b)}
|
||||
}
|
||||
|
||||
// propsPath is the server's main config file, and rconPasswordKey the one line in
|
||||
@@ -502,17 +442,7 @@ func redactSecretProps(name string, content []byte) []byte {
|
||||
// being overwritten, which is how two people editing the same file find out.
|
||||
// The world lock (internal/maintenance) already serialises writes, so the check
|
||||
// and the rename cannot interleave with another write.
|
||||
//
|
||||
// sum, when set, is the SHA-256 felis-api computed over content before handing
|
||||
// it to the Job: content that hashes otherwise changed on the way, and is
|
||||
// refused with CodeDigestMismatch before anything is touched.
|
||||
func write(r *os.Root, name string, content []byte, sum, expect string, createOnly bool) Result {
|
||||
if sum != "" {
|
||||
if got := digest(content); got != sum {
|
||||
return Result{Code: CodeDigestMismatch, Error: fmt.Sprintf(
|
||||
"the content hashes to %s and was sent as %s; nothing was written", got, sum)}
|
||||
}
|
||||
}
|
||||
func write(r *os.Root, name string, content []byte, expect string, createOnly bool) Result {
|
||||
if len(content) > MaxWriteBytes {
|
||||
// Defence in depth: felis-api already refuses an oversized write with a 413
|
||||
// before rendering the Job. Re-checking here keeps the ceiling true even if
|
||||
@@ -584,13 +514,9 @@ type transferError struct{ err error }
|
||||
func (e *transferError) Error() string { return "fetch upload: " + e.err.Error() }
|
||||
func (e *transferError) Unwrap() error { return e.err }
|
||||
|
||||
// NameMax is the longest name a folder entry can have on the volumes a world
|
||||
// lives on (NAME_MAX).
|
||||
const NameMax = 255
|
||||
|
||||
// land atomically puts the bytes fill writes at target, the path landingTarget
|
||||
// returned for name. Write and upload both land through it; unzip lands a whole
|
||||
// tree at once and has its own path (unzip.go).
|
||||
// returned for name. It is the single choke point both byte-landing ops (write and
|
||||
// upload) route through.
|
||||
//
|
||||
// The bytes go to a temporary sibling that is synced and then renamed over the
|
||||
// target, so a full disk, a Job killed at its deadline or a crashed node leaves
|
||||
@@ -598,9 +524,8 @@ const NameMax = 255
|
||||
// server.properties an in-place truncate would, which is a server that no longer
|
||||
// boots. The sibling gets mode and is handed to the game uid before the rename, so
|
||||
// the file the server finds is never root's. On failure it is removed; only a kill
|
||||
// between create and rename leaves one behind, named ".felis-edit-<hex>" so no
|
||||
// loader mistakes it for a plugin jar or a config. The name is its own rather than
|
||||
// the target's with a suffix, so a target named up to NameMax bytes can be written.
|
||||
// between create and rename leaves one behind, named ".<file>.felis-edit-<hex>" so
|
||||
// no loader mistakes it for a plugin jar or a config.
|
||||
//
|
||||
// A *transferError from fill comes back as the error; every other failure is a
|
||||
// Result.
|
||||
@@ -609,7 +534,7 @@ func land(r *os.Root, name, target string, mode fs.FileMode, fill func(io.Writer
|
||||
if _, err := rand.Read(suffix[:]); err != nil {
|
||||
return Result{Code: CodeBadPath, Error: fmt.Sprintf("generate a temporary name: %v", err)}, nil
|
||||
}
|
||||
tmp := path.Join(path.Dir(target), ".felis-edit-"+hex.EncodeToString(suffix[:]))
|
||||
tmp := path.Join(path.Dir(target), "."+path.Base(target)+".felis-edit-"+hex.EncodeToString(suffix[:]))
|
||||
f, err := r.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)
|
||||
if err != nil {
|
||||
return writeFailure(err, name), nil
|
||||
@@ -671,33 +596,24 @@ func syncDir(r *os.Root, dir string) {
|
||||
// The fetched bytes must match both the size and the SHA-256 felis-api received;
|
||||
// either mismatch is a broken transfer, and the target is left as it was. A
|
||||
// success has landed exactly what felis-api staged, whose digest it already holds.
|
||||
//
|
||||
// There is no size ceiling here. The volume is the bound, and it is checked up
|
||||
// front: the new bytes land beside the file they replace until the rename, so
|
||||
// they need their whole size free whatever is already at the path.
|
||||
func upload(r *os.Root, rootPath, name string, u Upload, overwrite bool, progress func(done, total int64)) (Result, error) {
|
||||
func upload(r *os.Root, name string, u Upload, overwrite bool) (Result, error) {
|
||||
if u.Size > MaxUploadBytes {
|
||||
return Result{Code: CodeTooLarge, Error: fmt.Sprintf(
|
||||
"the upload is %d bytes; the editor uploads at most %d", u.Size, MaxUploadBytes)}, nil
|
||||
}
|
||||
target, mode, res := landingTarget(r, name, overwrite)
|
||||
if res.Code != "" {
|
||||
return res, nil
|
||||
}
|
||||
if avail, _, err := statfs(rootPath); err == nil && uint64(u.Size) > avail {
|
||||
return Result{Code: CodeNoSpace, Need: u.Size, Avail: int64(min(avail, math.MaxInt64)), Error: fmt.Sprintf(
|
||||
"%s is %d bytes and the server's volume has %d free; nothing was changed", name, u.Size, avail)}, nil
|
||||
}
|
||||
res, err := land(r, name, target, mode, func(w io.Writer) error {
|
||||
return land(r, name, target, mode, func(w io.Writer) error {
|
||||
body, err := u.Open()
|
||||
if err != nil {
|
||||
return &transferError{err}
|
||||
}
|
||||
defer body.Close()
|
||||
h := sha256.New()
|
||||
var done int64
|
||||
out := io.MultiWriter(w, h)
|
||||
if progress != nil {
|
||||
out = countingWriter{out, func(n int) { done += int64(n); progress(done, u.Size) }}
|
||||
}
|
||||
// One byte past Size so a source that sends more than it promised is seen.
|
||||
n, err := io.Copy(out, sourceReader{io.LimitReader(body, u.Size+1)})
|
||||
n, err := io.Copy(io.MultiWriter(w, h), sourceReader{io.LimitReader(body, u.Size+1)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -709,10 +625,6 @@ func upload(r *os.Root, rootPath, name string, u Upload, overwrite bool, progres
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err == nil && res.Code == "" && u.Landed != nil {
|
||||
u.Landed()
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
|
||||
// sourceReader tags the source's read errors as transfer errors, so land can tell
|
||||
|
||||
@@ -2,11 +2,7 @@ package fileedit
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -180,36 +176,6 @@ func TestExecuteHappyPath(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a listing reports the room left on the volume", func(t *testing.T) {
|
||||
prev := statfs
|
||||
t.Cleanup(func() { statfs = prev })
|
||||
var asked string
|
||||
statfs = func(dir string) (uint64, uint64, error) { asked = dir; return 12345, 99999, nil }
|
||||
res, err := run(root, OpList, "config", nil, "")
|
||||
if err != nil || res.Code != "" {
|
||||
t.Fatalf("Execute: %v %+v", err, res)
|
||||
}
|
||||
if res.Avail != 12345 || asked != root {
|
||||
t.Fatalf("avail = %d measured at %q, want 12345 at %q", res.Avail, asked, root)
|
||||
}
|
||||
|
||||
statfs = func(string) (uint64, uint64, error) { return math.MaxUint64, math.MaxUint64, nil }
|
||||
if res, _ := run(root, OpList, "config", nil, ""); res.Avail != math.MaxInt64 {
|
||||
t.Fatalf("avail = %d, want it clamped to %d", res.Avail, int64(math.MaxInt64))
|
||||
}
|
||||
|
||||
statfs = func(string) (uint64, uint64, error) { return 0, 99999, nil }
|
||||
if res, _ := run(root, OpList, "config", nil, ""); res.Avail != 0 {
|
||||
t.Fatalf("avail = %d on a full volume, want 0", res.Avail)
|
||||
}
|
||||
|
||||
statfs = func(string) (uint64, uint64, error) { return 1, 1, errors.New("no statfs") }
|
||||
res, err = run(root, OpList, "config", nil, "")
|
||||
if err != nil || res.Code != "" || len(res.Entries) != 1 || res.Avail != -1 {
|
||||
t.Fatalf("a volume that cannot be measured still lists, with its room -1 (unknown): %v %+v", err, res)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("read a file", func(t *testing.T) {
|
||||
res, err := run(root, OpRead, "server.properties", nil, "")
|
||||
if err != nil {
|
||||
@@ -244,7 +210,7 @@ func TestExecuteHappyPath(t *testing.T) {
|
||||
if res, err := run(root, OpWrite, "ops.json", []byte("[]"), ""); err != nil || res.Code != "" {
|
||||
t.Fatalf("creating a new file should succeed: %v / %+v", err, res)
|
||||
}
|
||||
if len(owned) != 1 || !strings.HasPrefix(owned[0], ".felis-edit-") {
|
||||
if len(owned) != 1 || !strings.HasPrefix(owned[0], ".ops.json.felis-edit-") {
|
||||
t.Errorf("files handed to the game uid = %v, want the one temporary sibling of ops.json", owned)
|
||||
}
|
||||
res, err := run(root, OpWrite, "nope/deep.txt", []byte("x"), "")
|
||||
@@ -430,16 +396,6 @@ func TestReadRedactsRconPassword(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
// The hash is the file on disk, the one a save's expect_sha256 is checked
|
||||
// against, not the redacted copy the editor shows.
|
||||
if sum := sha256.Sum256([]byte(props)); res.SHA256 != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("sha256 = %s, want the hash of the file as stored", res.SHA256)
|
||||
}
|
||||
// The content digest is of the copy handed out, so the bytes that arrive
|
||||
// can be checked against it.
|
||||
if sum := sha256.Sum256(res.Content); res.ContentSHA256 != hex.EncodeToString(sum[:]) || res.ContentSHA256 == res.SHA256 {
|
||||
t.Fatalf("content_sha256 = %s, want the hash of the redacted copy (%x), apart from sha256 %s", res.ContentSHA256, sum, res.SHA256)
|
||||
}
|
||||
got := string(res.Content)
|
||||
if strings.Contains(got, "hunter2") {
|
||||
t.Fatalf("read returned the RCON password (spec §286):\n%s", got)
|
||||
@@ -464,51 +420,6 @@ func TestReadRedactsRconPassword(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadGuardsLinksToGuardedFiles: a plugin runs as the game uid and can leave
|
||||
// a link to either guarded file anywhere in the world. Read under the link's
|
||||
// name, the forwarding secret is still refused and the RCON password still
|
||||
// redacted, whether the link is symbolic, a hard link, or a linked folder.
|
||||
func TestReadGuardsLinksToGuardedFiles(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
const secret = "secret: aVeryRealForwardingKey"
|
||||
if err := os.WriteFile(filepath.Join(root, "config", "paper-global.yml"), []byte(secret), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "server.properties"), []byte("motd=hi\nrcon.password=hunter2\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
symlink(t, "config/paper-global.yml", filepath.Join(root, "sym.yml"))
|
||||
symlink(t, "config", filepath.Join(root, "cfg-link"))
|
||||
symlink(t, "server.properties", filepath.Join(root, "sym.properties"))
|
||||
for _, l := range [][2]string{
|
||||
{"config/paper-global.yml", "hard.yml"},
|
||||
{"server.properties", "hard.properties"},
|
||||
} {
|
||||
if err := os.Link(filepath.Join(root, l[0]), filepath.Join(root, l[1])); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range []string{"sym.yml", "cfg-link/paper-global.yml", "hard.yml"} {
|
||||
res, err := run(root, OpRead, name, nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if res.Code != CodeBadPath || strings.Contains(string(res.Content), "aVeryReal") {
|
||||
t.Errorf("%s: result = %+v, want bad_path and no secret", name, res)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"sym.properties", "hard.properties"} {
|
||||
res, err := run(root, OpRead, name, nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if want := "motd=hi\nrcon.password=" + redactedValue + "\n"; res.Code != "" || string(res.Content) != want {
|
||||
t.Errorf("%s: result = %+v, want content %q", name, res, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteIsAtomic is the durability contract: a write that fails part-way leaves
|
||||
// the original file byte-for-byte intact and no stray sibling behind, and a write
|
||||
// that succeeds keeps the file's mode.
|
||||
@@ -547,18 +458,6 @@ func TestWriteIsAtomic(t *testing.T) {
|
||||
assertNoTemporaries(t, root)
|
||||
})
|
||||
|
||||
t.Run("a name as long as a folder allows is written", func(t *testing.T) {
|
||||
long := strings.Repeat("n", NameMax-4) + ".yml"
|
||||
res, err := Execute(root, Request{Op: OpWrite, Path: "config/" + long, Content: []byte("a: 1\n"), CreateOnly: true})
|
||||
if err != nil || res.Code != "" {
|
||||
t.Fatalf("write a %d-byte name: %v / %+v", len(long), err, res)
|
||||
}
|
||||
if b, _ := os.ReadFile(filepath.Join(root, "config", long)); string(b) != "a: 1\n" {
|
||||
t.Fatalf("content = %q", b)
|
||||
}
|
||||
assertNoTemporaries(t, filepath.Join(root, "config"))
|
||||
})
|
||||
|
||||
t.Run("a link inside the root is written through, not replaced", func(t *testing.T) {
|
||||
if err := os.Symlink("config/paper.yml", filepath.Join(root, "paper-link.yml")); err != nil {
|
||||
t.Skipf("symlinks unavailable: %v", err)
|
||||
@@ -638,34 +537,6 @@ func TestWriteDetectsConcurrentChange(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteChecksTheContentDigest: a write lands only bytes that hash to the
|
||||
// SHA-256 felis-api sent with them; bytes changed on the way touch nothing.
|
||||
func TestWriteChecksTheContentDigest(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
props := filepath.Join(root, "server.properties")
|
||||
if err := os.WriteFile(props, []byte("motd=hello\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent := sha256.Sum256([]byte("motd=mine\n"))
|
||||
req := Request{Op: OpWrite, Path: "server.properties", Content: []byte("motd=mint\n"), ContentSHA256: hex.EncodeToString(sent[:])}
|
||||
res, err := Execute(root, req)
|
||||
if err != nil || res.Code != CodeDigestMismatch {
|
||||
t.Fatalf("changed write = %+v, %v; want %s", res, err, CodeDigestMismatch)
|
||||
}
|
||||
if b, _ := os.ReadFile(props); string(b) != "motd=hello\n" {
|
||||
t.Fatalf("a changed write replaced the file with %q", b)
|
||||
}
|
||||
assertNoTemporaries(t, root)
|
||||
|
||||
req.Content = []byte("motd=mine\n")
|
||||
if res, err := Execute(root, req); err != nil || res.Code != "" {
|
||||
t.Fatalf("write as sent = %+v, %v", res, err)
|
||||
}
|
||||
if b, _ := os.ReadFile(props); string(b) != "motd=mine\n" {
|
||||
t.Fatalf("on disk %q, want the bytes as sent", b)
|
||||
}
|
||||
}
|
||||
|
||||
func assertNoTemporaries(t *testing.T, dir string) {
|
||||
t.Helper()
|
||||
des, err := os.ReadDir(dir)
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
package fileedit
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
)
|
||||
|
||||
// What leaves a world mount — a read, a download, a world export, a backup
|
||||
// export — passes the same two guards: the forwarding-secret file
|
||||
// (secretConfigPath) is withheld, and server.properties has its RCON password
|
||||
// redacted (propsPath).
|
||||
//
|
||||
// On a live mount both are matched by the file itself (os.SameFile), not by the
|
||||
// name it was reached under. A plugin runs arbitrary code as the game uid and can
|
||||
// leave a symbolic or hard link to either file anywhere in the world; a name
|
||||
// check alone would hand the secret out under the link's name. A stored archive
|
||||
// has only names, so ArchiveRule matches those.
|
||||
|
||||
// Guard knows the two guarded files of one world mount.
|
||||
type Guard struct {
|
||||
secret, props fs.FileInfo
|
||||
}
|
||||
|
||||
// NewGuard looks the guarded files up in r. One that is missing guards nothing:
|
||||
// no file can be the same file as it.
|
||||
func NewGuard(r *os.Root) Guard {
|
||||
var g Guard
|
||||
if fi, err := r.Stat(secretConfigPath); err == nil {
|
||||
g.secret = fi
|
||||
}
|
||||
if fi, err := r.Stat(propsPath); err == nil {
|
||||
g.props = fi
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// Rule reports whether the file fi describes must be withheld, or sent only
|
||||
// through RedactProps.
|
||||
func (g Guard) Rule(fi fs.FileInfo) (withhold, redact bool) {
|
||||
if g.secret != nil && os.SameFile(g.secret, fi) {
|
||||
return true, false
|
||||
}
|
||||
return false, g.props != nil && os.SameFile(g.props, fi)
|
||||
}
|
||||
|
||||
// ArchiveRule is Rule for an entry of a stored world archive, by its name
|
||||
// cleaned as a path, so "./server.properties" is server.properties too.
|
||||
func ArchiveRule(name string) (withhold, redact bool) {
|
||||
name = path.Clean(name)
|
||||
return name == secretConfigPath, name == propsPath
|
||||
}
|
||||
|
||||
// RedactProps replaces the RCON password in server.properties content with
|
||||
// redactedValue (see redactSecretProps for why a placeholder and not a blank).
|
||||
func RedactProps(content []byte) []byte {
|
||||
return redactSecretProps(propsPath, content)
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package fileedit
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestArchiveRule: a stored archive's entry is matched by its name as a path,
|
||||
// however the archive spelled it.
|
||||
func TestArchiveRule(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
withhold, redact bool
|
||||
}{
|
||||
{"config/paper-global.yml", true, false},
|
||||
{"./config/paper-global.yml", true, false},
|
||||
{"config//paper-global.yml", true, false},
|
||||
{"server.properties", false, true},
|
||||
{"./server.properties", false, true},
|
||||
{"plugins/server.properties", false, false},
|
||||
{"plugins/config/paper-global.yml", false, false},
|
||||
{"config/paper.yml", false, false},
|
||||
} {
|
||||
if w, r := ArchiveRule(c.name); w != c.withhold || r != c.redact {
|
||||
t.Errorf("ArchiveRule(%q) = %v, %v; want %v, %v", c.name, w, r, c.withhold, c.redact)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25,11 +25,6 @@ const (
|
||||
LabelServer = "felis.lolicon.best/server"
|
||||
LabelOpID = "felis.lolicon.best/files-op"
|
||||
LabelMode = "felis.lolicon.best/files-mode"
|
||||
// LabelAsync marks the Job of an upload or unzip felis-api started and does
|
||||
// not wait on (Editor.StartUpload, Editor.StartUnzip); Ops finds them by it.
|
||||
// AnnotationPath names the file such a Job works on, for Ops to show.
|
||||
LabelAsync = "felis.lolicon.best/files-async"
|
||||
AnnotationPath = "felis.lolicon.best/files-path"
|
||||
|
||||
managedByValue = "felis-files"
|
||||
componentValue = "world-files"
|
||||
@@ -64,10 +59,6 @@ type JobParams struct {
|
||||
UploadSize int64
|
||||
UploadSHA256 string
|
||||
Overwrite bool
|
||||
// Async marks a Job felis-api does not wait on: it carries LabelAsync and
|
||||
// AnnotationPath, which Ops reads it back by. Only an upload or an unzip
|
||||
// runs so.
|
||||
Async bool
|
||||
WorldPVC string
|
||||
|
||||
Namespace string
|
||||
@@ -101,17 +92,13 @@ type JobParams struct {
|
||||
func FilesJobName(server, opID string) string { return "files-" + server + "-" + opID }
|
||||
|
||||
func filesLabels(p JobParams) map[string]string {
|
||||
l := map[string]string{
|
||||
return map[string]string{
|
||||
LabelManagedBy: managedByValue,
|
||||
LabelComponent: componentValue,
|
||||
LabelServer: p.Server,
|
||||
LabelOpID: p.OpID,
|
||||
LabelMode: p.Op,
|
||||
}
|
||||
if p.Async {
|
||||
l[LabelAsync] = "true"
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// FilesJob renders the file-editor Job. Its isolation is the strictest of the three
|
||||
@@ -165,9 +152,6 @@ func FilesJob(p JobParams) (*batchv1.Job, error) {
|
||||
if p.Op == OpUpload && (p.SourceURL == "" || p.UploadToken == "") {
|
||||
return nil, fmt.Errorf("fileedit: an upload needs a source URL and a token")
|
||||
}
|
||||
if p.Async && p.Op != OpUpload && p.Op != OpUnzip {
|
||||
return nil, fmt.Errorf("fileedit: only an upload or an unzip runs in the background, not %s", p.Op)
|
||||
}
|
||||
limits, err := resourceLimits(p.CPULimit, p.MemLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -195,9 +179,6 @@ func FilesJob(p JobParams) (*batchv1.Job, error) {
|
||||
// secret, so it rides argv; only the content itself needs the env channel.
|
||||
switch p.Op {
|
||||
case OpWrite:
|
||||
// The content's own SHA-256 goes beside it, so the Job writes only the
|
||||
// bytes felis-api handed over (Request.ContentSHA256).
|
||||
args = append(args, "--sha256", digest(p.Content))
|
||||
if p.Expect != "" {
|
||||
args = append(args, "--expect-sha256", p.Expect)
|
||||
}
|
||||
@@ -214,10 +195,6 @@ func FilesJob(p JobParams) (*batchv1.Job, error) {
|
||||
if p.Overwrite {
|
||||
args = append(args, "--overwrite")
|
||||
}
|
||||
case OpUnzip:
|
||||
if p.Overwrite {
|
||||
args = append(args, "--overwrite")
|
||||
}
|
||||
}
|
||||
container := corev1.Container{
|
||||
Name: containerName,
|
||||
@@ -261,7 +238,6 @@ func FilesJob(p JobParams) (*batchv1.Job, error) {
|
||||
Name: FilesJobName(p.Server, p.OpID),
|
||||
Namespace: p.Namespace,
|
||||
Labels: filesLabels(p),
|
||||
Annotations: filesAnnotations(p),
|
||||
},
|
||||
Spec: batchv1.JobSpec{
|
||||
// One shot: a file operation that failed must surface its failure, not be
|
||||
@@ -321,12 +297,12 @@ func int64Ptr(i int64) *int64 { return &i }
|
||||
|
||||
// filesCapabilities is what the root executor keeps after dropping ALL (see
|
||||
// Config.RunAsUser). DAC_OVERRIDE opens a mode-0600 file (level.dat) the game wrote
|
||||
// as its own uid, which a fixed non-root uid could not. A write, mkdir, upload or
|
||||
// unzip also keeps CHOWN so what it creates can be handed to naming.GameUID
|
||||
// (exec.go ownWritten). List, read, delete and rename create nothing and get no
|
||||
// more than they need.
|
||||
// as its own uid, which a fixed non-root uid could not. A write, mkdir or upload
|
||||
// also keeps CHOWN so what it creates can be handed to naming.GameUID (exec.go
|
||||
// ownWritten). List, read, delete and rename create nothing and get no more than
|
||||
// they need.
|
||||
func filesCapabilities(op string) []corev1.Capability {
|
||||
if op == OpWrite || op == OpMkdir || op == OpUpload || op == OpUnzip {
|
||||
if op == OpWrite || op == OpMkdir || op == OpUpload {
|
||||
return []corev1.Capability{"CHOWN", "DAC_OVERRIDE"}
|
||||
}
|
||||
return []corev1.Capability{"DAC_OVERRIDE"}
|
||||
@@ -347,12 +323,3 @@ func filesPodSecurityContext(p JobParams) *corev1.PodSecurityContext {
|
||||
}
|
||||
return sc
|
||||
}
|
||||
|
||||
// filesAnnotations names the file an async Job works on. A path does not fit a
|
||||
// label (63 characters, no slashes), so it rides an annotation.
|
||||
func filesAnnotations(p JobParams) map[string]string {
|
||||
if !p.Async {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{AnnotationPath: p.Path}
|
||||
}
|
||||
@@ -2,9 +2,7 @@ package fileedit
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -130,7 +128,7 @@ func TestFilesJobIsolation(t *testing.T) {
|
||||
chown bool
|
||||
}{
|
||||
{OpList, false}, {OpRead, false}, {OpDelete, false}, {OpRename, false},
|
||||
{OpWrite, true}, {OpMkdir, true}, {OpUpload, true}, {OpUnzip, true},
|
||||
{OpWrite, true}, {OpMkdir, true}, {OpUpload, true},
|
||||
} {
|
||||
j, err := FilesJob(opParams(tc.op))
|
||||
if err != nil {
|
||||
@@ -218,7 +216,6 @@ func TestFilesJobWorldMountIsReadOnlyForReads(t *testing.T) {
|
||||
{OpDelete, false},
|
||||
{OpRename, false},
|
||||
{OpUpload, false},
|
||||
{OpUnzip, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.op, func(t *testing.T) {
|
||||
@@ -276,12 +273,6 @@ func TestFilesJobContentEnv(t *testing.T) {
|
||||
if strings.Contains(strings.Join(job.Spec.Template.Spec.Containers[0].Args, " "), "motd=hello") {
|
||||
t.Fatal("content must not appear in the container arguments")
|
||||
}
|
||||
// Its SHA-256 does, so the Job writes only the bytes felis-api handed over.
|
||||
sum := sha256.Sum256(p.Content)
|
||||
args := job.Spec.Template.Spec.Containers[0].Args
|
||||
if i := slices.Index(args, "--sha256"); i < 0 || i+1 >= len(args) || args[i+1] != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("args %q, want --sha256 %x", args, sum)
|
||||
}
|
||||
})
|
||||
|
||||
// execve refuses one environment string over 128 KiB and the container never
|
||||
@@ -361,8 +352,7 @@ func TestFilesJobOpArgs(t *testing.T) {
|
||||
|
||||
create := testParams(OpWrite)
|
||||
create.CreateOnly = true
|
||||
// The content (none here) goes with its SHA-256.
|
||||
if got := args(create); !slices.Equal(got, []string{"--sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "--create-only"}) {
|
||||
if got := args(create); !slices.Equal(got, []string{"--create-only"}) {
|
||||
t.Errorf("create-only write args = %v", got)
|
||||
}
|
||||
readCreate := testParams(OpRead)
|
||||
@@ -445,9 +435,6 @@ func TestFilesJobRejectsBadParams(t *testing.T) {
|
||||
p.Op, p.Content = OpWrite, make([]byte, MaxWriteBytes+1)
|
||||
}},
|
||||
{"bad cpu limit", func(p *JobParams) { p.CPULimit = "half" }},
|
||||
{"a read in the background", func(p *JobParams) { p.Async = true }},
|
||||
{"a write in the background", func(p *JobParams) { p.Op, p.Async = OpWrite, true }},
|
||||
{"a delete in the background", func(p *JobParams) { p.Op, p.Async = OpDelete, true }},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -459,36 +446,3 @@ func TestFilesJobRejectsBadParams(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFilesJobAsync checks a background Job is marked so Ops finds it, on the
|
||||
// Job and on its Pod, and carries the path Ops shows; a Job felis-api waits on
|
||||
// carries neither, so Ops never reports it.
|
||||
func TestFilesJobAsync(t *testing.T) {
|
||||
for _, op := range []string{OpUpload, OpUnzip} {
|
||||
t.Run(op, func(t *testing.T) {
|
||||
p := opParams(op)
|
||||
p.Path, p.Async = "maps/world.zip", true
|
||||
j, err := FilesJob(p)
|
||||
if err != nil {
|
||||
t.Fatalf("FilesJob: %v", err)
|
||||
}
|
||||
if j.Labels[LabelAsync] != "true" || j.Spec.Template.Labels[LabelAsync] != "true" {
|
||||
t.Fatalf("job labels %v, pod labels %v, want %s=true on both", j.Labels, j.Spec.Template.Labels, LabelAsync)
|
||||
}
|
||||
if len(j.Annotations) != 1 || j.Annotations[AnnotationPath] != "maps/world.zip" {
|
||||
t.Fatalf("annotations = %v, want only %s", j.Annotations, AnnotationPath)
|
||||
}
|
||||
|
||||
p.Async = false
|
||||
j, err = FilesJob(p)
|
||||
if err != nil {
|
||||
t.Fatalf("FilesJob: %v", err)
|
||||
}
|
||||
_, onJob := j.Labels[LabelAsync]
|
||||
_, onPod := j.Spec.Template.Labels[LabelAsync]
|
||||
if onJob || onPod || j.Annotations != nil {
|
||||
t.Fatalf("a Job waited on is labelled %v / %v and annotated %v", j.Labels, j.Spec.Template.Labels, j.Annotations)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,15 +3,12 @@ package fileedit
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
@@ -188,157 +185,3 @@ func tail(log string) string {
|
||||
}
|
||||
return "..." + log[len(log)-n:]
|
||||
}
|
||||
|
||||
// Start creates the Job for p and returns without waiting on it. Ops reads it
|
||||
// back.
|
||||
func (k *K8sRunner) Start(ctx context.Context, p JobParams) error {
|
||||
job, err := FilesJob(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := k.cs.BatchV1().Jobs(p.Namespace).Create(ctx, job, metav1.CreateOptions{}); err != nil {
|
||||
return fmt.Errorf("fileedit: create file job: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// maxOps bounds what Ops reports, and so how many Pod logs one call reads. Only
|
||||
// one background op runs per server at a time (it holds the world volume), so
|
||||
// this many are the one running and the latest that finished within AsyncTTL.
|
||||
const maxOps = 10
|
||||
|
||||
// opLogLines is how much of an op's log Ops reads: the result line is the last
|
||||
// thing the Job prints to stdout, the progress lines come before it, and a
|
||||
// failed run ends with one line on stderr.
|
||||
const opLogLines = 20
|
||||
|
||||
// Ops lists the server's background Jobs, newest first, with how far each has
|
||||
// got and how it ended, read from the tail of its Pod's log. A Pod whose log
|
||||
// cannot be read yet (still pulling its image) or any more (its node went away)
|
||||
// reports no progress rather than failing the listing.
|
||||
func (k *K8sRunner) Ops(ctx context.Context, namespace, server string) ([]OpState, error) {
|
||||
sel := metav1.ListOptions{LabelSelector: LabelManagedBy + "=" + managedByValue + "," +
|
||||
LabelServer + "=" + server + "," + LabelAsync + "=true"}
|
||||
jobs, err := k.cs.BatchV1().Jobs(namespace).List(ctx, sel)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fileedit: list file jobs: %w", err)
|
||||
}
|
||||
pods, err := k.cs.CoreV1().Pods(namespace).List(ctx, sel)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fileedit: list file job pods: %w", err)
|
||||
}
|
||||
// backoffLimit is 0, so a Job has one Pod; the newest wins all the same.
|
||||
podOf := map[string]*corev1.Pod{}
|
||||
for i := range pods.Items {
|
||||
pod := &pods.Items[i]
|
||||
id := pod.Labels[LabelOpID]
|
||||
if cur := podOf[id]; cur == nil || pod.CreationTimestamp.After(cur.CreationTimestamp.Time) {
|
||||
podOf[id] = pod
|
||||
}
|
||||
}
|
||||
items := jobs.Items
|
||||
sort.SliceStable(items, func(i, j int) bool {
|
||||
return items[i].CreationTimestamp.After(items[j].CreationTimestamp.Time)
|
||||
})
|
||||
if len(items) > maxOps {
|
||||
items = items[:maxOps]
|
||||
}
|
||||
out := make([]OpState, 0, len(items))
|
||||
for i := range items {
|
||||
log := ""
|
||||
pod := podOf[items[i].Labels[LabelOpID]]
|
||||
if pod != nil && pod.Status.Phase != corev1.PodPending {
|
||||
log, _ = k.logTail(ctx, namespace, pod.Name)
|
||||
}
|
||||
out = append(out, opState(&items[i], pod, log))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// logTail reads the last opLogLines lines of a Pod's log.
|
||||
func (k *K8sRunner) logTail(ctx context.Context, namespace, pod string) (string, error) {
|
||||
lines := int64(opLogLines)
|
||||
stream, err := k.cs.CoreV1().Pods(namespace).GetLogs(pod, &corev1.PodLogOptions{
|
||||
Container: containerName, TailLines: &lines,
|
||||
}).Stream(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer stream.Close()
|
||||
b, err := io.ReadAll(io.LimitReader(stream, maxLogBytes))
|
||||
return string(b), err
|
||||
}
|
||||
|
||||
// opState reads one background Job, and the tail of its Pod's log, as an
|
||||
// OpState. The printed result decides the outcome whatever the Job's condition
|
||||
// says: a Job killed at its deadline just after printing did finish its work.
|
||||
// A Job that ended without one failed, and the condition's reason says how
|
||||
// (DeadlineExceeded, BackoffLimitExceeded), unless its Pod says the kernel
|
||||
// killed it for memory (ReasonOOMKilled), which the condition does not tell; a
|
||||
// Job that completed but whose log could not be read has an outcome no one can
|
||||
// tell, ResultUnavailable.
|
||||
func opState(job *batchv1.Job, pod *corev1.Pod, log string) OpState {
|
||||
st := OpState{
|
||||
ID: job.Labels[LabelOpID], Op: job.Labels[LabelMode], Path: job.Annotations[AnnotationPath],
|
||||
State: OpRunning, Started: job.CreationTimestamp.Time,
|
||||
}
|
||||
if p, ok := lastProgress(log); ok {
|
||||
st.Done, st.Total = p.Done, p.Total
|
||||
}
|
||||
ended, reason := false, ""
|
||||
for _, c := range job.Status.Conditions {
|
||||
if c.Status != corev1.ConditionTrue {
|
||||
continue
|
||||
}
|
||||
switch c.Type {
|
||||
case batchv1.JobComplete, batchv1.JobSuccessCriteriaMet:
|
||||
ended, reason = true, "ResultUnavailable"
|
||||
st.Finished = c.LastTransitionTime.Time
|
||||
case batchv1.JobFailed, batchv1.JobFailureTarget:
|
||||
ended, reason = true, c.Reason
|
||||
st.Finished = c.LastTransitionTime.Time
|
||||
}
|
||||
}
|
||||
if !ended {
|
||||
return st
|
||||
}
|
||||
if payload, ok := extractResult(log); ok {
|
||||
var res Result
|
||||
if json.Unmarshal(payload, &res) == nil {
|
||||
st.Result = &res
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case st.Result != nil && st.Result.Code == "":
|
||||
st.State = OpSucceeded
|
||||
case st.Result != nil:
|
||||
st.State = OpFailed
|
||||
default:
|
||||
st.State, st.Reason = OpFailed, reason
|
||||
if killedForMemory(pod) {
|
||||
st.Reason = ReasonOOMKilled
|
||||
} else if st.Reason == "" {
|
||||
st.Reason = "Failed"
|
||||
}
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
// ReasonOOMKilled is an op's Reason when the kernel killed its Job for going over
|
||||
// the Job's memory limit: an archive of more entries than the Job can hold the
|
||||
// list of.
|
||||
const ReasonOOMKilled = "OOMKilled"
|
||||
|
||||
// killedForMemory reports whether pod's container was killed for going over its
|
||||
// memory limit.
|
||||
func killedForMemory(pod *corev1.Pod) bool {
|
||||
if pod == nil {
|
||||
return false
|
||||
}
|
||||
for _, cs := range pod.Status.ContainerStatuses {
|
||||
if t := cs.State.Terminated; cs.Name == containerName && t != nil && t.Reason == ReasonOOMKilled {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,282 +0,0 @@
|
||||
package fileedit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
k8stesting "k8s.io/client-go/testing"
|
||||
)
|
||||
|
||||
var (
|
||||
opCreated = time.Date(2026, 9, 28, 10, 0, 0, 0, time.UTC)
|
||||
opEnded = opCreated.Add(3 * time.Minute)
|
||||
)
|
||||
|
||||
func asyncJob(conds ...batchv1.JobCondition) *batchv1.Job {
|
||||
return &batchv1.Job{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "files-survival-0a",
|
||||
CreationTimestamp: metav1.NewTime(opCreated),
|
||||
Labels: map[string]string{LabelOpID: "0a", LabelMode: OpUnzip},
|
||||
Annotations: map[string]string{AnnotationPath: "maps/world.zip"},
|
||||
},
|
||||
Status: batchv1.JobStatus{Conditions: conds},
|
||||
}
|
||||
}
|
||||
|
||||
func cond(typ batchv1.JobConditionType, status corev1.ConditionStatus, reason string) batchv1.JobCondition {
|
||||
return batchv1.JobCondition{Type: typ, Status: status, Reason: reason, LastTransitionTime: metav1.NewTime(opEnded)}
|
||||
}
|
||||
|
||||
// TestOpState pins how a background Job and the tail of its log read as an
|
||||
// OpState: the printed result decides the outcome whatever the Job's condition
|
||||
// says, and a Job that ended without one failed for the condition's reason.
|
||||
func TestOpState(t *testing.T) {
|
||||
progress := ProgressPrefix + `{"done":10,"total":100}` + "\n" +
|
||||
"a stderr line\n" +
|
||||
ProgressPrefix + `{"done":40,"total":100}` + "\n"
|
||||
ok := ResultPrefix + `{"files":3,"bytes":40}` + "\n"
|
||||
conflict := ResultPrefix + `{"code":"exists","conflicts":["a.txt"],"conflict_count":1}` + "\n"
|
||||
complete := cond(batchv1.JobComplete, corev1.ConditionTrue, "")
|
||||
backoff := cond(batchv1.JobFailed, corev1.ConditionTrue, "BackoffLimitExceeded")
|
||||
killed := func(container, reason string) *corev1.Pod {
|
||||
return &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodFailed, ContainerStatuses: []corev1.ContainerStatus{{
|
||||
Name: container, State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 137, Reason: reason}},
|
||||
}}}}
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
conds []batchv1.JobCondition
|
||||
pod *corev1.Pod
|
||||
log string
|
||||
state string
|
||||
reason string
|
||||
code string
|
||||
files int
|
||||
done int64
|
||||
finished bool
|
||||
wantResult bool
|
||||
}{
|
||||
{name: "running, at its latest progress", log: progress, state: OpRunning, done: 40},
|
||||
{name: "running, before any progress", state: OpRunning},
|
||||
{name: "a condition not yet true is still running",
|
||||
conds: []batchv1.JobCondition{cond(batchv1.JobFailed, corev1.ConditionFalse, "DeadlineExceeded")},
|
||||
log: progress, state: OpRunning, done: 40},
|
||||
{name: "complete with a clean result",
|
||||
conds: []batchv1.JobCondition{complete}, log: progress + ok,
|
||||
state: OpSucceeded, files: 3, done: 40, finished: true, wantResult: true},
|
||||
{name: "success criteria met before complete",
|
||||
conds: []batchv1.JobCondition{cond(batchv1.JobSuccessCriteriaMet, corev1.ConditionTrue, "")}, log: ok,
|
||||
state: OpSucceeded, files: 3, finished: true, wantResult: true},
|
||||
{name: "killed at its deadline after printing a clean result",
|
||||
conds: []batchv1.JobCondition{cond(batchv1.JobFailed, corev1.ConditionTrue, "DeadlineExceeded")}, log: ok,
|
||||
state: OpSucceeded, files: 3, finished: true, wantResult: true},
|
||||
{name: "complete with a refusal",
|
||||
conds: []batchv1.JobCondition{complete}, log: conflict,
|
||||
state: OpFailed, code: CodeExists, finished: true, wantResult: true},
|
||||
{name: "killed at its deadline without a result",
|
||||
conds: []batchv1.JobCondition{cond(batchv1.JobFailed, corev1.ConditionTrue, "DeadlineExceeded")}, log: progress,
|
||||
state: OpFailed, reason: "DeadlineExceeded", done: 40, finished: true},
|
||||
{name: "failure target before failed",
|
||||
conds: []batchv1.JobCondition{cond(batchv1.JobFailureTarget, corev1.ConditionTrue, "BackoffLimitExceeded")},
|
||||
state: OpFailed, reason: "BackoffLimitExceeded", finished: true},
|
||||
{name: "failed without a reason",
|
||||
conds: []batchv1.JobCondition{cond(batchv1.JobFailed, corev1.ConditionTrue, "")},
|
||||
state: OpFailed, reason: "Failed", finished: true},
|
||||
{name: "complete but its log is gone",
|
||||
conds: []batchv1.JobCondition{complete},
|
||||
state: OpFailed, reason: "ResultUnavailable", finished: true},
|
||||
{name: "complete with a result that does not parse",
|
||||
conds: []batchv1.JobCondition{complete}, log: ResultPrefix + "{\n",
|
||||
state: OpFailed, reason: "ResultUnavailable", finished: true},
|
||||
{name: "killed for memory without a result",
|
||||
conds: []batchv1.JobCondition{backoff}, pod: killed(containerName, "OOMKilled"), log: progress,
|
||||
state: OpFailed, reason: "OOMKilled", done: 40, finished: true},
|
||||
{name: "killed for memory after printing a clean result",
|
||||
conds: []batchv1.JobCondition{backoff}, pod: killed(containerName, "OOMKilled"), log: ok,
|
||||
state: OpSucceeded, files: 3, finished: true, wantResult: true},
|
||||
{name: "killed another way",
|
||||
conds: []batchv1.JobCondition{backoff}, pod: killed(containerName, "Error"),
|
||||
state: OpFailed, reason: "BackoffLimitExceeded", finished: true},
|
||||
{name: "another container killed for memory",
|
||||
conds: []batchv1.JobCondition{backoff}, pod: killed("sidecar", "OOMKilled"),
|
||||
state: OpFailed, reason: "BackoffLimitExceeded", finished: true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
st := opState(asyncJob(tc.conds...), tc.pod, tc.log)
|
||||
if st.ID != "0a" || st.Op != OpUnzip || st.Path != "maps/world.zip" || !st.Started.Equal(opCreated) {
|
||||
t.Fatalf("identity = %q %q %q %v", st.ID, st.Op, st.Path, st.Started)
|
||||
}
|
||||
if st.State != tc.state || st.Reason != tc.reason {
|
||||
t.Fatalf("state = %q reason %q, want %q reason %q", st.State, st.Reason, tc.state, tc.reason)
|
||||
}
|
||||
if st.Done != tc.done || (tc.done != 0 && st.Total != 100) {
|
||||
t.Fatalf("progress = %d/%d, want %d/100", st.Done, st.Total, tc.done)
|
||||
}
|
||||
if want := map[bool]time.Time{true: opEnded}[tc.finished]; !st.Finished.Equal(want) {
|
||||
t.Fatalf("finished = %v, want %v", st.Finished, want)
|
||||
}
|
||||
if (st.Result != nil) != tc.wantResult {
|
||||
t.Fatalf("result = %+v, want one: %v", st.Result, tc.wantResult)
|
||||
}
|
||||
if st.Result != nil && (st.Result.Code != tc.code || st.Result.Files != tc.files) {
|
||||
t.Fatalf("result = %+v, want code %q and %d files", st.Result, tc.code, tc.files)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestK8sRunnerOps checks what Ops lists: this server's background Jobs only,
|
||||
// newest first and at most maxOps of them, with a log read for each started Pod
|
||||
// and none for a Pod still waiting to run.
|
||||
func TestK8sRunnerOps(t *testing.T) {
|
||||
job := func(server, id string, age time.Duration, async bool) *batchv1.Job {
|
||||
p := testParams(OpUnzip)
|
||||
p.Server, p.OpID, p.Path, p.Async = server, id, "maps/"+id+".zip", async
|
||||
j, err := FilesJob(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j.CreationTimestamp = metav1.NewTime(opCreated.Add(-age))
|
||||
return j
|
||||
}
|
||||
pod := func(j *batchv1.Job, phase corev1.PodPhase, age time.Duration) *corev1.Pod {
|
||||
return &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: fmt.Sprintf("%s-%d", j.Name, age), Namespace: "minecraft", Labels: j.Spec.Template.Labels,
|
||||
CreationTimestamp: metav1.NewTime(opCreated.Add(-age)),
|
||||
},
|
||||
Status: corev1.PodStatus{Phase: phase},
|
||||
}
|
||||
}
|
||||
|
||||
var objs []runtime.Object
|
||||
for i := range maxOps + 2 {
|
||||
j := job("survival", fmt.Sprintf("%02d", i), time.Duration(i)*time.Minute, true)
|
||||
objs = append(objs, j, pod(j, corev1.PodSucceeded, time.Duration(i)*time.Minute))
|
||||
}
|
||||
// The newest Job's newest Pod has not started, so its log is not read; the
|
||||
// older Pod beside it is not the one Ops reports on.
|
||||
newest := job("survival", "new", -time.Minute, true)
|
||||
objs = append(objs, newest,
|
||||
pod(newest, corev1.PodPending, -time.Minute), pod(newest, corev1.PodFailed, 0))
|
||||
objs = append(objs,
|
||||
job("creative", "other", -2*time.Minute, true),
|
||||
job("survival", "sync", -3*time.Minute, false))
|
||||
cs := fake.NewSimpleClientset(objs...)
|
||||
|
||||
ops, err := NewK8sRunner(cs).Ops(context.Background(), "minecraft", "survival")
|
||||
if err != nil {
|
||||
t.Fatalf("Ops: %v", err)
|
||||
}
|
||||
var ids []string
|
||||
for _, op := range ops {
|
||||
ids = append(ids, op.ID)
|
||||
}
|
||||
want := []string{"new", "00", "01", "02", "03", "04", "05", "06", "07", "08"}
|
||||
if fmt.Sprint(ids) != fmt.Sprint(want) {
|
||||
t.Fatalf("ops = %v, want %v", ids, want)
|
||||
}
|
||||
if ops[0].Path != "maps/new.zip" || ops[0].Op != OpUnzip {
|
||||
t.Fatalf("newest op = %+v", ops[0])
|
||||
}
|
||||
|
||||
logs := 0
|
||||
for _, a := range cs.Actions() {
|
||||
if a.GetVerb() == "get" && a.GetSubresource() == "log" {
|
||||
logs++
|
||||
opts := a.(k8stesting.GenericAction).GetValue().(*corev1.PodLogOptions)
|
||||
if opts.Container != containerName || opts.TailLines == nil || *opts.TailLines != opLogLines {
|
||||
t.Fatalf("log options = %+v", opts)
|
||||
}
|
||||
}
|
||||
}
|
||||
if logs != maxOps-1 {
|
||||
t.Fatalf("read %d logs, want %d: one per listed op whose Pod has started", logs, maxOps-1)
|
||||
}
|
||||
}
|
||||
|
||||
// TestK8sRunnerOpsReadsAMemoryKill checks Ops hands each Job's Pod to opState: a
|
||||
// Pod the kernel killed for memory is what names an op's reason OOMKilled.
|
||||
func TestK8sRunnerOpsReadsAMemoryKill(t *testing.T) {
|
||||
p := testParams(OpUnzip)
|
||||
p.Server, p.OpID, p.Path, p.Async = "survival", "oom", "maps/tiles.zip", true
|
||||
j, err := FilesJob(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j.Status.Conditions = []batchv1.JobCondition{cond(batchv1.JobFailed, corev1.ConditionTrue, "BackoffLimitExceeded")}
|
||||
pod := &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: j.Name + "-x", Namespace: "minecraft", Labels: j.Spec.Template.Labels},
|
||||
Status: corev1.PodStatus{Phase: corev1.PodFailed, ContainerStatuses: []corev1.ContainerStatus{{
|
||||
Name: containerName, State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 137, Reason: "OOMKilled"}},
|
||||
}}},
|
||||
}
|
||||
ops, err := NewK8sRunner(fake.NewSimpleClientset(j, pod)).Ops(context.Background(), "minecraft", "survival")
|
||||
if err != nil || len(ops) != 1 || ops[0].State != OpFailed || ops[0].Reason != "OOMKilled" {
|
||||
t.Fatalf("Ops = %+v, %v; want the one op failed for OOMKilled", ops, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestK8sRunnerOpsFailsLoudly checks a listing the cluster refused is an error,
|
||||
// never an empty list that would read as nothing running.
|
||||
func TestK8sRunnerOpsFailsLoudly(t *testing.T) {
|
||||
for _, resource := range []string{"jobs", "pods"} {
|
||||
t.Run(resource, func(t *testing.T) {
|
||||
cs := fake.NewSimpleClientset()
|
||||
cs.PrependReactor("list", resource, func(k8stesting.Action) (bool, runtime.Object, error) {
|
||||
return true, nil, fmt.Errorf("forbidden")
|
||||
})
|
||||
ops, err := NewK8sRunner(cs).Ops(context.Background(), "minecraft", "survival")
|
||||
if err == nil || ops != nil {
|
||||
t.Fatalf("Ops = %v, %v; want the refusal", ops, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestK8sRunnerStart checks Start creates the rendered Job and nothing else,
|
||||
// and refuses params the renderer refuses without touching the cluster.
|
||||
func TestK8sRunnerStart(t *testing.T) {
|
||||
cs := fake.NewSimpleClientset()
|
||||
p := testParams(OpUnzip)
|
||||
p.Path, p.Async = "maps/world.zip", true
|
||||
if err := NewK8sRunner(cs).Start(context.Background(), p); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
want, _ := FilesJob(p)
|
||||
got, err := cs.BatchV1().Jobs("minecraft").Get(context.Background(), want.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("the Job was not created: %v", err)
|
||||
}
|
||||
if got.Labels[LabelAsync] != "true" || got.Annotations[AnnotationPath] != "maps/world.zip" {
|
||||
t.Fatalf("created %+v", got.ObjectMeta)
|
||||
}
|
||||
if n := len(cs.Actions()); n != 2 { // the create, and this test's get
|
||||
t.Fatalf("%d calls to the cluster, want the one create", n-1)
|
||||
}
|
||||
|
||||
cs = fake.NewSimpleClientset()
|
||||
p.Op = OpRead
|
||||
if err := NewK8sRunner(cs).Start(context.Background(), p); err == nil || len(cs.Actions()) != 0 {
|
||||
t.Fatalf("err %v after %d calls, want a refusal before any", err, len(cs.Actions()))
|
||||
}
|
||||
|
||||
cs = fake.NewSimpleClientset()
|
||||
cs.PrependReactor("create", "jobs", func(k8stesting.Action) (bool, runtime.Object, error) {
|
||||
return true, nil, fmt.Errorf("quota exceeded")
|
||||
})
|
||||
p.Op = OpUnzip
|
||||
if err := NewK8sRunner(cs).Start(context.Background(), p); err == nil {
|
||||
t.Fatal("a refused create must be an error")
|
||||
}
|
||||
}
|
||||
@@ -397,18 +397,16 @@ func TestRename(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// fakeSource is an upload's bytes as the Job would fetch them. landed counts
|
||||
// the reports that they are in place.
|
||||
// fakeSource is an upload's bytes as the Job would fetch them.
|
||||
type fakeSource struct {
|
||||
body string
|
||||
opened int
|
||||
landed int
|
||||
err error // returned by Open
|
||||
readErr error // returned by the body once it runs out
|
||||
}
|
||||
|
||||
func (s *fakeSource) upload(size int64, sum string) *Upload {
|
||||
return &Upload{Size: size, SHA256: sum, Landed: func() { s.landed++ }, Open: func() (io.ReadCloser, error) {
|
||||
return &Upload{Size: size, SHA256: sum, Open: func() (io.ReadCloser, error) {
|
||||
s.opened++
|
||||
if s.err != nil {
|
||||
return nil, s.err
|
||||
@@ -436,17 +434,9 @@ func TestUpload(t *testing.T) {
|
||||
t.Run("lands the bytes as a new file", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
src := &fakeSource{body: jar}
|
||||
u := whole(src)
|
||||
// Reported once the file is in place, never before.
|
||||
var there string
|
||||
u.Landed = func() {
|
||||
src.landed++
|
||||
b, _ := os.ReadFile(filepath.Join(root, "config", "Geyser.jar"))
|
||||
there = string(b)
|
||||
}
|
||||
res, err := send(t, root, "config/Geyser.jar", u, false)
|
||||
if err != nil || res.Code != "" || src.landed != 1 || there != jar {
|
||||
t.Fatalf("result = %+v, %v; landed %d with %q in place", res, err, src.landed, there)
|
||||
res, err := send(t, root, "config/Geyser.jar", whole(src), false)
|
||||
if err != nil || res.Code != "" {
|
||||
t.Fatalf("result = %+v, %v", res, err)
|
||||
}
|
||||
if got := mustRead(t, filepath.Join(root, "config", "Geyser.jar")); got != jar {
|
||||
t.Fatalf("content = %q", got)
|
||||
@@ -464,8 +454,8 @@ func TestUpload(t *testing.T) {
|
||||
for _, name := range []string{"server.properties", "dangling.jar"} {
|
||||
src := &fakeSource{body: jar}
|
||||
res, err := send(t, root, name, whole(src), false)
|
||||
if err != nil || res.Code != CodeExists || src.opened != 0 || src.landed != 0 {
|
||||
t.Fatalf("%s: result = %+v, %v, opened %d, landed %d; want exists and no fetch", name, res, err, src.opened, src.landed)
|
||||
if err != nil || res.Code != CodeExists || src.opened != 0 {
|
||||
t.Fatalf("%s: result = %+v, %v, opened %d; want exists and no fetch", name, res, err, src.opened)
|
||||
}
|
||||
}
|
||||
if got := mustRead(t, filepath.Join(root, "server.properties")); got != "motd=hello\n" {
|
||||
@@ -501,39 +491,17 @@ func TestUpload(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("more than the volume has free is no_space and never fetched; exactly the free room is fetched", func(t *testing.T) {
|
||||
t.Run("over the cap is too_large and never fetched; at the cap is fetched", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
stubStatfs(t, uint64(len(jar)), 1<<30)
|
||||
src := &fakeSource{body: jar}
|
||||
res, err := send(t, root, "big.jar", src.upload(int64(len(jar))+1, ""), false)
|
||||
if err != nil || res.Code != CodeNoSpace || res.Need != int64(len(jar))+1 || res.Avail != int64(len(jar)) || src.opened != 0 {
|
||||
t.Fatalf("result = %+v, %v, opened %d; want no_space with need %d, avail %d", res, err, src.opened, len(jar)+1, len(jar))
|
||||
res, err := send(t, root, "big.jar", src.upload(MaxUploadBytes+1, ""), false)
|
||||
if err != nil || res.Code != CodeTooLarge || src.opened != 0 {
|
||||
t.Fatalf("result = %+v, %v, opened %d", res, err, src.opened)
|
||||
}
|
||||
assertAbsent(t, filepath.Join(root, "big.jar"))
|
||||
if res, err := send(t, root, "big.jar", whole(src), false); err != nil || res.Code != "" || src.opened != 1 {
|
||||
t.Fatalf("at the free room: result = %+v, %v, opened %d; want it landed", res, err, src.opened)
|
||||
}
|
||||
})
|
||||
|
||||
// The editor used to stop at MaxUploadBytes; a file sent in parts is bounded
|
||||
// by the volume alone.
|
||||
t.Run("past the single-request limit is fetched", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
stubStatfs(t, 1<<40, 1<<41)
|
||||
src := &fakeSource{body: jar}
|
||||
// This source then comes up short, which is a broken transfer, not a refusal.
|
||||
if _, err := send(t, root, "big.jar", src.upload(MaxUploadBytes+1, ""), false); err == nil || src.opened != 1 {
|
||||
t.Fatalf("err = %v, opened %d; want a fetch", err, src.opened)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("progress hears every byte", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
var last, calls, total int64
|
||||
res := exec(t, root, Request{Op: OpUpload, Path: "x.jar", Upload: whole(&fakeSource{body: jar}),
|
||||
Progress: func(done, all int64) { calls++; last, total = done, all }})
|
||||
if res.Code != "" || calls == 0 || last != int64(len(jar)) || total != int64(len(jar)) {
|
||||
t.Fatalf("result = %+v; progress calls %d, last %d of %d; want the whole %d", res, calls, last, total, len(jar))
|
||||
// At the cap the size passes and the transfer starts; this source then
|
||||
// comes up short, which is a broken transfer rather than a refusal.
|
||||
if _, err := send(t, root, "big.jar", src.upload(MaxUploadBytes, ""), false); err == nil || src.opened != 1 {
|
||||
t.Fatalf("at the cap: err = %v, opened %d; want a fetch", err, src.opened)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -551,11 +519,10 @@ func TestUpload(t *testing.T) {
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
src := &fakeSource{body: jar}
|
||||
res, err := send(t, root, "server.properties", u(src), true)
|
||||
res, err := send(t, root, "server.properties", u(&fakeSource{body: jar}), true)
|
||||
var te *transferError
|
||||
if !errors.As(err, &te) || res.Code != "" || src.landed != 0 {
|
||||
t.Fatalf("result = %+v, err = %v, landed %d; want a transfer error and no report", res, err, src.landed)
|
||||
if !errors.As(err, &te) || res.Code != "" {
|
||||
t.Fatalf("result = %+v, err = %v; want a transfer error", res, err)
|
||||
}
|
||||
if got := mustRead(t, filepath.Join(root, "server.properties")); got != "motd=hello\n" {
|
||||
t.Fatalf("server.properties became %q", got)
|
||||
@@ -570,11 +537,9 @@ func TestUpload(t *testing.T) {
|
||||
prev := syncWritten
|
||||
syncWritten = func(*os.File) error { return syscall.ENOSPC }
|
||||
defer func() { syncWritten = prev }()
|
||||
src := &fakeSource{body: jar}
|
||||
res, err := send(t, root, "server.properties", whole(src), true)
|
||||
// The bytes were fetched but never landed, so felis-api keeps them.
|
||||
if err != nil || res.Code != CodeNoSpace || src.opened != 1 || src.landed != 0 {
|
||||
t.Fatalf("result = %+v, %v, opened %d, landed %d; want no_space after a fetch and no report", res, err, src.opened, src.landed)
|
||||
res, err := send(t, root, "server.properties", whole(&fakeSource{body: jar}), true)
|
||||
if err != nil || res.Code != CodeNoSpace {
|
||||
t.Fatalf("result = %+v, %v; want no_space", res, err)
|
||||
}
|
||||
if got := mustRead(t, filepath.Join(root, "server.properties")); got != "motd=hello\n" {
|
||||
t.Fatalf("server.properties became %q", got)
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
package fileedit
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProgressPrefix marks a progress line: how many of an upload's or an unzip's
|
||||
// bytes are in so far, as JSON. Those two run as Jobs felis-api does not wait on,
|
||||
// and the panel shows how far one has got by reading the latest such line from
|
||||
// the tail of the Pod's log. Like ResultPrefix it is found by its marker, since
|
||||
// the log is stdout and stderr merged.
|
||||
const ProgressPrefix = "FELIS-FILES-PROGRESS: "
|
||||
|
||||
// Progress is one progress line.
|
||||
type Progress struct {
|
||||
Done int64 `json:"done"`
|
||||
Total int64 `json:"total"`
|
||||
}
|
||||
|
||||
// ThrottledProgress returns a progress func that prints to w at most once per
|
||||
// every, plus the first call (the last print starts at the zero time) and the
|
||||
// one that reaches the total, so the log grows by a line a second however fast
|
||||
// the bytes move and still ends on the true final count. A write error is
|
||||
// dropped: progress is a courtesy, and the result line that follows is what
|
||||
// felis-api acts on.
|
||||
func ThrottledProgress(w io.Writer, every time.Duration, now func() time.Time) func(done, total int64) {
|
||||
var last time.Time
|
||||
return func(done, total int64) {
|
||||
t := now()
|
||||
if done < total && t.Sub(last) < every {
|
||||
return
|
||||
}
|
||||
last = t
|
||||
b, _ := json.Marshal(Progress{Done: done, Total: total})
|
||||
fmt.Fprintf(w, "%s%s\n", ProgressPrefix, b)
|
||||
}
|
||||
}
|
||||
|
||||
// lastProgress finds the latest well-formed progress line in a log.
|
||||
func lastProgress(log string) (Progress, bool) {
|
||||
var p Progress
|
||||
found := false
|
||||
sc := bufio.NewScanner(strings.NewReader(log))
|
||||
sc.Buffer(make([]byte, 0, 4096), maxLogBytes)
|
||||
for sc.Scan() {
|
||||
rest, ok := strings.CutPrefix(sc.Text(), ProgressPrefix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var q Progress
|
||||
if json.Unmarshal([]byte(rest), &q) == nil {
|
||||
p, found = q, true
|
||||
}
|
||||
}
|
||||
return p, found
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
package fileedit
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestThrottledProgress(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
clock := time.Unix(1000, 0)
|
||||
progress := ThrottledProgress(&out, time.Second, func() time.Time { return clock })
|
||||
|
||||
progress(0, 100) // first call: printed
|
||||
progress(10, 100) // same instant: dropped
|
||||
clock = clock.Add(999 * time.Millisecond)
|
||||
progress(20, 100) // not a second yet: dropped
|
||||
clock = clock.Add(time.Millisecond)
|
||||
progress(30, 100) // a second on: printed
|
||||
progress(100, 100) // the total, however soon: printed
|
||||
want := ProgressPrefix + `{"done":0,"total":100}` + "\n" +
|
||||
ProgressPrefix + `{"done":30,"total":100}` + "\n" +
|
||||
ProgressPrefix + `{"done":100,"total":100}` + "\n"
|
||||
if out.String() != want {
|
||||
t.Fatalf("printed:\n%s\nwant:\n%s", out.String(), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLastProgress(t *testing.T) {
|
||||
log := "noise\n" +
|
||||
ProgressPrefix + `{"done":1,"total":9}` + "\n" +
|
||||
"a runtime warning on stderr\n" +
|
||||
ProgressPrefix + `{"done":5,"total":9}` + "\n" +
|
||||
ProgressPrefix + `{"done":` + "\n" // cut off mid-line by the tail
|
||||
p, ok := lastProgress(log)
|
||||
if !ok || p != (Progress{Done: 5, Total: 9}) {
|
||||
t.Fatalf("lastProgress = %+v, %v; want {5 9}, true", p, ok)
|
||||
}
|
||||
if _, ok := lastProgress("no marker here\n"); ok {
|
||||
t.Fatal("a log without a progress line reported one")
|
||||
}
|
||||
}
|
||||
@@ -1,372 +0,0 @@
|
||||
package fileedit
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A file too big for one request body arrives as a session: Begin declares its
|
||||
// size and where it goes, Append adds one part at a time in order, and Seal
|
||||
// hands the finished file to the Job that lands it, which fetches it through
|
||||
// Open like any other staged upload. The Cloudflare edge refuses bodies over
|
||||
// 100 MB, so a part is at most PartBytes; the file itself has no ceiling but the
|
||||
// room on the staging disk, and Begin reserves all of it up front, so an upload
|
||||
// that starts is one the disk can finish.
|
||||
//
|
||||
// Every call names the user and the server the session was begun for, and a
|
||||
// session answers no one else: an id that is someone else's reads as unknown.
|
||||
//
|
||||
// Each part carries the SHA-256 the client computed over it, and a part whose
|
||||
// bytes hash to anything else was changed on the way and is refused. A part that
|
||||
// fails midway (the connection dropped, the edge cut it off, the digest did not
|
||||
// match) is rolled back to where it started, so the session's length is always
|
||||
// the resume point. The session keeps each part's size and digest, so a client
|
||||
// resuming with a file from disk can check the file still holds the bytes
|
||||
// already sent before it sends the rest.
|
||||
//
|
||||
// A sealed session stays until the Job that fetched it says its file has landed
|
||||
// (Landed), so a Job that failed at any point before that (a broken fetch, a
|
||||
// full volume, a file in the way) can be started again without the file being
|
||||
// sent again; one left idle for SessionIdle is dropped (Expire).
|
||||
|
||||
// PartBytes is the largest part Append takes, matching the modpack upload's
|
||||
// parts (submit.DefaultPartMaxBytes).
|
||||
const PartBytes = 32 << 20
|
||||
|
||||
// SessionIdle is how long a session may sit untouched before Expire drops it:
|
||||
// long enough to resume after a lost connection or a laptop lid, short enough
|
||||
// that an abandoned upload gives its room back the same day.
|
||||
const SessionIdle = 6 * time.Hour
|
||||
|
||||
// MaxSessionsPerUser bounds the sessions one user holds open at once. Each
|
||||
// reserves its whole size on the staging disk, so without a bound one user
|
||||
// could reserve the disk out from under everyone for SessionIdle.
|
||||
const MaxSessionsPerUser = 4
|
||||
|
||||
var (
|
||||
// ErrTooManySessions is a Begin by a user who already holds
|
||||
// MaxSessionsPerUser sessions.
|
||||
ErrTooManySessions = errors.New("fileedit: too many uploads in progress")
|
||||
// ErrUploadBusy is a call on a session another request is still appending
|
||||
// to. Parts go one at a time.
|
||||
ErrUploadBusy = errors.New("fileedit: another request is still writing this upload")
|
||||
// ErrPartTooLarge is a part over PartBytes, or one that runs past the size
|
||||
// the session was begun with.
|
||||
ErrPartTooLarge = errors.New("fileedit: the part is too large")
|
||||
// ErrUploadIncomplete is a Seal before every byte has arrived.
|
||||
ErrUploadIncomplete = errors.New("fileedit: the upload has not finished arriving")
|
||||
)
|
||||
|
||||
// OffsetError is a part that does not start where the session ends. Received is
|
||||
// where it does end, so the client resumes from there.
|
||||
type OffsetError struct{ Received int64 }
|
||||
|
||||
func (e *OffsetError) Error() string {
|
||||
return fmt.Sprintf("fileedit: the upload holds %d bytes; send the part that starts there", e.Received)
|
||||
}
|
||||
|
||||
// Session is where one session stands.
|
||||
type Session struct {
|
||||
ID string
|
||||
Path string
|
||||
Size int64
|
||||
Received int64
|
||||
// Parts are the parts that make up Received, in order.
|
||||
Parts []Part
|
||||
}
|
||||
|
||||
// Part is one part a session took: its length and the SHA-256 (lowercase hex)
|
||||
// it arrived with and matched.
|
||||
type Part struct {
|
||||
Size int64
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
type session struct {
|
||||
user, server, path string
|
||||
file string
|
||||
size, received int64
|
||||
h hash.Hash
|
||||
parts []Part
|
||||
busy bool
|
||||
touched time.Time
|
||||
|
||||
// armed is set by Seal with the digest of the token it minted and cleared by
|
||||
// the Open that spends it. tokenHash stays until the next Seal, so the Job
|
||||
// holding that token can still report its file landed (Landed).
|
||||
armed bool
|
||||
tokenHash [sha256.Size]byte
|
||||
}
|
||||
|
||||
func (s *Stage) now() time.Time {
|
||||
if s.Now != nil {
|
||||
return s.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// Begin opens a session for a file of size bytes that will land at path on
|
||||
// server, reserving room for all of it.
|
||||
func (s *Stage) Begin(user, server, path string, size int64) (Session, error) {
|
||||
if size < 0 {
|
||||
return Session{}, fmt.Errorf("fileedit: an upload of %d bytes", size)
|
||||
}
|
||||
id, err := randomHex(16)
|
||||
if err != nil {
|
||||
return Session{}, fmt.Errorf("fileedit: generate an upload id: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(s.Dir, 0o700); err != nil {
|
||||
return Session{}, fmt.Errorf("fileedit: create the upload stage: %w", err)
|
||||
}
|
||||
if err := s.reserve(size); err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
f, err := os.CreateTemp(s.Dir, "session-*")
|
||||
if err != nil {
|
||||
s.unreserve(size)
|
||||
return Session{}, fmt.Errorf("fileedit: stage the upload: %w", err)
|
||||
}
|
||||
f.Close()
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
n := 0
|
||||
for _, ss := range s.sessions {
|
||||
if ss.user == user {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n >= MaxSessionsPerUser {
|
||||
s.reserved -= size
|
||||
os.Remove(f.Name())
|
||||
return Session{}, fmt.Errorf("%w: finish or cancel one of your %d uploads first", ErrTooManySessions, n)
|
||||
}
|
||||
if s.sessions == nil {
|
||||
s.sessions = map[string]*session{}
|
||||
}
|
||||
s.sessions[id] = &session{
|
||||
user: user, server: server, path: path, file: f.Name(),
|
||||
size: size, h: sha256.New(), touched: s.now(),
|
||||
}
|
||||
return Session{ID: id, Path: path, Size: size}, nil
|
||||
}
|
||||
|
||||
// lookup finds the caller's session. s.mu must be held.
|
||||
func (s *Stage) lookup(user, server, id string) (*session, error) {
|
||||
ss, ok := s.sessions[id]
|
||||
if !ok || ss.user != user || ss.server != server {
|
||||
return nil, ErrNotStaged
|
||||
}
|
||||
return ss, nil
|
||||
}
|
||||
|
||||
func (ss *session) view(id string) Session {
|
||||
return Session{ID: id, Path: ss.path, Size: ss.size, Received: ss.received, Parts: slices.Clone(ss.parts)}
|
||||
}
|
||||
|
||||
// Status reports where the caller's session stands.
|
||||
func (s *Stage) Status(user, server, id string) (Session, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ss, err := s.lookup(user, server, id)
|
||||
if err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
return ss.view(id), nil
|
||||
}
|
||||
|
||||
// Append adds the n bytes of body at offset, which must be where the session
|
||||
// ends. body must end right after them (an HTTP body of that Content-Length
|
||||
// does), and they must hash to want, the SHA-256 the client computed over them
|
||||
// (ErrDigestMismatch otherwise). On any failure the session is left as it was
|
||||
// before the call.
|
||||
func (s *Stage) Append(user, server, id string, offset int64, body io.Reader, n int64, want []byte) (Session, error) {
|
||||
s.mu.Lock()
|
||||
ss, err := s.lookup(user, server, id)
|
||||
switch {
|
||||
case err != nil:
|
||||
case len(want) != sha256.Size:
|
||||
err = ErrNoDigest
|
||||
case ss.busy:
|
||||
err = ErrUploadBusy
|
||||
case offset != ss.received:
|
||||
err = &OffsetError{Received: ss.received}
|
||||
case n < 0 || n > PartBytes || n > ss.size-ss.received:
|
||||
err = fmt.Errorf("%w: %d bytes at %d of a %d-byte upload; parts are at most %d bytes",
|
||||
ErrPartTooLarge, n, offset, ss.size, PartBytes)
|
||||
}
|
||||
if err != nil {
|
||||
var view Session
|
||||
if ss != nil {
|
||||
view = ss.view(id)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return view, err
|
||||
}
|
||||
ss.busy = true
|
||||
s.mu.Unlock()
|
||||
|
||||
// The session is ours until busy is cleared, so the file and the hash are
|
||||
// touched without the lock. The hash's state is kept to undo a failed part.
|
||||
before, err := ss.h.(encoding.BinaryMarshaler).MarshalBinary()
|
||||
if err == nil {
|
||||
part := sha256.New()
|
||||
err = appendPart(ss.file, offset, body, n, io.MultiWriter(ss.h, part))
|
||||
if err == nil {
|
||||
err = checkDigest(part.Sum(nil), want)
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Truncate(ss.file, offset)
|
||||
_ = ss.h.(encoding.BinaryUnmarshaler).UnmarshalBinary(before)
|
||||
}
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ss.busy = false
|
||||
ss.touched = s.now()
|
||||
if err != nil {
|
||||
return ss.view(id), err
|
||||
}
|
||||
ss.received += n
|
||||
s.reserved -= n
|
||||
ss.parts = append(ss.parts, Part{Size: n, SHA256: hex.EncodeToString(want)})
|
||||
return ss.view(id), nil
|
||||
}
|
||||
|
||||
// appendPart writes exactly n bytes of body at offset in the file named file,
|
||||
// feeding them to h as well.
|
||||
func appendPart(file string, offset int64, body io.Reader, n int64, h io.Writer) error {
|
||||
f, err := os.OpenFile(file, os.O_WRONLY, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fileedit: open the staged upload: %w", err)
|
||||
}
|
||||
src := &bodyReader{r: body}
|
||||
// One byte past n, so the read that finds the end happens here.
|
||||
got, copyErr := io.Copy(io.MultiWriter(io.NewOffsetWriter(f, offset), h), io.LimitReader(src, n+1))
|
||||
closeErr := f.Close()
|
||||
return stageFailure(src.err, copyErr, closeErr, got, n)
|
||||
}
|
||||
|
||||
// Seal ends the caller's session and arms it for one fetch: the Staged it
|
||||
// returns carries a fresh token, and any token an earlier Seal minted stops
|
||||
// working. Every byte must have arrived.
|
||||
func (s *Stage) Seal(user, server, id string) (Staged, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ss, err := s.lookup(user, server, id)
|
||||
if err != nil {
|
||||
return Staged{}, err
|
||||
}
|
||||
if ss.busy {
|
||||
return Staged{}, ErrUploadBusy
|
||||
}
|
||||
if ss.received != ss.size {
|
||||
return Staged{}, fmt.Errorf("%w: %d of %d bytes are here", ErrUploadIncomplete, ss.received, ss.size)
|
||||
}
|
||||
st, tokenHash, err := newHandle(ss.h, ss.size)
|
||||
if err != nil {
|
||||
return Staged{}, err
|
||||
}
|
||||
st.ID = id
|
||||
ss.armed, ss.tokenHash = true, tokenHash
|
||||
ss.touched = s.now()
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// openSession is Open for a sealed session. s.mu must be held; ok is false when
|
||||
// id names no session.
|
||||
func (s *Stage) openSession(id string, sum [sha256.Size]byte) (path string, size int64, ok bool, err error) {
|
||||
ss, found := s.sessions[id]
|
||||
if !found {
|
||||
return "", 0, false, nil
|
||||
}
|
||||
if !ss.armed || subtle.ConstantTimeCompare(sum[:], ss.tokenHash[:]) != 1 {
|
||||
return "", 0, true, ErrNotStaged
|
||||
}
|
||||
ss.armed = false
|
||||
ss.touched = s.now()
|
||||
return ss.file, ss.size, true, nil
|
||||
}
|
||||
|
||||
// Landed tells the stage the Job holding token has put session id's file in
|
||||
// place, and deletes the session: nothing will fetch it again. Only the token
|
||||
// the latest Seal minted says so, spent or not, so a Job an earlier commit
|
||||
// started, or anyone else, changes nothing (ErrNotStaged). An upload staged by
|
||||
// Put answers to its own token too and is left to the release that deletes it.
|
||||
func (s *Stage) Landed(id, token string) error {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
s.mu.Lock()
|
||||
ss, ok := s.sessions[id]
|
||||
if !ok {
|
||||
it, found := s.items[id]
|
||||
s.mu.Unlock()
|
||||
if !found || subtle.ConstantTimeCompare(sum[:], it.tokenHash[:]) != 1 {
|
||||
return ErrNotStaged
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// Before the first Seal tokenHash is zero, which no token hashes to. A sealed
|
||||
// session has every byte, so a part arriving now could only be an empty one
|
||||
// and leaves nothing to account for: no busy check, unlike Drop.
|
||||
if subtle.ConstantTimeCompare(sum[:], ss.tokenHash[:]) != 1 {
|
||||
s.mu.Unlock()
|
||||
return ErrNotStaged
|
||||
}
|
||||
s.dropLocked(id, ss)
|
||||
s.mu.Unlock()
|
||||
os.Remove(ss.file)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Drop cancels the caller's session and deletes what it holds.
|
||||
func (s *Stage) Drop(user, server, id string) error {
|
||||
s.mu.Lock()
|
||||
ss, err := s.lookup(user, server, id)
|
||||
if err == nil && ss.busy {
|
||||
err = ErrUploadBusy
|
||||
}
|
||||
if err != nil {
|
||||
s.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
s.dropLocked(id, ss)
|
||||
s.mu.Unlock()
|
||||
os.Remove(ss.file)
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropLocked forgets a session and gives back the room it still had reserved.
|
||||
// s.mu must be held; the caller deletes the file.
|
||||
func (s *Stage) dropLocked(id string, ss *session) {
|
||||
delete(s.sessions, id)
|
||||
s.reserved -= ss.size - ss.received
|
||||
}
|
||||
|
||||
// Expire drops every session untouched for SessionIdle, sealed or not, and
|
||||
// reports how many it dropped. A session a part is arriving for is never idle.
|
||||
func (s *Stage) Expire() int {
|
||||
cutoff := s.now().Add(-SessionIdle)
|
||||
var files []string
|
||||
s.mu.Lock()
|
||||
for id, ss := range s.sessions {
|
||||
if !ss.busy && ss.touched.Before(cutoff) {
|
||||
s.dropLocked(id, ss)
|
||||
files = append(files, ss.file)
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
for _, f := range files {
|
||||
os.Remove(f)
|
||||
}
|
||||
return len(files)
|
||||
}
|
||||
@@ -1,600 +0,0 @@
|
||||
package fileedit
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// diskStage is a stage on a disk of total bytes whose free space is what the
|
||||
// files in it leave of free: statfs sees parts land, as a real disk would.
|
||||
func diskStage(t *testing.T, free, total uint64, minFree float64) *Stage {
|
||||
t.Helper()
|
||||
s := &Stage{Dir: filepath.Join(t.TempDir(), "stage"), MinFree: minFree}
|
||||
prev := statfs
|
||||
statfs = func(string) (uint64, uint64, error) {
|
||||
var used uint64
|
||||
des, _ := os.ReadDir(s.Dir)
|
||||
for _, de := range des {
|
||||
if info, err := de.Info(); err == nil {
|
||||
used += uint64(info.Size())
|
||||
}
|
||||
}
|
||||
return free - used, total, nil
|
||||
}
|
||||
t.Cleanup(func() { statfs = prev })
|
||||
return s
|
||||
}
|
||||
|
||||
func appendString(s *Stage, user, server, id string, offset int64, part string) (Session, error) {
|
||||
return s.Append(user, server, id, offset, strings.NewReader(part), int64(len(part)), sumOf(part))
|
||||
}
|
||||
|
||||
func readStaged(t *testing.T, s *Stage, id, token string) string {
|
||||
t.Helper()
|
||||
f, size, err := s.Open(id, token)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
b, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if size != int64(len(b)) {
|
||||
t.Fatalf("Open said %d bytes and served %d", size, len(b))
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// TestSessionArrivesInParts: parts land in order and are listed with their
|
||||
// digests, Seal hands the Job a token for exactly those bytes, and they stay
|
||||
// until that Job reports them landed.
|
||||
func TestSessionArrivesInParts(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
const whole = "PK\x03\x04 first part, second part"
|
||||
sess, err := s.Begin("u1", "survival", "plugins/big.jar", int64(len(whole)))
|
||||
if err != nil {
|
||||
t.Fatalf("Begin: %v", err)
|
||||
}
|
||||
if !hexID.MatchString(sess.ID) || !reflect.DeepEqual(sess, Session{ID: sess.ID, Path: "plugins/big.jar", Size: int64(len(whole))}) {
|
||||
t.Fatalf("Begin = %+v", sess)
|
||||
}
|
||||
got, err := appendString(s, "u1", "survival", sess.ID, 0, whole[:16])
|
||||
if err != nil || got.Received != 16 || got.Size != int64(len(whole)) {
|
||||
t.Fatalf("first part: %+v, %v", got, err)
|
||||
}
|
||||
first := Part{Size: 16, SHA256: digest([]byte(whole[:16]))}
|
||||
if at, err := s.Status("u1", "survival", sess.ID); err != nil || at.Received != 16 || at.Path != "plugins/big.jar" ||
|
||||
!reflect.DeepEqual(at.Parts, []Part{first}) {
|
||||
t.Fatalf("Status = %+v, %v", at, err)
|
||||
}
|
||||
if got, err = appendString(s, "u1", "survival", sess.ID, 16, whole[16:]); err != nil || got.Received != int64(len(whole)) {
|
||||
t.Fatalf("second part: %+v, %v", got, err)
|
||||
}
|
||||
if want := []Part{first, {Size: int64(len(whole) - 16), SHA256: digest([]byte(whole[16:]))}}; !reflect.DeepEqual(got.Parts, want) {
|
||||
t.Fatalf("parts = %+v, want %+v", got.Parts, want)
|
||||
}
|
||||
|
||||
st, err := s.Seal("u1", "survival", sess.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Seal: %v", err)
|
||||
}
|
||||
if st.ID != sess.ID || !hexToken.MatchString(st.Token) || st.Size != int64(len(whole)) || st.SHA256 != digest([]byte(whole)) {
|
||||
t.Fatalf("Seal = %+v, want the digest of %q", st, whole)
|
||||
}
|
||||
if body := readStaged(t, s, st.ID, st.Token); body != whole {
|
||||
t.Fatalf("served %q, want %q", body, whole)
|
||||
}
|
||||
if _, _, err := s.Open(st.ID, st.Token); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("second Open with the same token: err = %v, want ErrNotStaged", err)
|
||||
}
|
||||
|
||||
// Served whole, and still here: the Job has yet to check the bytes and put
|
||||
// them in place, and a Job that fails at either is started again on them.
|
||||
if at, err := s.Status("u1", "survival", sess.ID); err != nil || at.Received != int64(len(whole)) {
|
||||
t.Fatalf("Status once served: %+v, %v", at, err)
|
||||
}
|
||||
if err := s.Landed(st.ID, strings.Repeat("0", 64)); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("Landed with a wrong token: err = %v, want ErrNotStaged", err)
|
||||
}
|
||||
if names := stagedNames(t, s); len(names) != 1 {
|
||||
t.Fatalf("after a wrong token: %v", names)
|
||||
}
|
||||
if err := s.Landed(st.ID, st.Token); err != nil {
|
||||
t.Fatalf("Landed: %v", err)
|
||||
}
|
||||
if names := stagedNames(t, s); len(names) != 0 {
|
||||
t.Fatalf("after Landed: %v", names)
|
||||
}
|
||||
if _, err := s.Status("u1", "survival", sess.ID); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("Status after Landed: err = %v, want ErrNotStaged", err)
|
||||
}
|
||||
if err := s.Landed(st.ID, st.Token); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("Landed twice: err = %v, want ErrNotStaged", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A session answers only the user and the server it was begun for.
|
||||
func TestSessionAnswersItsOwnerOnly(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
sess, err := s.Begin("u1", "survival", "a.zip", 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, who := range map[string][2]string{
|
||||
"another user": {"u2", "survival"},
|
||||
"another server": {"u1", "creative"},
|
||||
} {
|
||||
if _, err := s.Status(who[0], who[1], sess.ID); !errors.Is(err, ErrNotStaged) {
|
||||
t.Errorf("%s: Status err = %v", name, err)
|
||||
}
|
||||
if _, err := appendString(s, who[0], who[1], sess.ID, 0, "abcd"); !errors.Is(err, ErrNotStaged) {
|
||||
t.Errorf("%s: Append err = %v", name, err)
|
||||
}
|
||||
if _, err := s.Seal(who[0], who[1], sess.ID); !errors.Is(err, ErrNotStaged) {
|
||||
t.Errorf("%s: Seal err = %v", name, err)
|
||||
}
|
||||
if err := s.Drop(who[0], who[1], sess.ID); !errors.Is(err, ErrNotStaged) {
|
||||
t.Errorf("%s: Drop err = %v", name, err)
|
||||
}
|
||||
}
|
||||
if _, err := s.Status("u1", "survival", strings.Repeat("0", 32)); !errors.Is(err, ErrNotStaged) {
|
||||
t.Errorf("unknown id: err = %v", err)
|
||||
}
|
||||
if at, err := s.Status("u1", "survival", sess.ID); err != nil || at.Received != 0 {
|
||||
t.Fatalf("the owner's session after the others tried: %+v, %v", at, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionRefusesAPartThatDoesNotFit(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
sess, err := s.Begin("u1", "survival", "a.zip", 6)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 0, "abc"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var off *OffsetError
|
||||
for _, offset := range []int64{0, 2, 4} {
|
||||
at, err := appendString(s, "u1", "survival", sess.ID, offset, "d")
|
||||
if !errors.As(err, &off) || off.Received != 3 || at.Received != 3 {
|
||||
t.Fatalf("offset %d: %+v, err = %v; want an OffsetError at 3", offset, at, err)
|
||||
}
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 3, "defg"); !errors.Is(err, ErrPartTooLarge) {
|
||||
t.Fatalf("past the declared size: err = %v, want ErrPartTooLarge", err)
|
||||
}
|
||||
if _, err := s.Append("u1", "survival", sess.ID, 3, strings.NewReader(""), -1, sumOf("")); !errors.Is(err, ErrPartTooLarge) {
|
||||
t.Fatalf("negative length: err = %v, want ErrPartTooLarge", err)
|
||||
}
|
||||
if at, err := appendString(s, "u1", "survival", sess.ID, 3, "def"); err != nil || at.Received != 6 {
|
||||
t.Fatalf("the part that fits exactly: %+v, %v", at, err)
|
||||
}
|
||||
|
||||
big, err := s.Begin("u1", "survival", "b.zip", PartBytes+2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Append("u1", "survival", big.ID, 0, strings.NewReader(""), PartBytes+1, sumOf("")); !errors.Is(err, ErrPartTooLarge) {
|
||||
t.Fatalf("a part over PartBytes: err = %v, want ErrPartTooLarge", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A part that breaks, runs long or does not match its digest leaves the session
|
||||
// as it was: the file is cut back and the digest forgets it, so the resent part
|
||||
// makes the right file.
|
||||
func TestSessionRollsBackAFailedPart(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
body io.Reader
|
||||
want error // nil: any other failure
|
||||
}{
|
||||
"breaks": {io.MultiReader(strings.NewReader("XY"), errReader{io.ErrUnexpectedEOF}), ErrShortUpload},
|
||||
"ends": {strings.NewReader("XY"), ErrShortUpload},
|
||||
"runs long": {strings.NewReader("XYZWV"), nil},
|
||||
// Four bytes, as declared, that are not the four the digest was made of.
|
||||
"changed on the way": {strings.NewReader("dXfg"), ErrDigestMismatch},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
sess, err := s.Begin("u1", "survival", "a.zip", 7)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 0, "abc"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at, err := s.Append("u1", "survival", sess.ID, 3, tc.body, 4, sumOf("defg"))
|
||||
kind := tc.want
|
||||
if kind == nil {
|
||||
kind = ErrShortUpload // must not be it
|
||||
}
|
||||
if err == nil || errors.Is(err, kind) != (tc.want != nil) || at.Received != 3 || len(at.Parts) != 1 {
|
||||
t.Fatalf("%+v, err = %v; want a failure at 3 (%v)", at, err, tc.want)
|
||||
}
|
||||
info, err := os.Stat(filepath.Join(s.Dir, stagedNames(t, s)[0]))
|
||||
if err != nil || info.Size() != 3 {
|
||||
t.Fatalf("staged file is %v bytes (%v), want it cut back to 3", info.Size(), err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 3, "defg"); err != nil {
|
||||
t.Fatalf("resent part: %v", err)
|
||||
}
|
||||
st, err := s.Seal("u1", "survival", sess.ID)
|
||||
if err != nil || st.SHA256 != digest([]byte("abcdefg")) {
|
||||
t.Fatalf("Seal = %+v, %v; want the digest of abcdefg", st, err)
|
||||
}
|
||||
if body := readStaged(t, s, st.ID, st.Token); body != "abcdefg" {
|
||||
t.Fatalf("served %q", body)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// While a part is arriving nothing else may touch the session, and it is never
|
||||
// idle.
|
||||
func TestSessionIsBusyWhileAPartArrives(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
now := time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)
|
||||
s.Now = func() time.Time { return now }
|
||||
sess, err := s.Begin("u1", "survival", "a.zip", 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pr, pw := io.Pipe()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := s.Append("u1", "survival", sess.ID, 0, pr, 4, sumOf("abcd"))
|
||||
done <- err
|
||||
}()
|
||||
// The write returns once Append is copying, which is after it marked busy.
|
||||
if _, err := pw.Write([]byte("ab")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 0, "abcd"); !errors.Is(err, ErrUploadBusy) {
|
||||
t.Errorf("a second part: err = %v, want ErrUploadBusy", err)
|
||||
}
|
||||
if _, err := s.Seal("u1", "survival", sess.ID); !errors.Is(err, ErrUploadBusy) {
|
||||
t.Errorf("Seal: err = %v, want ErrUploadBusy", err)
|
||||
}
|
||||
if err := s.Drop("u1", "survival", sess.ID); !errors.Is(err, ErrUploadBusy) {
|
||||
t.Errorf("Drop: err = %v, want ErrUploadBusy", err)
|
||||
}
|
||||
now = now.Add(SessionIdle + time.Hour)
|
||||
if n := s.Expire(); n != 0 {
|
||||
t.Errorf("Expire dropped %d sessions with a part arriving", n)
|
||||
}
|
||||
if _, err := pw.Write([]byte("cd")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pw.Close()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("the part in flight: %v", err)
|
||||
}
|
||||
if _, err := s.Seal("u1", "survival", sess.ID); err != nil {
|
||||
t.Fatalf("Seal once the part is in: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Each Seal arms one fetch with a fresh token, so a Job that failed can be
|
||||
// started again on the same bytes.
|
||||
func TestSessionSealArmsOneFetch(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
sess, err := s.Begin("u1", "survival", "a.zip", 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 0, "abc"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Seal("u1", "survival", sess.ID); !errors.Is(err, ErrUploadIncomplete) {
|
||||
t.Fatalf("Seal at 3 of 4: err = %v, want ErrUploadIncomplete", err)
|
||||
}
|
||||
if _, _, err := s.Open(sess.ID, ""); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("Open before any Seal: err = %v, want ErrNotStaged", err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 3, "d"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
first, err := s.Seal("u1", "survival", sess.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := s.Seal("u1", "survival", sess.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.Token == second.Token || first.SHA256 != second.SHA256 {
|
||||
t.Fatalf("two Seals: %+v and %+v; want fresh tokens for the same bytes", first, second)
|
||||
}
|
||||
if _, _, err := s.Open(sess.ID, first.Token); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("the replaced token: err = %v, want ErrNotStaged", err)
|
||||
}
|
||||
if _, _, err := s.Open(sess.ID, strings.Repeat("0", 64)); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("a wrong token: err = %v, want ErrNotStaged", err)
|
||||
}
|
||||
// Neither wrong token spent the armed one.
|
||||
if body := readStaged(t, s, sess.ID, second.Token); body != "abcd" {
|
||||
t.Fatalf("served %q", body)
|
||||
}
|
||||
if _, _, err := s.Open(sess.ID, second.Token); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("the spent token: err = %v, want ErrNotStaged", err)
|
||||
}
|
||||
|
||||
// Opened but not served whole: the Job broke midway, and a new Seal serves
|
||||
// the same bytes again.
|
||||
third, err := s.Seal("u1", "survival", sess.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body := readStaged(t, s, sess.ID, third.Token); body != "abcd" {
|
||||
t.Fatalf("served %q after a new Seal", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the token the latest Seal minted reports a session landed: a Job an
|
||||
// earlier commit started changes nothing, and neither does anyone before the
|
||||
// first Seal.
|
||||
func TestLandedTakesTheLatestToken(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
sess, err := s.Begin("u1", "survival", "a.zip", 4)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 0, "abcd"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Landed(sess.ID, ""); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("Landed before any Seal: err = %v, want ErrNotStaged", err)
|
||||
}
|
||||
first, err := s.Seal("u1", "survival", sess.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readStaged(t, s, sess.ID, first.Token)
|
||||
second, err := s.Seal("u1", "survival", sess.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Landed(sess.ID, first.Token); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("the replaced token: err = %v, want ErrNotStaged", err)
|
||||
}
|
||||
if _, err := s.Status("u1", "survival", sess.ID); err != nil {
|
||||
t.Fatalf("after the replaced token: %v", err)
|
||||
}
|
||||
// Not yet fetched with it, and it still says so: the Job holds the token
|
||||
// whatever became of its fetch.
|
||||
if err := s.Landed(sess.ID, second.Token); err != nil {
|
||||
t.Fatalf("the latest token: %v", err)
|
||||
}
|
||||
if names := stagedNames(t, s); len(names) != 0 {
|
||||
t.Fatalf("after Landed: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// An upload staged by Put answers Landed to its own token and is left to the
|
||||
// release func Put returned.
|
||||
func TestLandedLeavesPutAlone(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
st, release, err := s.Put(strings.NewReader("abc"), 3, sumOf("abc"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release()
|
||||
if err := s.Landed(st.ID, strings.Repeat("0", 64)); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("a wrong token: err = %v, want ErrNotStaged", err)
|
||||
}
|
||||
if err := s.Landed(st.ID, st.Token); err != nil {
|
||||
t.Fatalf("Landed: %v", err)
|
||||
}
|
||||
if body := readStaged(t, s, st.ID, st.Token); body != "abc" {
|
||||
t.Fatalf("served %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
// A session reserves its whole size when it begins, and gives back what it has
|
||||
// not yet received when it is dropped or expires.
|
||||
func TestSessionReservesItsSize(t *testing.T) {
|
||||
t.Run("begin reserves the whole size", func(t *testing.T) {
|
||||
s := diskStage(t, 1000, 1200, 0.5) // room for 400
|
||||
if _, err := s.Begin("u1", "survival", "a.zip", 300); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Begin("u2", "survival", "b.zip", 101); !errors.Is(err, ErrStageFull) {
|
||||
t.Fatalf("101 bytes beside a 300-byte session: err = %v, want ErrStageFull", err)
|
||||
}
|
||||
if _, err := s.Begin("u2", "survival", "b.zip", 100); err != nil {
|
||||
t.Fatalf("100 bytes beside a 300-byte session: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a part moves its room from the reservation to the disk", func(t *testing.T) {
|
||||
s := diskStage(t, 1000, 1200, 0.5)
|
||||
sess, err := s.Begin("u1", "survival", "a.zip", 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 0, strings.Repeat("x", 200)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Begin("u2", "survival", "b.zip", 101); !errors.Is(err, ErrStageFull) {
|
||||
t.Fatalf("after a part landed: err = %v, want ErrStageFull", err)
|
||||
}
|
||||
if _, err := s.Begin("u2", "survival", "b.zip", 100); err != nil {
|
||||
t.Fatalf("after a part landed: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("drop gives it all back", func(t *testing.T) {
|
||||
s := diskStage(t, 1000, 1200, 0.5)
|
||||
sess, err := s.Begin("u1", "survival", "a.zip", 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 0, strings.Repeat("x", 200)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Drop("u1", "survival", sess.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if names := stagedNames(t, s); len(names) != 0 {
|
||||
t.Fatalf("after Drop: %v", names)
|
||||
}
|
||||
if _, err := s.Begin("u2", "survival", "b.zip", 400); err != nil {
|
||||
t.Fatalf("after Drop: %v", err)
|
||||
}
|
||||
if _, err := s.Status("u1", "survival", sess.ID); !errors.Is(err, ErrNotStaged) {
|
||||
t.Fatalf("Status after Drop: err = %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("expiry gives it all back", func(t *testing.T) {
|
||||
s := diskStage(t, 1000, 1200, 0.5)
|
||||
now := time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)
|
||||
s.Now = func() time.Time { return now }
|
||||
sess, err := s.Begin("u1", "survival", "a.zip", 300)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 0, strings.Repeat("x", 200)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = now.Add(SessionIdle + time.Nanosecond)
|
||||
if n := s.Expire(); n != 1 {
|
||||
t.Fatalf("Expire dropped %d, want 1", n)
|
||||
}
|
||||
if _, err := s.Begin("u2", "survival", "b.zip", 400); err != nil {
|
||||
t.Fatalf("after Expire: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestSessionsPerUserAreBounded(t *testing.T) {
|
||||
s := diskStage(t, 1000, 1200, 0.5)
|
||||
var ids []string
|
||||
for i := range MaxSessionsPerUser {
|
||||
sess, err := s.Begin("u1", "survival", "a.zip", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("session %d: %v", i+1, err)
|
||||
}
|
||||
ids = append(ids, sess.ID)
|
||||
}
|
||||
if _, err := s.Begin("u1", "creative", "a.zip", 10); !errors.Is(err, ErrTooManySessions) {
|
||||
t.Fatalf("one more on another server: err = %v, want ErrTooManySessions", err)
|
||||
}
|
||||
// The refused Begin kept neither a file nor its reservation: room for 400
|
||||
// less the four sessions' 40.
|
||||
if names := stagedNames(t, s); len(names) != MaxSessionsPerUser {
|
||||
t.Fatalf("staged %d files, want %d", len(names), MaxSessionsPerUser)
|
||||
}
|
||||
if _, err := s.Begin("u2", "survival", "b.zip", 360); err != nil {
|
||||
t.Fatalf("another user: %v", err)
|
||||
}
|
||||
if err := s.Drop("u1", "survival", ids[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Begin("u1", "survival", "a.zip", 10); err != nil {
|
||||
t.Fatalf("after dropping one: %v", err)
|
||||
}
|
||||
// With room reserved, a negative size would wrap the reservation sum round
|
||||
// to a small number and pass the room check.
|
||||
if _, err := s.Begin("u3", "survival", "a.zip", -1); err == nil || errors.Is(err, ErrStageFull) {
|
||||
t.Fatalf("a negative size: err = %v, want it refused for being negative", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Expire drops what has sat untouched for longer than SessionIdle, sealed or
|
||||
// not, and a part keeps a session alive.
|
||||
func TestSessionExpiry(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
start := time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)
|
||||
now := start
|
||||
s.Now = func() time.Time { return now }
|
||||
idle, err := s.Begin("u1", "survival", "idle.zip", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sealed, err := s.Begin("u1", "survival", "sealed.zip", 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sealed.ID, 0, "x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Seal("u1", "survival", sealed.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
active, err := s.Begin("u1", "survival", "active.zip", 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
now = start.Add(time.Hour)
|
||||
if _, err := appendString(s, "u1", "survival", active.ID, 0, "a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = start.Add(SessionIdle)
|
||||
if n := s.Expire(); n != 0 {
|
||||
t.Fatalf("at exactly SessionIdle Expire dropped %d", n)
|
||||
}
|
||||
now = start.Add(SessionIdle + time.Minute)
|
||||
if n := s.Expire(); n != 2 {
|
||||
t.Fatalf("Expire dropped %d, want the idle and the sealed one", n)
|
||||
}
|
||||
for _, id := range []string{idle.ID, sealed.ID} {
|
||||
if _, err := s.Status("u1", "survival", id); !errors.Is(err, ErrNotStaged) {
|
||||
t.Errorf("%s survived Expire: %v", id, err)
|
||||
}
|
||||
}
|
||||
if at, err := s.Status("u1", "survival", active.ID); err != nil || at.Received != 1 {
|
||||
t.Fatalf("the session a part touched: %+v, %v", at, err)
|
||||
}
|
||||
if names := stagedNames(t, s); len(names) != 1 {
|
||||
t.Fatalf("files left: %v, want the active session's", names)
|
||||
}
|
||||
|
||||
// A Seal, and the Job's Open, each start the idle clock again: a Job begun
|
||||
// on an upload that sat for hours still finds it there.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
// sealAt and openAt are how long after the last part the Seal and the
|
||||
// Job's Open come; a negative openAt is no Open.
|
||||
sealAt, openAt time.Duration
|
||||
}{
|
||||
{"seal", 4 * time.Hour, -1},
|
||||
{"open", 0, 4 * time.Hour},
|
||||
} {
|
||||
begun := now
|
||||
sess, err := s.Begin("u1", "survival", tc.name+".zip", 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := appendString(s, "u1", "survival", sess.ID, 0, "x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now = begun.Add(tc.sealAt)
|
||||
st, err := s.Seal("u1", "survival", sess.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.openAt >= 0 {
|
||||
now = begun.Add(tc.openAt)
|
||||
readStaged(t, s, sess.ID, st.Token)
|
||||
}
|
||||
now = begun.Add(SessionIdle + time.Minute)
|
||||
s.Expire()
|
||||
if _, err := s.Status("u1", "survival", sess.ID); err != nil {
|
||||
t.Errorf("%s: a session touched 4h after its last part expired 6h after it: %v", tc.name, err)
|
||||
}
|
||||
if err := s.Drop("u1", "survival", sess.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
+20
-63
@@ -12,7 +12,6 @@ import (
|
||||
"os"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Stage holds uploads between the request that brought them and the Job that
|
||||
@@ -26,9 +25,7 @@ import (
|
||||
// Job carries in its environment. Only a digest of the token is kept, compared in
|
||||
// constant time, and the first successful Open spends it: the Job never retries,
|
||||
// so a second Open could only be someone else. The release func Put returns
|
||||
// deletes the file once the Job has answered, whatever it answered. A file too
|
||||
// big for one request arrives in parts instead (session.go) and is fetched the
|
||||
// same way.
|
||||
// deletes the file once the Job has answered, whatever it answered.
|
||||
//
|
||||
// Nothing here outlives the process: the index is in memory, so Sweep empties
|
||||
// Dir at startup of whatever a previous process left behind.
|
||||
@@ -41,12 +38,8 @@ type Stage struct {
|
||||
// the submission store shares.
|
||||
MinFree float64
|
||||
|
||||
// Now is the clock sessions are aged by (time.Now when nil).
|
||||
Now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
items map[string]*stagedFile
|
||||
sessions map[string]*session
|
||||
reserved int64
|
||||
}
|
||||
|
||||
@@ -79,22 +72,8 @@ var (
|
||||
// ErrNotStaged is an Open with an unknown id, a wrong token, or a spent one.
|
||||
// They are one error on purpose: the internal face answers all three the same.
|
||||
ErrNotStaged = errors.New("fileedit: no such staged upload")
|
||||
// ErrNoDigest is bytes sent without the SHA-256 the client computed over
|
||||
// them, so what arrived cannot be told apart from what was sent.
|
||||
ErrNoDigest = errors.New("fileedit: the upload carries no SHA-256 digest")
|
||||
// ErrDigestMismatch is bytes that do not hash to the digest they were sent
|
||||
// with: they were changed on the way.
|
||||
ErrDigestMismatch = errors.New("fileedit: the bytes that arrived do not match the digest they were sent with")
|
||||
)
|
||||
|
||||
// checkDigest compares the digest of what arrived with the one it was sent with.
|
||||
func checkDigest(got, want []byte) error {
|
||||
if subtle.ConstantTimeCompare(got, want) != 1 {
|
||||
return fmt.Errorf("%w: they hash to %x, sent as %x", ErrDigestMismatch, got, want)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// statfs reports a filesystem's available and total bytes. A var so a test can
|
||||
// stage against a disk of a chosen size.
|
||||
var statfs = func(dir string) (avail, total uint64, err error) {
|
||||
@@ -118,17 +97,10 @@ func (s *Stage) Sweep() error {
|
||||
// it by, plus the func that deletes it. body must end right after size bytes (an
|
||||
// HTTP body with that Content-Length does): Put reads to its end, which is also
|
||||
// what tells the server the body is done.
|
||||
//
|
||||
// want is the SHA-256 the client computed over the bytes it sent (the request's
|
||||
// Content-Digest). Bytes that hash to anything else were changed on the way and
|
||||
// are refused with ErrDigestMismatch; nothing is staged without one.
|
||||
func (s *Stage) Put(body io.Reader, size int64, want []byte) (Staged, func(), error) {
|
||||
func (s *Stage) Put(body io.Reader, size int64) (Staged, func(), error) {
|
||||
if size < 0 {
|
||||
return Staged{}, nil, fmt.Errorf("fileedit: an upload of %d bytes", size)
|
||||
}
|
||||
if len(want) != sha256.Size {
|
||||
return Staged{}, nil, ErrNoDigest
|
||||
}
|
||||
if err := os.MkdirAll(s.Dir, 0o700); err != nil {
|
||||
return Staged{}, nil, fmt.Errorf("fileedit: create the upload stage: %w", err)
|
||||
}
|
||||
@@ -146,11 +118,7 @@ func (s *Stage) Put(body io.Reader, size int64, want []byte) (Staged, func(), er
|
||||
// One byte past size, so the read that finds the end happens here.
|
||||
n, copyErr := io.Copy(io.MultiWriter(f, h), io.LimitReader(src, size+1))
|
||||
closeErr := f.Close()
|
||||
err = stageFailure(src.err, copyErr, closeErr, n, size)
|
||||
if err == nil {
|
||||
err = checkDigest(h.Sum(nil), want)
|
||||
}
|
||||
if err != nil {
|
||||
if err := stageFailure(src.err, copyErr, closeErr, n, size); err != nil {
|
||||
os.Remove(f.Name())
|
||||
return Staged{}, nil, err
|
||||
}
|
||||
@@ -204,27 +172,23 @@ func stageFailure(readErr, copyErr, closeErr error, n, size int64) error {
|
||||
|
||||
// newHandle mints the id and token for a staged upload whose bytes h hashed.
|
||||
func newHandle(h hash.Hash, size int64) (Staged, [sha256.Size]byte, error) {
|
||||
id, err := randomHex(16)
|
||||
if err != nil {
|
||||
var id [16]byte
|
||||
var token [32]byte
|
||||
if _, err := rand.Read(id[:]); err != nil {
|
||||
return Staged{}, [sha256.Size]byte{}, fmt.Errorf("fileedit: generate an upload id: %w", err)
|
||||
}
|
||||
token, err := randomHex(32)
|
||||
if err != nil {
|
||||
if _, err := rand.Read(token[:]); err != nil {
|
||||
return Staged{}, [sha256.Size]byte{}, fmt.Errorf("fileedit: generate an upload token: %w", err)
|
||||
}
|
||||
st := Staged{ID: id, Token: token, SHA256: hex.EncodeToString(h.Sum(nil)), Size: size}
|
||||
st := Staged{
|
||||
ID: hex.EncodeToString(id[:]),
|
||||
Token: hex.EncodeToString(token[:]),
|
||||
SHA256: hex.EncodeToString(h.Sum(nil)),
|
||||
Size: size,
|
||||
}
|
||||
return st, sha256.Sum256([]byte(st.Token)), nil
|
||||
}
|
||||
|
||||
// randomHex is n random bytes in hex.
|
||||
func randomHex(n int) (string, error) {
|
||||
b := make([]byte, n)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
// reserve admits an upload of size bytes if the disk keeps MinFree free after it
|
||||
// and after every upload still being written. Those have not reached the disk
|
||||
// yet, so statfs alone would let two of them through on room for one.
|
||||
@@ -255,30 +219,23 @@ func (s *Stage) unreserve(size int64) {
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// Open spends a staged upload's token, or a sealed session's (Seal), and returns
|
||||
// its file and size. Any mismatch is ErrNotStaged.
|
||||
// Open spends a staged upload's token and returns its file and size. Any
|
||||
// mismatch is ErrNotStaged.
|
||||
func (s *Stage) Open(id, token string) (*os.File, int64, error) {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
s.mu.Lock()
|
||||
name, size, found, err := s.openSession(id, sum)
|
||||
if !found {
|
||||
it, ok := s.items[id]
|
||||
if !ok || it.used || subtle.ConstantTimeCompare(sum[:], it.tokenHash[:]) != 1 {
|
||||
err = ErrNotStaged
|
||||
} else {
|
||||
it.used = true
|
||||
name, size = it.path, it.size
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
return nil, 0, ErrNotStaged
|
||||
}
|
||||
f, err := os.Open(name)
|
||||
it.used = true
|
||||
s.mu.Unlock()
|
||||
f, err := os.Open(it.path)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("fileedit: open the staged upload: %w", err)
|
||||
}
|
||||
return f, size, nil
|
||||
return f, it.size, nil
|
||||
}
|
||||
|
||||
// bodyReader remembers the body's own read error, so Put can tell a client that
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package fileedit
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
@@ -40,12 +39,6 @@ func stagedNames(t *testing.T, s *Stage) []string {
|
||||
return names
|
||||
}
|
||||
|
||||
// sumOf is the SHA-256 a client sends with s.
|
||||
func sumOf(s string) []byte {
|
||||
sum := sha256.Sum256([]byte(s))
|
||||
return sum[:]
|
||||
}
|
||||
|
||||
var hexID = regexp.MustCompile(`^[0-9a-f]{32}$`)
|
||||
var hexToken = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
|
||||
@@ -54,7 +47,7 @@ var hexToken = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
func TestStageOpensOnce(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
const body = "PK\x03\x04 staged"
|
||||
st, release, err := s.Put(strings.NewReader(body), int64(len(body)), sumOf(body))
|
||||
st, release, err := s.Put(strings.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
@@ -62,7 +55,7 @@ func TestStageOpensOnce(t *testing.T) {
|
||||
st.Size != int64(len(body)) || st.SHA256 != digest([]byte(body)) {
|
||||
t.Fatalf("staged = %+v", st)
|
||||
}
|
||||
other, releaseOther, err := s.Put(strings.NewReader(body), int64(len(body)), sumOf(body))
|
||||
other, releaseOther, err := s.Put(strings.NewReader(body), int64(len(body)))
|
||||
if err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
@@ -111,7 +104,7 @@ func TestStageOpensOnce(t *testing.T) {
|
||||
// is readable by anyone but felis-api's own uid.
|
||||
func TestStageIsPrivate(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
_, release, err := s.Put(strings.NewReader("x"), 1, sumOf("x"))
|
||||
_, release, err := s.Put(strings.NewReader("x"), 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
@@ -127,14 +120,13 @@ func TestStageIsPrivate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// eofReader serves r and records whether it was read at all, and to its end.
|
||||
// eofReader serves body and records whether it was read to its end.
|
||||
type eofReader struct {
|
||||
r io.Reader
|
||||
read, hitEOF bool
|
||||
hitEOF bool
|
||||
}
|
||||
|
||||
func (e *eofReader) Read(p []byte) (int, error) {
|
||||
e.read = true
|
||||
n, err := e.r.Read(p)
|
||||
if err == io.EOF {
|
||||
e.hitEOF = true
|
||||
@@ -148,7 +140,7 @@ func (e *eofReader) Read(p []byte) (int, error) {
|
||||
func TestStagePutReadsToTheEnd(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
body := &eofReader{r: strings.NewReader("abc")}
|
||||
_, release, err := s.Put(body, 3, sumOf("abc"))
|
||||
_, release, err := s.Put(body, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
@@ -158,37 +150,6 @@ func TestStagePutReadsToTheEnd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Bytes that do not hash to the digest they came with were changed on the way:
|
||||
// nothing is staged and their room is given back. Without a digest the body is
|
||||
// not read at all.
|
||||
func TestStageChecksTheDigest(t *testing.T) {
|
||||
stubStatfs(t, 1000, 1200) // floor 600 at MinFree 0.5: room for 400
|
||||
s := &Stage{Dir: t.TempDir(), MinFree: 0.5}
|
||||
body := strings.Repeat("x", 400)
|
||||
_, _, err := s.Put(strings.NewReader(body), 400, sumOf(strings.Repeat("y", 400)))
|
||||
if !errors.Is(err, ErrDigestMismatch) {
|
||||
t.Fatalf("a wrong digest: err = %v, want ErrDigestMismatch", err)
|
||||
}
|
||||
if names := stagedNames(t, s); len(names) != 0 {
|
||||
t.Fatalf("left behind: %v", names)
|
||||
}
|
||||
st, release, err := s.Put(strings.NewReader(body), 400, sumOf(body))
|
||||
if err != nil {
|
||||
t.Fatalf("the same bytes with their digest, in the room the refused ones held: %v", err)
|
||||
}
|
||||
defer release()
|
||||
if st.SHA256 != digest([]byte(body)) {
|
||||
t.Fatalf("staged %+v", st)
|
||||
}
|
||||
|
||||
for name, want := range map[string][]byte{"none": nil, "too short": sumOf("x")[:31]} {
|
||||
unread := &eofReader{r: strings.NewReader("abc")}
|
||||
if _, _, err := roomyStage(t).Put(unread, 3, want); !errors.Is(err, ErrNoDigest) || unread.read {
|
||||
t.Errorf("%s: err = %v, body read = %v; want ErrNoDigest before any read", name, err, unread.read)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStageRefusesABodyOfTheWrongLength(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
body io.Reader
|
||||
@@ -203,7 +164,7 @@ func TestStageRefusesABodyOfTheWrongLength(t *testing.T) {
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s := roomyStage(t)
|
||||
_, release, err := s.Put(tc.body, tc.size, sumOf(""))
|
||||
_, release, err := s.Put(tc.body, tc.size)
|
||||
if err == nil {
|
||||
release()
|
||||
t.Fatal("Put accepted it")
|
||||
@@ -216,7 +177,7 @@ func TestStageRefusesABodyOfTheWrongLength(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, _, err := roomyStage(t).Put(strings.NewReader(""), -1, sumOf("")); err == nil {
|
||||
if _, _, err := roomyStage(t).Put(strings.NewReader(""), -1); err == nil {
|
||||
t.Fatal("a negative size was accepted")
|
||||
}
|
||||
}
|
||||
@@ -225,7 +186,7 @@ func TestStageRefusesABodyOfTheWrongLength(t *testing.T) {
|
||||
// MinFree of the disk free, counting uploads still arriving.
|
||||
func TestStageKeepsItsFloor(t *testing.T) {
|
||||
put := func(s *Stage, size int64) error {
|
||||
_, release, err := s.Put(strings.NewReader(strings.Repeat("x", int(size))), size, sumOf(strings.Repeat("x", int(size))))
|
||||
_, release, err := s.Put(strings.NewReader(strings.Repeat("x", int(size))), size)
|
||||
if err == nil {
|
||||
release()
|
||||
}
|
||||
@@ -270,7 +231,7 @@ func TestStageKeepsItsFloor(t *testing.T) {
|
||||
pr, pw := io.Pipe()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, release, err := s.Put(pr, 300, sumOf(strings.Repeat("x", 300)))
|
||||
_, release, err := s.Put(pr, 300)
|
||||
if err == nil {
|
||||
release()
|
||||
}
|
||||
@@ -310,7 +271,7 @@ func TestStageSweep(t *testing.T) {
|
||||
if names := stagedNames(t, s); len(names) != 0 {
|
||||
t.Fatalf("after Sweep: %v", names)
|
||||
}
|
||||
if _, release, err := s.Put(strings.NewReader("x"), 1, sumOf("x")); err != nil {
|
||||
if _, release, err := s.Put(strings.NewReader("x"), 1); err != nil {
|
||||
t.Fatalf("Put after Sweep: %v", err)
|
||||
} else {
|
||||
release()
|
||||
|
||||
@@ -1,556 +0,0 @@
|
||||
package fileedit
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
)
|
||||
|
||||
// The codes an unzip refuses an archive with. Each names what is wrong with the
|
||||
// ARCHIVE, so the panel can say "this zip is broken" rather than "your path is
|
||||
// wrong"; Result.Entry names the entry.
|
||||
const (
|
||||
// CodeArchiveInvalid is a file that is not a zip, a damaged one, one whose
|
||||
// entry bytes disagree with their header (size or CRC), or one with nothing in it.
|
||||
CodeArchiveInvalid = "archive_invalid"
|
||||
// CodeArchiveUnsafe is an entry naming a path outside the folder it is
|
||||
// extracted into ("../", "/etc/x", "C:\x"), or a device, pipe or socket.
|
||||
CodeArchiveUnsafe = "archive_unsafe"
|
||||
// CodeArchiveSymlink is an entry that is a symbolic link. A link extracted into
|
||||
// the world could point anywhere, and every later op would have to reason about
|
||||
// it, so an archive carrying one is refused whole.
|
||||
CodeArchiveSymlink = "archive_symlink"
|
||||
// CodeTypeConflict is an archive with a file where the server has a folder, a
|
||||
// folder where it has a file, or anything where it has a link. Overwrite never
|
||||
// resolves it: replacing a folder with a file would delete the folder.
|
||||
CodeTypeConflict = "type_conflict"
|
||||
)
|
||||
|
||||
// MaxConflicts bounds Result.Conflicts, keeping the result line bounded when an
|
||||
// archive would replace a whole world; ConflictCount still says how many there
|
||||
// are. maxConflictBytes bounds the names listed as well: felis-api reads an
|
||||
// unzip's result from the last opLogLines lines of its Pod's log, and the
|
||||
// container runtime splits a line longer than 16 KiB into several, so 200 long
|
||||
// paths would push the result marker out of that tail and the op would read as
|
||||
// ended without a result. A file that exists has a path under PATH_MAX (4 KiB),
|
||||
// so the first conflict always fits.
|
||||
const (
|
||||
MaxConflicts = 200
|
||||
maxConflictBytes = 8 << 10
|
||||
)
|
||||
|
||||
// unzipEntryOverhead is what the space check adds per entry for the inode and
|
||||
// directory block it takes beyond its bytes.
|
||||
const unzipEntryOverhead = 4096
|
||||
|
||||
// unzipTempPrefix names the working folder an unzip extracts into. It sits in
|
||||
// the destination folder, so every move out of it is a rename on one volume.
|
||||
const unzipTempPrefix = ".felis-unzip-"
|
||||
|
||||
// unzip extracts the .zip at name into the folder holding it.
|
||||
//
|
||||
// It is all or nothing. Every check that can refuse the archive runs before a
|
||||
// byte is written: entry names, entry types, what is already on the server, and
|
||||
// the room on the volume. The entries are then extracted into a working folder
|
||||
// beside the destination, and only once every one of them has been written and
|
||||
// verified are they renamed into place. A failure at any point before that
|
||||
// leaves the destination exactly as it was. The renames themselves are journaled
|
||||
// and undone in reverse if one fails, so a replaced file comes back. Only a Job
|
||||
// killed in the middle of the renames — a few milliseconds for thousands of
|
||||
// files — can leave the archive half applied.
|
||||
//
|
||||
// An existing file the archive would replace is a conflict: without overwrite the
|
||||
// unzip lists them (CodeExists, Conflicts) and changes nothing; with it they are
|
||||
// replaced. Folders merge. A file where the server has a folder, or the reverse,
|
||||
// is CodeTypeConflict whatever overwrite says.
|
||||
//
|
||||
// The only size bound is the volume. Each entry's declared size is summed and
|
||||
// checked against the free space up front, and archive/zip itself refuses an
|
||||
// entry whose bytes run past its declared size or fail its CRC, so an archive
|
||||
// that lies about its sizes (a zip bomb) stops at the first lying entry and
|
||||
// nothing it wrote survives.
|
||||
func unzip(r *os.Root, rootPath, name string, overwrite bool, progress func(done, total int64)) Result {
|
||||
name = path.Clean(name)
|
||||
if !strings.EqualFold(path.Ext(name), ".zip") {
|
||||
return Result{Code: CodeBadPath, Error: fmt.Sprintf("%s is not a .zip archive", name)}
|
||||
}
|
||||
f, err := r.Open(name)
|
||||
if err != nil {
|
||||
return failure(err, name)
|
||||
}
|
||||
defer f.Close()
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return failure(err, name)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return Result{Code: CodeBadPath, Error: fmt.Sprintf("%s is not a file", name)}
|
||||
}
|
||||
// ErrInsecurePath comes back WITH a usable reader, and only under
|
||||
// GODEBUG=zipinsecurepath=0; planUnzip does that check itself, for every
|
||||
// entry, whatever the setting.
|
||||
zr, err := zip.NewReader(f, info.Size())
|
||||
if err != nil && !errors.Is(err, zip.ErrInsecurePath) {
|
||||
return Result{Code: CodeArchiveInvalid, Error: fmt.Sprintf("%s is not a readable zip archive: %v", name, err)}
|
||||
}
|
||||
p, res := planUnzip(zr.File)
|
||||
if res.Code != "" {
|
||||
return res
|
||||
}
|
||||
|
||||
dest := path.Dir(name)
|
||||
present, replaced, res := checkTargets(r, dest, p)
|
||||
if res.Code != "" {
|
||||
return res
|
||||
}
|
||||
if len(replaced) > 0 && !overwrite {
|
||||
list := make([]string, 0, len(replaced))
|
||||
for n := range replaced {
|
||||
list = append(list, path.Join(dest, n))
|
||||
}
|
||||
sort.Strings(list)
|
||||
count, n, size := len(list), 0, 0
|
||||
for n < count && n < MaxConflicts && size+len(list[n]) <= maxConflictBytes {
|
||||
size += len(list[n])
|
||||
n++
|
||||
}
|
||||
list = list[:n]
|
||||
return Result{Code: CodeExists, Conflicts: list, ConflictCount: count, Error: fmt.Sprintf(
|
||||
"%d files in the archive already exist on the server; extract again with overwrite to replace them", count)}
|
||||
}
|
||||
|
||||
// A working folder left by an unzip that was killed holds only a copy, and
|
||||
// clearing it first gives its room back to the check below.
|
||||
sweepUnzipTemps(r, dest)
|
||||
need := p.bytes + int64(len(p.files)+len(p.dirs))*unzipEntryOverhead
|
||||
if avail, _, err := statfs(rootPath); err == nil && uint64(need) > avail {
|
||||
free := int64(math.MaxInt64)
|
||||
if avail < math.MaxInt64 {
|
||||
free = int64(avail)
|
||||
}
|
||||
return Result{Code: CodeNoSpace, Need: need, Avail: free, Error: fmt.Sprintf(
|
||||
"extracting %s needs %d bytes and the server's volume has %d free; nothing was changed", name, need, free)}
|
||||
}
|
||||
|
||||
var suffix [6]byte
|
||||
if _, err := rand.Read(suffix[:]); err != nil {
|
||||
return Result{Code: CodeBadPath, Error: fmt.Sprintf("generate a temporary name: %v", err)}
|
||||
}
|
||||
tmp := path.Join(dest, unzipTempPrefix+hex.EncodeToString(suffix[:]))
|
||||
staged, old := path.Join(tmp, "new"), path.Join(tmp, "old")
|
||||
for _, d := range []string{tmp, staged, old} {
|
||||
if err := r.Mkdir(d, 0o700); err != nil {
|
||||
_ = r.RemoveAll(tmp)
|
||||
return unzipWriteFailure(err, dest)
|
||||
}
|
||||
}
|
||||
// After a success tmp holds only the files the archive replaced; after a
|
||||
// failure, everything the archive wrote. Either way it goes.
|
||||
defer func() { _ = r.RemoveAll(tmp) }()
|
||||
|
||||
if res := extractAll(r, staged, p, progress); res.Code != "" {
|
||||
return res
|
||||
}
|
||||
if res := placeAll(r, dest, staged, old, p, present, replaced); res.Code != "" {
|
||||
return res
|
||||
}
|
||||
return Result{Files: len(p.files), Bytes: p.bytes}
|
||||
}
|
||||
|
||||
// unzipPlan is an archive's entries once every one has passed planUnzip. Names
|
||||
// are cleaned, slash-separated and relative to the destination folder.
|
||||
type unzipPlan struct {
|
||||
files []zipFile
|
||||
// dirs is every folder the archive makes, named in it or implied by a file
|
||||
// inside it, sorted so a folder comes before everything in it.
|
||||
dirs []string
|
||||
isDir map[string]bool
|
||||
bytes int64 // the declared size of every file, summed
|
||||
}
|
||||
|
||||
type zipFile struct {
|
||||
f *zip.File
|
||||
name string
|
||||
mode fs.FileMode
|
||||
}
|
||||
|
||||
// planUnzip checks every entry's name and type and works out what the archive
|
||||
// makes. Nothing about the server is consulted yet.
|
||||
func planUnzip(entries []*zip.File) (unzipPlan, Result) {
|
||||
p := unzipPlan{isDir: map[string]bool{}}
|
||||
for _, f := range entries {
|
||||
raw := entryName(f)
|
||||
name, ok := cleanEntry(raw)
|
||||
if !ok {
|
||||
return p, Result{Code: CodeArchiveUnsafe, Entry: raw, Error: fmt.Sprintf(
|
||||
"%s leads outside the folder it would be extracted into", raw)}
|
||||
}
|
||||
// macOS's Finder adds __MACOSX/ to every zip it makes: resource forks that
|
||||
// mean nothing on the server.
|
||||
if name == "__MACOSX" || strings.HasPrefix(name, "__MACOSX/") {
|
||||
continue
|
||||
}
|
||||
mode := f.Mode()
|
||||
isDir := mode.IsDir() || strings.HasSuffix(raw, "/")
|
||||
switch {
|
||||
case mode&fs.ModeSymlink != 0:
|
||||
return p, Result{Code: CodeArchiveSymlink, Entry: raw, Error: fmt.Sprintf(
|
||||
"%s is a symbolic link; an archive containing links is not extracted", raw)}
|
||||
case isDir:
|
||||
if name != "." {
|
||||
p.isDir[name] = true
|
||||
}
|
||||
continue
|
||||
case !mode.IsRegular():
|
||||
return p, Result{Code: CodeArchiveUnsafe, Entry: raw, Error: fmt.Sprintf(
|
||||
"%s is not a regular file", raw)}
|
||||
case name == ".":
|
||||
return p, Result{Code: CodeArchiveUnsafe, Entry: raw, Error: fmt.Sprintf(
|
||||
"%s names the destination folder itself", raw)}
|
||||
case f.UncompressedSize64 > uint64(math.MaxInt64-p.bytes):
|
||||
return p, Result{Code: CodeArchiveInvalid, Entry: raw, Error: fmt.Sprintf(
|
||||
"%s declares an impossible size", raw)}
|
||||
}
|
||||
p.bytes += int64(f.UncompressedSize64)
|
||||
// Anything the archive marks executable (a start.sh) stays executable;
|
||||
// every other permission is the server's usual.
|
||||
perm := fs.FileMode(0o644)
|
||||
if mode.Perm()&0o111 != 0 {
|
||||
perm = 0o755
|
||||
}
|
||||
p.files = append(p.files, zipFile{f: f, name: name, mode: perm})
|
||||
}
|
||||
// Sorted, a name the archive holds twice sits next to itself; a set of names
|
||||
// would cost as much again as the entries for an archive of many small files.
|
||||
sort.Slice(p.files, func(i, j int) bool { return p.files[i].name < p.files[j].name })
|
||||
for i := 1; i < len(p.files); i++ {
|
||||
if p.files[i].name == p.files[i-1].name {
|
||||
return p, Result{Code: CodeArchiveInvalid, Entry: p.files[i].name, Error: fmt.Sprintf(
|
||||
"%s appears in the archive twice", p.files[i].name)}
|
||||
}
|
||||
}
|
||||
for _, zf := range p.files {
|
||||
// cleanEntry already refused a rooted name; stopping at "/" as well keeps
|
||||
// this loop finite should that check ever move.
|
||||
for d := path.Dir(zf.name); d != "." && d != "/"; d = path.Dir(d) {
|
||||
p.isDir[d] = true
|
||||
}
|
||||
}
|
||||
for _, zf := range p.files {
|
||||
if p.isDir[zf.name] {
|
||||
return p, Result{Code: CodeArchiveInvalid, Entry: zf.name, Error: fmt.Sprintf(
|
||||
"%s is both a file and a folder in the archive", zf.name)}
|
||||
}
|
||||
}
|
||||
if len(p.files) == 0 && len(p.isDir) == 0 {
|
||||
return p, Result{Code: CodeArchiveInvalid, Error: "the archive has nothing to extract"}
|
||||
}
|
||||
for d := range p.isDir {
|
||||
p.dirs = append(p.dirs, d)
|
||||
}
|
||||
// A folder's name is a prefix of everything in it, and a prefix sorts first.
|
||||
sort.Strings(p.dirs)
|
||||
return p, Result{}
|
||||
}
|
||||
|
||||
// entryName is an entry's name as its maker meant it. A zip made on Chinese
|
||||
// Windows stores names in the system code page (GBK) without the UTF-8 flag; a
|
||||
// name that is not valid UTF-8 is decoded as GB18030, GBK's superset. The flag
|
||||
// alone is no guide: archive/zip reports NonUTF8 for every name made without it,
|
||||
// which includes plain ASCII and macOS's UTF-8. Decoding comes before the
|
||||
// backslash below because a GBK trail byte can be 0x5C.
|
||||
func entryName(f *zip.File) string {
|
||||
name := f.Name
|
||||
if !utf8.ValidString(name) {
|
||||
if s, err := simplifiedchinese.GB18030.NewDecoder().String(name); err == nil {
|
||||
name = s
|
||||
}
|
||||
}
|
||||
return strings.ReplaceAll(name, `\`, "/")
|
||||
}
|
||||
|
||||
// cleanEntry cleans an entry name and reports false for one that must not be
|
||||
// extracted: a NUL, an absolute path, a drive letter, or a climb out of the
|
||||
// destination. It judges the name as TEXT, which is sound here because nothing
|
||||
// is resolved through it until checkTargets and extraction, and those go through
|
||||
// os.Root, which would refuse an escape anyway.
|
||||
func cleanEntry(raw string) (string, bool) {
|
||||
if raw == "" || strings.ContainsRune(raw, 0) || strings.HasPrefix(raw, "/") {
|
||||
return "", false
|
||||
}
|
||||
if len(raw) >= 2 && raw[1] == ':' && (raw[0]|0x20) >= 'a' && (raw[0]|0x20) <= 'z' {
|
||||
return "", false
|
||||
}
|
||||
name := path.Clean(raw)
|
||||
if name == ".." || strings.HasPrefix(name, "../") {
|
||||
return "", false
|
||||
}
|
||||
return name, true
|
||||
}
|
||||
|
||||
// checkTargets looks at what the server already has where the archive lands.
|
||||
// present is the archive's folders that exist on the server (they merge);
|
||||
// replaced is its files that do (conflicts). A folder the server lacks cannot
|
||||
// hold anything, so its contents are not looked up.
|
||||
func checkTargets(r *os.Root, dest string, p unzipPlan) (present, replaced map[string]bool, res Result) {
|
||||
present, replaced = map[string]bool{}, map[string]bool{}
|
||||
absent := func(name string) bool {
|
||||
parent := path.Dir(name)
|
||||
return parent != "." && !present[parent]
|
||||
}
|
||||
for _, d := range p.dirs {
|
||||
if absent(d) {
|
||||
continue
|
||||
}
|
||||
at := path.Join(dest, d)
|
||||
info, err := r.Lstat(at)
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
case err != nil:
|
||||
return nil, nil, failure(err, at)
|
||||
case info.Mode()&fs.ModeSymlink != 0:
|
||||
return nil, nil, typeConflict(at, "is a link on the server, and the archive has a folder there")
|
||||
case info.IsDir():
|
||||
present[d] = true
|
||||
default:
|
||||
return nil, nil, typeConflict(at, "is a file on the server, and the archive has a folder there")
|
||||
}
|
||||
}
|
||||
for _, zf := range p.files {
|
||||
if absent(zf.name) {
|
||||
continue
|
||||
}
|
||||
at := path.Join(dest, zf.name)
|
||||
info, err := r.Lstat(at)
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
case err != nil:
|
||||
return nil, nil, failure(err, at)
|
||||
case info.Mode().IsRegular():
|
||||
replaced[zf.name] = true
|
||||
case info.IsDir():
|
||||
return nil, nil, typeConflict(at, "is a folder on the server, and the archive has a file there")
|
||||
default:
|
||||
return nil, nil, typeConflict(at, "is a link or special file on the server, and the archive has a file there")
|
||||
}
|
||||
}
|
||||
return present, replaced, Result{}
|
||||
}
|
||||
|
||||
func typeConflict(at, why string) Result {
|
||||
return Result{Code: CodeTypeConflict, Entry: at, Error: fmt.Sprintf("%s %s; nothing was changed", at, why)}
|
||||
}
|
||||
|
||||
// extractAll writes every folder and file of p under staged, handed to the game
|
||||
// uid, and syncs each file.
|
||||
func extractAll(r *os.Root, staged string, p unzipPlan, progress func(done, total int64)) Result {
|
||||
for _, d := range p.dirs {
|
||||
at := path.Join(staged, d)
|
||||
if err := r.Mkdir(at, 0o755); err != nil {
|
||||
return unzipWriteFailure(err, d)
|
||||
}
|
||||
_ = ownWritten(r, at)
|
||||
}
|
||||
var done int64
|
||||
count := func(n int) {
|
||||
done += int64(n)
|
||||
if progress != nil {
|
||||
progress(done, p.bytes)
|
||||
}
|
||||
}
|
||||
for _, zf := range p.files {
|
||||
if res := extractOne(r, path.Join(staged, zf.name), zf, count); res.Code != "" {
|
||||
return res
|
||||
}
|
||||
}
|
||||
for _, d := range p.dirs {
|
||||
syncDir(r, path.Join(staged, d))
|
||||
}
|
||||
syncDir(r, staged)
|
||||
return Result{}
|
||||
}
|
||||
|
||||
func extractOne(r *os.Root, at string, zf zipFile, count func(int)) Result {
|
||||
invalid := func(err error) Result {
|
||||
return Result{Code: CodeArchiveInvalid, Entry: zf.name, Error: fmt.Sprintf(
|
||||
"%s in the archive is damaged: %v; nothing was changed", zf.name, err)}
|
||||
}
|
||||
src, err := zf.f.Open()
|
||||
if err != nil {
|
||||
return invalid(err)
|
||||
}
|
||||
defer src.Close()
|
||||
w, err := r.OpenFile(at, os.O_WRONLY|os.O_CREATE|os.O_EXCL, zf.mode)
|
||||
if err != nil {
|
||||
return unzipWriteFailure(err, zf.name)
|
||||
}
|
||||
if err := w.Chmod(zf.mode); err != nil {
|
||||
w.Close()
|
||||
return unzipWriteFailure(err, zf.name)
|
||||
}
|
||||
if _, err := io.Copy(countingWriter{w, count}, archiveReader{src}); err != nil {
|
||||
w.Close()
|
||||
var ae *archiveError
|
||||
if errors.As(err, &ae) {
|
||||
return invalid(ae.err)
|
||||
}
|
||||
return unzipWriteFailure(err, zf.name)
|
||||
}
|
||||
if err := syncWritten(w); err != nil {
|
||||
w.Close()
|
||||
return unzipWriteFailure(err, zf.name)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return unzipWriteFailure(err, zf.name)
|
||||
}
|
||||
_ = ownWritten(r, at)
|
||||
return Result{}
|
||||
}
|
||||
|
||||
// placeAll renames the extracted tree into dest. A folder the server lacks moves
|
||||
// whole; one it has is descended into. A file it has is first moved aside into
|
||||
// old, so undoing the journal puts it back.
|
||||
func placeAll(r *os.Root, dest, staged, old string, p unzipPlan, present, replaced map[string]bool) Result {
|
||||
// Only the destination and the folders the server has are descended into;
|
||||
// everything else moves with the folder it is in, so its name is not kept.
|
||||
kids := map[string][]string{}
|
||||
add := func(name string) {
|
||||
if parent := path.Dir(name); parent == "." || present[parent] {
|
||||
kids[parent] = append(kids[parent], name)
|
||||
}
|
||||
}
|
||||
for _, d := range p.dirs {
|
||||
add(d)
|
||||
}
|
||||
for _, zf := range p.files {
|
||||
add(zf.name)
|
||||
}
|
||||
|
||||
type move struct{ from, to string }
|
||||
var journal []move
|
||||
mv := func(from, to string) error {
|
||||
if err := renameEntry(r, from, to); err != nil {
|
||||
return err
|
||||
}
|
||||
journal = append(journal, move{from, to})
|
||||
return nil
|
||||
}
|
||||
var place func(dir string) error
|
||||
place = func(dir string) error {
|
||||
for _, c := range kids[dir] {
|
||||
src, dst := path.Join(staged, c), path.Join(dest, c)
|
||||
switch {
|
||||
case p.isDir[c] && present[c]:
|
||||
if err := place(c); err != nil {
|
||||
return err
|
||||
}
|
||||
case replaced[c]:
|
||||
aside := path.Join(old, c)
|
||||
if err := r.MkdirAll(path.Dir(aside), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := mv(dst, aside); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := mv(src, dst); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
if err := mv(src, dst); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := place("."); err != nil {
|
||||
for i := len(journal) - 1; i >= 0; i-- {
|
||||
_ = r.Rename(journal[i].to, journal[i].from)
|
||||
}
|
||||
return unzipWriteFailure(err, dest)
|
||||
}
|
||||
syncDir(r, dest)
|
||||
for d := range present {
|
||||
syncDir(r, path.Join(dest, d))
|
||||
}
|
||||
return Result{}
|
||||
}
|
||||
|
||||
// renameEntry is the rename placeAll moves with. A var so a test can fail one
|
||||
// part-way through and watch the journal undo the rest.
|
||||
var renameEntry = func(r *os.Root, from, to string) error { return r.Rename(from, to) }
|
||||
|
||||
// sweepUnzipTemps removes working folders a killed unzip left in dir.
|
||||
func sweepUnzipTemps(r *os.Root, dir string) {
|
||||
d, err := r.Open(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
names, _ := d.Readdirnames(-1)
|
||||
d.Close()
|
||||
for _, n := range names {
|
||||
if isUnzipTemp(n) {
|
||||
_ = r.RemoveAll(path.Join(dir, n))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isUnzipTemp(name string) bool {
|
||||
suffix, ok := strings.CutPrefix(name, unzipTempPrefix)
|
||||
if !ok || len(suffix) != 12 {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(suffix)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// unzipWriteFailure is writeFailure worded for an unzip, which by then has
|
||||
// changed nothing whatever the step that failed.
|
||||
func unzipWriteFailure(err error, name string) Result {
|
||||
res := writeFailure(err, name)
|
||||
if res.Code == CodeNoSpace {
|
||||
res.Error = "the server's volume filled up while extracting; nothing was changed"
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// archiveError marks a failure reading an entry's bytes out of the archive, so
|
||||
// extractOne can tell a damaged archive from the volume failing underneath.
|
||||
type archiveError struct{ err error }
|
||||
|
||||
func (e *archiveError) Error() string { return "read archive: " + e.err.Error() }
|
||||
func (e *archiveError) Unwrap() error { return e.err }
|
||||
|
||||
type archiveReader struct{ r io.Reader }
|
||||
|
||||
func (a archiveReader) Read(p []byte) (int, error) {
|
||||
n, err := a.r.Read(p)
|
||||
if err != nil && err != io.EOF {
|
||||
err = &archiveError{err}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// countingWriter reports each write's length to add.
|
||||
type countingWriter struct {
|
||||
w io.Writer
|
||||
add func(int)
|
||||
}
|
||||
|
||||
func (c countingWriter) Write(p []byte) (int, error) {
|
||||
n, err := c.w.Write(p)
|
||||
c.add(n)
|
||||
return n, err
|
||||
}
|
||||
@@ -1,562 +0,0 @@
|
||||
package fileedit
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/text/encoding/simplifiedchinese"
|
||||
)
|
||||
|
||||
// zent is one entry of a test archive: a deflated file, or a folder when the name
|
||||
// ends in "/".
|
||||
type zent struct {
|
||||
name string
|
||||
body string
|
||||
mode fs.FileMode // set on the header when non-zero
|
||||
flat bool // written without the UTF-8 flag, as old Windows tools do
|
||||
// raw writes body stored as-is under a header declaring size and crc, so a
|
||||
// test can build an archive whose entries lie about themselves.
|
||||
raw bool
|
||||
size uint64
|
||||
crc uint32
|
||||
}
|
||||
|
||||
func file(name, body string) zent { return zent{name: name, body: body} }
|
||||
|
||||
// lie is an entry declaring size bytes while holding body.
|
||||
func lie(name, body string, size uint64) zent {
|
||||
return zent{name: name, body: body, raw: true, size: size, crc: crc32.ChecksumIEEE([]byte(body))}
|
||||
}
|
||||
|
||||
func writeZip(t *testing.T, at string, entries ...zent) {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
w := zip.NewWriter(&buf)
|
||||
for _, e := range entries {
|
||||
fh := &zip.FileHeader{Name: e.name, Method: zip.Deflate, NonUTF8: e.flat}
|
||||
if e.mode != 0 {
|
||||
fh.SetMode(e.mode)
|
||||
}
|
||||
var dst io.Writer
|
||||
var err error
|
||||
if e.raw {
|
||||
fh.Method = zip.Store
|
||||
fh.CRC32, fh.CompressedSize64, fh.UncompressedSize64 = e.crc, uint64(len(e.body)), e.size
|
||||
dst, err = w.CreateRaw(fh)
|
||||
} else {
|
||||
dst, err = w.CreateHeader(fh)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("zip entry %q: %v", e.name, err)
|
||||
}
|
||||
if _, err := io.WriteString(dst, e.body); err != nil {
|
||||
t.Fatalf("zip entry %q: %v", e.name, err)
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(at, buf.Bytes(), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// tree is everything under dir: each file's content, "<dir>" for a folder and
|
||||
// "-> target" for a link. Comparing two trees is how a test says "nothing
|
||||
// changed", working folders included.
|
||||
func tree(t *testing.T, dir string) map[string]string {
|
||||
t.Helper()
|
||||
out := map[string]string{}
|
||||
err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(dir, p)
|
||||
switch {
|
||||
case rel == ".":
|
||||
case d.Type()&fs.ModeSymlink != 0:
|
||||
target, err := os.Readlink(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out[rel] = "-> " + target
|
||||
case d.IsDir():
|
||||
out[rel] = "<dir>"
|
||||
default:
|
||||
b, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out[rel] = string(b)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func assertSameTree(t *testing.T, before, after map[string]string) {
|
||||
t.Helper()
|
||||
if !reflect.DeepEqual(before, after) {
|
||||
t.Fatalf("the tree changed:\nbefore %v\nafter %v", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func unzipAt(t *testing.T, root, name string, overwrite bool) Result {
|
||||
t.Helper()
|
||||
return exec(t, root, Request{Op: OpUnzip, Path: name, Overwrite: overwrite})
|
||||
}
|
||||
|
||||
func TestUnzip(t *testing.T) {
|
||||
t.Run("extracts files and folders into the folder holding the archive", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
if err := os.Mkdir(filepath.Join(root, "plugins"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeZip(t, filepath.Join(root, "plugins", "pack.zip"),
|
||||
zent{name: "Essentials/"},
|
||||
file("Essentials/config.yml", "locale: zh\n"),
|
||||
zent{name: "empty/"},
|
||||
file("./readme.txt", "hi"),
|
||||
file(`win\sub\a.txt`, "from windows"),
|
||||
)
|
||||
var owned []string
|
||||
prev := ownWritten
|
||||
ownWritten = func(_ *os.Root, name string) error { owned = append(owned, name); return nil }
|
||||
defer func() { ownWritten = prev }()
|
||||
|
||||
res := unzipAt(t, root, "plugins/pack.zip", false)
|
||||
if res.Code != "" || res.Files != 3 || res.Bytes != int64(len("locale: zh\n")+len("hi")+len("from windows")) {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
got := tree(t, filepath.Join(root, "plugins"))
|
||||
delete(got, "pack.zip")
|
||||
want := map[string]string{
|
||||
"Essentials": "<dir>", "Essentials/config.yml": "locale: zh\n", "empty": "<dir>",
|
||||
"readme.txt": "hi", "win": "<dir>", "win/sub": "<dir>", "win/sub/a.txt": "from windows",
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("plugins/ = %v\nwant %v", got, want)
|
||||
}
|
||||
if info, _ := os.Stat(filepath.Join(root, "plugins", "readme.txt")); info.Mode().Perm() != 0o644 {
|
||||
t.Fatalf("mode = %v, want 0644", info.Mode().Perm())
|
||||
}
|
||||
// Every folder and file is handed to the game uid, while still in the
|
||||
// working folder: 4 folders and 3 files.
|
||||
if len(owned) != 7 {
|
||||
t.Fatalf("handed to the game uid: %v, want 7", owned)
|
||||
}
|
||||
for _, o := range owned {
|
||||
if !strings.HasPrefix(o, "plugins/"+unzipTempPrefix) {
|
||||
t.Fatalf("%s was chowned outside the working folder", o)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an archive named in capitals extracts", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "PACK.ZIP"), file("a.txt", "a"))
|
||||
if res := unzipAt(t, root, "PACK.ZIP", false); res.Code != "" || res.Files != 1 {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
if got := mustRead(t, filepath.Join(root, "a.txt")); got != "a" {
|
||||
t.Fatalf("a.txt = %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("__MACOSX is left out", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "mac.zip"), file("a.txt", "a"), zent{name: "__MACOSX/"}, file("__MACOSX/._a.txt", "fork"))
|
||||
if res := unzipAt(t, root, "mac.zip", false); res.Code != "" || res.Files != 1 {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
assertAbsent(t, filepath.Join(root, "__MACOSX"))
|
||||
})
|
||||
|
||||
t.Run("a name made on Chinese Windows is decoded from GBK; UTF-8 without the flag is kept", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
gbk, err := simplifiedchinese.GBK.NewEncoder().String("存档/说明.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeZip(t, filepath.Join(root, "cn.zip"),
|
||||
zent{name: gbk, body: "中文", flat: true},
|
||||
zent{name: "macOS名字.txt", body: "utf8", flat: true},
|
||||
)
|
||||
if res := unzipAt(t, root, "cn.zip", false); res.Code != "" || res.Files != 2 {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
if got := mustRead(t, filepath.Join(root, "存档", "说明.txt")); got != "中文" {
|
||||
t.Fatalf("存档/说明.txt = %q", got)
|
||||
}
|
||||
if got := mustRead(t, filepath.Join(root, "macOS名字.txt")); got != "utf8" {
|
||||
t.Fatalf("macOS名字.txt = %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an executable entry stays executable; every other file is 0644", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "sh.zip"),
|
||||
zent{name: "start.sh", body: "#!/bin/sh\n", mode: 0o755},
|
||||
zent{name: "secret.txt", body: "x", mode: 0o600},
|
||||
)
|
||||
if res := unzipAt(t, root, "sh.zip", false); res.Code != "" {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
for name, want := range map[string]fs.FileMode{"start.sh": 0o755, "secret.txt": 0o644} {
|
||||
if info, _ := os.Stat(filepath.Join(root, name)); info.Mode().Perm() != want {
|
||||
t.Errorf("%s mode = %v, want %v", name, info.Mode().Perm(), want)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a name leading outside is archive_unsafe and nothing is written", func(t *testing.T) {
|
||||
for raw, entry := range map[string]string{
|
||||
"../evil.txt": "../evil.txt",
|
||||
"a/../../evil.txt": "a/../../evil.txt",
|
||||
`..\evil.txt`: "../evil.txt",
|
||||
"/evil.txt": "/evil.txt",
|
||||
"C:/evil.txt": "C:/evil.txt",
|
||||
`c:\evil.txt`: "c:/evil.txt",
|
||||
"nul\x00.txt": "nul\x00.txt",
|
||||
"a/..": "a/..",
|
||||
} {
|
||||
t.Run(entry, func(t *testing.T) {
|
||||
root, outside := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "bad.zip"), file("ok.txt", "ok"), file(raw, "evil"))
|
||||
before := tree(t, root)
|
||||
res := unzipAt(t, root, "bad.zip", true)
|
||||
if res.Code != CodeArchiveUnsafe || res.Entry != entry {
|
||||
t.Fatalf("result = %+v; want archive_unsafe naming %q", res, entry)
|
||||
}
|
||||
assertSameTree(t, before, tree(t, root))
|
||||
assertAbsent(t, filepath.Join(outside, "evil.txt"))
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a link entry is archive_symlink and a pipe is archive_unsafe", func(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
mode fs.FileMode
|
||||
code string
|
||||
}{
|
||||
{fs.ModeSymlink | 0o777, CodeArchiveSymlink},
|
||||
{fs.ModeNamedPipe | 0o644, CodeArchiveUnsafe},
|
||||
} {
|
||||
root, _ := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "odd.zip"),
|
||||
file("ok.txt", "ok"), zent{name: "odd", body: "../../outside/secret.txt", mode: tc.mode})
|
||||
before := tree(t, root)
|
||||
res := unzipAt(t, root, "odd.zip", true)
|
||||
if res.Code != tc.code || res.Entry != "odd" {
|
||||
t.Fatalf("%v: result = %+v; want %s naming odd", tc.mode, res, tc.code)
|
||||
}
|
||||
assertSameTree(t, before, tree(t, root))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a damaged or senseless archive is archive_invalid and nothing is written", func(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
entries []zent
|
||||
entry string
|
||||
}{
|
||||
"bytes past the declared size": {[]zent{file("ok.txt", "ok"), lie("lie.txt", "0123456789", 3)}, "lie.txt"},
|
||||
"bytes short of the declared size": {[]zent{file("ok.txt", "ok"), lie("lie.txt", "0123456789", 20)}, "lie.txt"},
|
||||
"a wrong checksum": {[]zent{file("ok.txt", "ok"),
|
||||
{name: "crc.txt", body: "0123456789", raw: true, size: 10, crc: crc32.ChecksumIEEE([]byte("0123456789")) + 1}}, "crc.txt"},
|
||||
"the same file twice": {[]zent{file("a.txt", "1"), file("a.txt", "2")}, "a.txt"},
|
||||
"a file and a folder at once": {[]zent{file("a", "1"), file("a/b.txt", "2")}, "a"},
|
||||
"nothing in it": {nil, ""},
|
||||
"nothing but __MACOSX": {[]zent{file("__MACOSX/._a", "fork")}, ""},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "bad.zip"), tc.entries...)
|
||||
before := tree(t, root)
|
||||
res := unzipAt(t, root, "bad.zip", true)
|
||||
if res.Code != CodeArchiveInvalid || res.Entry != tc.entry {
|
||||
t.Fatalf("result = %+v; want archive_invalid naming %q", res, tc.entry)
|
||||
}
|
||||
assertSameTree(t, before, tree(t, root))
|
||||
})
|
||||
}
|
||||
t.Run("not a zip at all", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
if err := os.WriteFile(filepath.Join(root, "fake.zip"), []byte("hello"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res := unzipAt(t, root, "fake.zip", false); res.Code != CodeArchiveInvalid {
|
||||
t.Fatalf("result = %+v; want archive_invalid", res)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("the archive's own path is checked", func(t *testing.T) {
|
||||
root, outside := worldRoot(t)
|
||||
if err := os.Mkdir(filepath.Join(root, "dir.zip"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeZip(t, filepath.Join(outside, "x.zip"), file("evil.txt", "evil"))
|
||||
symlink(t, outside, filepath.Join(root, "escape-link"))
|
||||
for name, code := range map[string]string{
|
||||
"server.properties": CodeBadPath,
|
||||
"missing.zip": CodeNotFound,
|
||||
"dir.zip": CodeBadPath,
|
||||
"../outside/x.zip": CodeBadPath,
|
||||
"escape-link/x.zip": CodeBadPath,
|
||||
"config/../../x.zip": CodeBadPath,
|
||||
} {
|
||||
if res := unzipAt(t, root, name, true); res.Code != code {
|
||||
t.Errorf("%s: result = %+v; want %s", name, res, code)
|
||||
}
|
||||
}
|
||||
assertAbsent(t, filepath.Join(root, "evil.txt"))
|
||||
assertAbsent(t, filepath.Join(outside, "evil.txt"))
|
||||
})
|
||||
|
||||
t.Run("files already there are listed and nothing changes without overwrite", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "pack.zip"),
|
||||
file("server.properties", "motd=new\n"), file("config/paper.yml", "verbose: true\n"),
|
||||
file("config/new.yml", "new"), file("fresh/x.txt", "x"))
|
||||
before := tree(t, root)
|
||||
res := unzipAt(t, root, "pack.zip", false)
|
||||
if res.Code != CodeExists || res.ConflictCount != 2 ||
|
||||
!reflect.DeepEqual(res.Conflicts, []string{"config/paper.yml", "server.properties"}) {
|
||||
t.Fatalf("result = %+v; want exists listing config/paper.yml and server.properties", res)
|
||||
}
|
||||
assertSameTree(t, before, tree(t, root))
|
||||
})
|
||||
|
||||
t.Run("conflicts are named from the server's root when the archive sits in a folder", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "config", "pack.zip"), file("paper.yml", "verbose: true\n"))
|
||||
res := unzipAt(t, root, "config/pack.zip", false)
|
||||
if res.Code != CodeExists || !reflect.DeepEqual(res.Conflicts, []string{"config/paper.yml"}) {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the list stops at MaxConflicts and the count does not", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
var entries []zent
|
||||
for i := range MaxConflicts + 1 {
|
||||
name := fmt.Sprintf("f%03d.txt", i)
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte("old"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries = append(entries, file(name, "new"))
|
||||
}
|
||||
writeZip(t, filepath.Join(root, "many.zip"), entries...)
|
||||
res := unzipAt(t, root, "many.zip", false)
|
||||
if res.Code != CodeExists || res.ConflictCount != MaxConflicts+1 || len(res.Conflicts) != MaxConflicts ||
|
||||
res.Conflicts[0] != "f000.txt" || res.Conflicts[MaxConflicts-1] != "f199.txt" {
|
||||
t.Fatalf("code %q, count %d, %d listed (%v … %v)", res.Code, res.ConflictCount, len(res.Conflicts),
|
||||
res.Conflicts[:1], res.Conflicts[len(res.Conflicts)-1:])
|
||||
}
|
||||
})
|
||||
|
||||
// The result comes back through the tail of the Pod's log, where a line past
|
||||
// 16 KiB is split and its head can fall out of the tail.
|
||||
t.Run("the list stops at 8 KiB of names too, keeping the result line whole", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
var entries []zent
|
||||
for i := range 40 {
|
||||
name := fmt.Sprintf("%02d", i) + strings.Repeat("n", 248) // 250 bytes
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte("old"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries = append(entries, file(name, "new"))
|
||||
}
|
||||
writeZip(t, filepath.Join(root, "long.zip"), entries...)
|
||||
res := unzipAt(t, root, "long.zip", false)
|
||||
line, _ := json.Marshal(res)
|
||||
// 32 names are 8000 bytes; a 33rd would pass 8192.
|
||||
if res.Code != CodeExists || res.ConflictCount != 40 || len(res.Conflicts) != 32 ||
|
||||
!strings.HasPrefix(res.Conflicts[31], "31n") || len(line) >= 16<<10 {
|
||||
t.Fatalf("code %q, count %d, %d listed, result line %d bytes", res.Code, res.ConflictCount, len(res.Conflicts), len(line))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("overwrite replaces files, merges folders and keeps everything else", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
if err := os.WriteFile(filepath.Join(root, "config", "keep.yml"), []byte("keep"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writeZip(t, filepath.Join(root, "pack.zip"),
|
||||
file("server.properties", "motd=new\n"), file("config/paper.yml", "verbose: true\n"),
|
||||
file("config/new.yml", "new"), file("fresh/x.txt", "x"))
|
||||
res := unzipAt(t, root, "pack.zip", true)
|
||||
if res.Code != "" || res.Files != 4 {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
got := tree(t, root)
|
||||
delete(got, "pack.zip")
|
||||
want := map[string]string{
|
||||
"server.properties": "motd=new\n", "config": "<dir>", "config/paper.yml": "verbose: true\n",
|
||||
"config/keep.yml": "keep", "config/new.yml": "new", "fresh": "<dir>", "fresh/x.txt": "x",
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("world = %v\nwant %v", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a file where the server has a folder, the reverse, or a link is type_conflict even with overwrite", func(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
entry string
|
||||
want string
|
||||
}{
|
||||
"folder over a file": {"server.properties/x.txt", "server.properties"},
|
||||
"file over a folder": {"config", "config"},
|
||||
"folder over a link": {"escape-link/evil.txt", "escape-link"},
|
||||
"file over a link": {"planted.txt", "planted.txt"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
root, outside := worldRoot(t)
|
||||
symlink(t, outside, filepath.Join(root, "escape-link"))
|
||||
symlink(t, "server.properties", filepath.Join(root, "planted.txt"))
|
||||
writeZip(t, filepath.Join(root, "pack.zip"), file("ok.txt", "ok"), file(tc.entry, "evil"))
|
||||
before := tree(t, root)
|
||||
res := unzipAt(t, root, "pack.zip", true)
|
||||
if res.Code != CodeTypeConflict || res.Entry != tc.want {
|
||||
t.Fatalf("result = %+v; want type_conflict naming %s", res, tc.want)
|
||||
}
|
||||
assertSameTree(t, before, tree(t, root))
|
||||
assertAbsent(t, filepath.Join(outside, "evil.txt"))
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("more than the volume has free is no_space before anything is written", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "pack.zip"), file("d/a.txt", "0123456789"), file("d/b.txt", "0123456789"))
|
||||
need := int64(20 + 3*unzipEntryOverhead) // two files and the folder d
|
||||
stubStatfs(t, uint64(need-1), 1<<30)
|
||||
before := tree(t, root)
|
||||
res := unzipAt(t, root, "pack.zip", false)
|
||||
if res.Code != CodeNoSpace || res.Need != need || res.Avail != need-1 {
|
||||
t.Fatalf("result = %+v; want no_space with need %d, avail %d", res, need, need-1)
|
||||
}
|
||||
assertSameTree(t, before, tree(t, root))
|
||||
stubStatfs(t, uint64(need), 1<<30)
|
||||
if res := unzipAt(t, root, "pack.zip", false); res.Code != "" {
|
||||
t.Fatalf("with exactly enough room: %+v", res)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("the volume filling up mid-way is no_space and changes nothing", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "pack.zip"), file("a.txt", "a"), file("server.properties", "motd=new\n"))
|
||||
calls := 0
|
||||
prev := syncWritten
|
||||
syncWritten = func(f *os.File) error {
|
||||
if calls++; calls == 2 {
|
||||
return syscall.ENOSPC
|
||||
}
|
||||
return f.Sync()
|
||||
}
|
||||
defer func() { syncWritten = prev }()
|
||||
before := tree(t, root)
|
||||
if res := unzipAt(t, root, "pack.zip", true); res.Code != CodeNoSpace {
|
||||
t.Fatalf("result = %+v; want no_space", res)
|
||||
}
|
||||
assertSameTree(t, before, tree(t, root))
|
||||
})
|
||||
|
||||
// The renames are the one step that touches the server's files; one failing
|
||||
// part-way must put back every file already moved, replaced ones included.
|
||||
t.Run("a rename failing part-way is undone, whichever it is", func(t *testing.T) {
|
||||
// config present: paper.yml aside + in, z.yml in; zeta in; server.properties aside + in.
|
||||
const moves = 6
|
||||
for failAt := 1; failAt <= moves; failAt++ {
|
||||
root, _ := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "pack.zip"),
|
||||
file("config/paper.yml", "verbose: true\n"), file("config/z.yml", "z"),
|
||||
file("server.properties", "motd=new\n"), file("zeta/x.txt", "x"))
|
||||
calls := 0
|
||||
prev := renameEntry
|
||||
renameEntry = func(r *os.Root, from, to string) error {
|
||||
if calls++; calls == failAt {
|
||||
return errors.New("injected rename failure")
|
||||
}
|
||||
return r.Rename(from, to)
|
||||
}
|
||||
before := tree(t, root)
|
||||
res := unzipAt(t, root, "pack.zip", true)
|
||||
renameEntry = prev
|
||||
if res.Code == "" {
|
||||
t.Fatalf("rename %d failing: result = %+v; want a failure", failAt, res)
|
||||
}
|
||||
assertSameTree(t, before, tree(t, root))
|
||||
}
|
||||
// And with none failing, the count above is the real number of moves.
|
||||
root, _ := worldRoot(t)
|
||||
writeZip(t, filepath.Join(root, "pack.zip"),
|
||||
file("config/paper.yml", "verbose: true\n"), file("config/z.yml", "z"),
|
||||
file("server.properties", "motd=new\n"), file("zeta/x.txt", "x"))
|
||||
calls := 0
|
||||
prev := renameEntry
|
||||
renameEntry = func(r *os.Root, from, to string) error { calls++; return r.Rename(from, to) }
|
||||
defer func() { renameEntry = prev }()
|
||||
if res := unzipAt(t, root, "pack.zip", true); res.Code != "" || calls != moves {
|
||||
t.Fatalf("result = %+v, %d moves; want success in %d", res, calls, moves)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a working folder a killed unzip left is cleared; a lookalike is kept", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
stale := filepath.Join(root, unzipTempPrefix+"0123456789ab")
|
||||
if err := os.MkdirAll(filepath.Join(stale, "new"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Twelve characters that are not hex, and hex a byte short or long.
|
||||
lookalikes := []string{"notahexname!", "0123456789", "0123456789abcd"}
|
||||
for _, l := range lookalikes {
|
||||
if err := os.Mkdir(filepath.Join(root, unzipTempPrefix+l), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
writeZip(t, filepath.Join(root, "pack.zip"), file("a.txt", "a"))
|
||||
if res := unzipAt(t, root, "pack.zip", false); res.Code != "" {
|
||||
t.Fatalf("result = %+v", res)
|
||||
}
|
||||
assertAbsent(t, stale)
|
||||
for _, l := range lookalikes {
|
||||
if _, err := os.Stat(filepath.Join(root, unzipTempPrefix+l)); err != nil {
|
||||
t.Fatalf("lookalike %q removed: %v", l, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("progress climbs to the total", func(t *testing.T) {
|
||||
root, _ := worldRoot(t)
|
||||
big := strings.Repeat("x", 100<<10)
|
||||
writeZip(t, filepath.Join(root, "pack.zip"), file("a.bin", big), file("b.bin", big))
|
||||
var seen []int64
|
||||
var total int64
|
||||
res := exec(t, root, Request{Op: OpUnzip, Path: "pack.zip",
|
||||
Progress: func(done, all int64) { seen = append(seen, done); total = all }})
|
||||
if res.Code != "" || total != int64(2*len(big)) || len(seen) < 2 || seen[len(seen)-1] != total {
|
||||
t.Fatalf("result = %+v; progress %d calls ending at %v of %d", res, len(seen), seen[len(seen)-1:], total)
|
||||
}
|
||||
for i := 1; i < len(seen); i++ {
|
||||
if seen[i] < seen[i-1] {
|
||||
t.Fatalf("progress went backwards: %v", seen)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -62,9 +62,8 @@ const (
|
||||
// rename, upload) a files Job performs. Every one but list and read holds the
|
||||
// volume.
|
||||
LabelFilesMode = "felis.lolicon.best/files-mode"
|
||||
// LabelExportMode is what an export Job sends: ExportModeWorld (the live
|
||||
// world) or ExportModeFiles (one file or folder of it), which hold the
|
||||
// volume, or ExportModeBackup (a stored archive), which does not.
|
||||
// LabelExportMode is what an export Job archives: ExportModeWorld (the live
|
||||
// world, which holds the volume) or ExportModeBackup (a stored archive).
|
||||
LabelExportMode = "felis.lolicon.best/export-mode"
|
||||
|
||||
// LabelThenRestore marks a backup Job that is the safety snapshot in front of
|
||||
@@ -111,7 +110,6 @@ const (
|
||||
const (
|
||||
ExportModeWorld = "world"
|
||||
ExportModeBackup = "backup"
|
||||
ExportModeFiles = "files"
|
||||
)
|
||||
|
||||
// JobKind names the holder a Job represents, or reports false for a Job that
|
||||
|
||||
@@ -73,7 +73,7 @@ func exportJob(t *testing.T, server, mode string) batchv1.Job {
|
||||
Server: server, ID: "0011223344556677", Mode: mode, WorldPVC: "world-" + server + "-0",
|
||||
BackupPVC: "felis-backups", BackupRef: "/backups/a.tar.gz", TargetURL: "http://api/x", Token: "t",
|
||||
Namespace: "minecraft", ServiceAccount: "felis-restore", Image: "felis:1",
|
||||
BackupRoot: "/backups", WorldsRoot: "/world", Path: "plugins",
|
||||
BackupRoot: "/backups", WorldsRoot: "/world",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ExportJob: %v", err)
|
||||
@@ -106,7 +106,6 @@ func TestJobKindMatchesTheExecutors(t *testing.T) {
|
||||
{"file list", filesJob(t, "survival", fileedit.OpList), "", false},
|
||||
{"world export", exportJob(t, "survival", worldexport.ModeWorld), KindExport, true},
|
||||
{"backup export", exportJob(t, "survival", worldexport.ModeBackup), "", false},
|
||||
{"files export", exportJob(t, "survival", worldexport.ModeFiles), KindExport, true},
|
||||
} {
|
||||
kind, ok := JobKind(&tc.job)
|
||||
if kind != tc.kind || ok != tc.ok {
|
||||
|
||||
@@ -93,7 +93,7 @@ func TestSpecChangeReplacesAPodThatGaveUp(t *testing.T) {
|
||||
// The admin corrects the image; the StatefulSet takes the new template but,
|
||||
// the pod never having been ready, leaves the pod as it is.
|
||||
editImage(t, c, "registry.internal/felis/paper:fixed")
|
||||
at(time.Hour + time.Second)
|
||||
s = at(time.Hour + time.Second)
|
||||
if !podPresent(t, c) {
|
||||
t.Fatal("the pod was deleted before the StatefulSet had observed the new template")
|
||||
}
|
||||
|
||||
@@ -75,7 +75,6 @@ func guardInAppWebView(w http.ResponseWriter, r *http.Request) bool {
|
||||
Value: "1",
|
||||
Path: "/",
|
||||
HttpOnly: true,
|
||||
Secure: true,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
MaxAge: 3600,
|
||||
})
|
||||
|
||||
@@ -138,12 +138,12 @@ func TestGuardHonorsAcknowledgement(t *testing.T) {
|
||||
}
|
||||
var acked bool
|
||||
for _, c := range w.Result().Cookies() {
|
||||
if c.Name == webViewAckCookie && c.Value == "1" && c.Secure {
|
||||
if c.Name == webViewAckCookie && c.Value == "1" {
|
||||
acked = true
|
||||
}
|
||||
}
|
||||
if !acked {
|
||||
t.Fatalf("ua_ack did not set a Secure ack cookie")
|
||||
t.Fatalf("ua_ack did not set the ack cookie")
|
||||
}
|
||||
|
||||
// A subsequent navigation carrying the ack cookie is not interrupted.
|
||||
|
||||
@@ -31,7 +31,7 @@ func TestUserDetailLinkReadFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
mustExec(t, "CREATE ROLE "+role+" LOGIN PASSWORD 'pgint'")
|
||||
mustExec(t, "CREATE ROLE "+role+" LOGIN")
|
||||
t.Cleanup(drop)
|
||||
mustExec(t, "GRANT USAGE ON SCHEMA public TO "+role)
|
||||
mustExec(t, "GRANT SELECT ON users, servers TO "+role)
|
||||
@@ -39,7 +39,7 @@ func TestUserDetailLinkReadFailure(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dsn.User = url.UserPassword(role, "pgint")
|
||||
dsn.User = url.User(role)
|
||||
drv, err := store.Open(ctx, dsn.String())
|
||||
if err != nil {
|
||||
t.Fatalf("open as %s: %v", role, err)
|
||||
|
||||
@@ -211,13 +211,11 @@ func (c *Conn) nextID() int32 {
|
||||
// writePacket encodes one RCON packet: little-endian length, id, type, the
|
||||
// null-terminated body, and a trailing null byte.
|
||||
func writePacket(w io.Writer, id, typ int32, body string) error {
|
||||
// Bounded on the body, before the int32 conversion: one past 2 GiB would
|
||||
// wrap the length negative and slip under a check made after it.
|
||||
if len(body) > maxPacketLen-minPacketLen {
|
||||
return fmt.Errorf("rcon: outgoing packet too large: %d bytes", len(body)+minPacketLen)
|
||||
}
|
||||
bodyBytes := []byte(body)
|
||||
length := int32(4 + 4 + len(bodyBytes) + 2)
|
||||
if length > maxPacketLen {
|
||||
return fmt.Errorf("rcon: outgoing packet too large: %d bytes", length)
|
||||
}
|
||||
buf := make([]byte, 0, 4+length)
|
||||
buf = appendInt32(buf, length)
|
||||
buf = appendInt32(buf, id)
|
||||
|
||||
@@ -169,27 +169,6 @@ func TestDialAndExecute(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A command goes out in one packet of at most 4096 bytes; a longer one is
|
||||
// refused before anything is sent.
|
||||
func TestExecuteRefusesAnOversizeCommand(t *testing.T) {
|
||||
longest := strings.Repeat("a", 4096-10) // 10: id, type, two terminators
|
||||
f := startFakeRCON(t, "s3cret", map[string]string{longest: "ok"})
|
||||
defer f.stop()
|
||||
|
||||
c, err := rcon.Dial(f.addr(), "s3cret", 2*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("Dial: %v", err)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
if got, err := c.Execute(longest); err != nil || got != "ok" {
|
||||
t.Fatalf("Execute(%d bytes) = %q, %v; want ok", len(longest), got, err)
|
||||
}
|
||||
if _, err := c.Execute(longest + "a"); err == nil || !strings.Contains(err.Error(), "too large") {
|
||||
t.Fatalf("Execute(%d bytes) err = %v, want the too-large refusal", len(longest)+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialAuthFailure(t *testing.T) {
|
||||
f := startFakeRCON(t, "correct-horse", nil)
|
||||
defer f.stop()
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
package submit
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
)
|
||||
|
||||
// ErrDigestMismatch reports bytes that do not hash to the SHA-256 their sender
|
||||
// computed over them (the request's Content-Digest): they changed on the way.
|
||||
// The API answers 400 digest_mismatch, and the client sends them again.
|
||||
var ErrDigestMismatch = errors.New("submit: the bytes that arrived do not match the digest they were sent with")
|
||||
|
||||
// VerifyDigest passes r through and, where it ends, answers ErrDigestMismatch in
|
||||
// place of io.EOF unless what went by hashes to want, a SHA-256. The stores
|
||||
// treat that like any read that breaks off: UploadPart cuts the part back off,
|
||||
// and UploadContext never lets the blob replace the one before it. So bytes
|
||||
// changed on the way are never kept.
|
||||
func VerifyDigest(r io.Reader, want []byte) io.Reader {
|
||||
return &digestReader{r: r, h: sha256.New(), want: want}
|
||||
}
|
||||
|
||||
type digestReader struct {
|
||||
r io.Reader
|
||||
h hash.Hash
|
||||
want []byte
|
||||
}
|
||||
|
||||
func (d *digestReader) Read(p []byte) (int, error) {
|
||||
n, err := d.r.Read(p)
|
||||
d.h.Write(p[:n])
|
||||
if err == io.EOF {
|
||||
if got := d.h.Sum(nil); !bytes.Equal(got, d.want) {
|
||||
return n, fmt.Errorf("%w: they hash to %x, sent as %x", ErrDigestMismatch, got, d.want)
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
package submit
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// sumOf is the SHA-256 a client sends with body.
|
||||
func sumOf(body string) []byte {
|
||||
s := sha256.Sum256([]byte(body))
|
||||
return s[:]
|
||||
}
|
||||
|
||||
func TestVerifyDigest(t *testing.T) {
|
||||
t.Run("bytes that hash to the digest end cleanly", func(t *testing.T) {
|
||||
b, err := io.ReadAll(VerifyDigest(strings.NewReader("modpack"), sumOf("modpack")))
|
||||
if err != nil || string(b) != "modpack" {
|
||||
t.Fatalf("ReadAll = (%q, %v), want (modpack, nil)", b, err)
|
||||
}
|
||||
})
|
||||
t.Run("bytes that do not end in ErrDigestMismatch", func(t *testing.T) {
|
||||
b, err := io.ReadAll(VerifyDigest(strings.NewReader("modpacX"), sumOf("modpack")))
|
||||
if !errors.Is(err, ErrDigestMismatch) {
|
||||
t.Fatalf("err = %v, want ErrDigestMismatch", err)
|
||||
}
|
||||
if string(b) != "modpacX" {
|
||||
t.Fatalf("passed on %q, want every byte (the store decides what to keep)", b)
|
||||
}
|
||||
})
|
||||
t.Run("a read that breaks off is passed on as it is", func(t *testing.T) {
|
||||
_, err := io.ReadAll(VerifyDigest(&failingReader{data: "mod"}, sumOf("mod")))
|
||||
if err == nil || errors.Is(err, ErrDigestMismatch) || err.Error() != "connection reset" {
|
||||
t.Fatalf("err = %v, want the connection reset itself", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A part that changed on the way is cut back off like one that broke off, below
|
||||
// the part cap and exactly at it (where the cap reads one byte past the part).
|
||||
func TestChunkedUploadCutsBackAPartChangedOnTheWay(t *testing.T) {
|
||||
for _, tc := range []struct{ name, sent, meant string }{
|
||||
{"under the part cap", "abX", "abc"},
|
||||
{"at the part cap", "abcX", "abcd"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
m, _, _, id := newChunkedManager(t)
|
||||
sendPart(t, m, id, 0, "\x1f\x8b\x08\x00")
|
||||
|
||||
_, err := m.UploadPart(context.Background(), id, "user-1", 4, VerifyDigest(strings.NewReader(tc.sent), sumOf(tc.meant)))
|
||||
if !errors.Is(err, ErrDigestMismatch) {
|
||||
t.Fatalf("err = %v, want ErrDigestMismatch", err)
|
||||
}
|
||||
if got := staged(t, m, id); got != 4 {
|
||||
t.Fatalf("staged after a changed part = %d, want 4", got)
|
||||
}
|
||||
p, err := m.UploadPart(context.Background(), id, "user-1", 4, VerifyDigest(strings.NewReader(tc.meant), sumOf(tc.meant)))
|
||||
if err != nil || p.Received != int64(4+len(tc.meant)) {
|
||||
t.Fatalf("the part sent again = (%d, %v), want (%d, nil)", p.Received, err, 4+len(tc.meant))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A context that changed on the way leaves the one before it stored, and its
|
||||
// digest recorded.
|
||||
func TestUploadContextChangedOnTheWayKeepsThePreviousContext(t *testing.T) {
|
||||
m, st, _ := newManager()
|
||||
base := t.TempDir()
|
||||
m.Blobs = &LocalContextStore{Base: base, MinFree: 1e-9}
|
||||
ctx := context.Background()
|
||||
seed, _ := m.Create(ctx, CreateRequest{DisplayName: "Pack", SubmittedBy: "user-1"})
|
||||
first, second := gzBody("first"), gzBody("second")
|
||||
if _, err := m.UploadContext(ctx, seed.ID, "user-1", VerifyDigest(strings.NewReader(first), sumOf(first))); err != nil {
|
||||
t.Fatalf("first upload: %v", err)
|
||||
}
|
||||
|
||||
_, err := m.UploadContext(ctx, seed.ID, "user-1", VerifyDigest(strings.NewReader(second+"X"), sumOf(second)))
|
||||
|
||||
if !errors.Is(err, ErrDigestMismatch) {
|
||||
t.Fatalf("err = %v, want ErrDigestMismatch", err)
|
||||
}
|
||||
if got, _ := os.ReadFile(filepath.Join(base, seed.ID, contextBlobName)); string(got) != first {
|
||||
t.Fatalf("stored %q, want the first upload %q", got, first)
|
||||
}
|
||||
if got := st.subs[seed.ID].ContextSHA256; got != hex.EncodeToString(sumOf(first)) {
|
||||
t.Fatalf("recorded digest %s, want the first upload's", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Through minio-go, a mismatch found at the end of the stream aborts the
|
||||
// multipart upload, whether the last part is short or the stream ends right at a
|
||||
// part boundary.
|
||||
func TestS3ContextStorePutKeepsNothingChangedOnTheWay(t *testing.T) {
|
||||
for _, size := range []int{2*s3PartSize + 5, 2 * s3PartSize} {
|
||||
fake := &multipartS3{objects: map[string]string{}}
|
||||
ts := httptest.NewServer(fake)
|
||||
s, err := NewS3ContextStore(S3StoreConfig{Base: "s3://felis-uploads/builds", Endpoint: ts.URL, Region: "us-east-1", AccessKey: "a", SecretKey: "b"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload := make([]byte, size)
|
||||
rand.New(rand.NewSource(2)).Read(payload)
|
||||
meant := sumOf(string(payload))
|
||||
payload[size-1] ^= 1
|
||||
|
||||
_, err = s.Put(context.Background(), "sub-big", VerifyDigest(bytes.NewReader(payload), meant))
|
||||
ts.Close()
|
||||
|
||||
if !errors.Is(err, ErrDigestMismatch) {
|
||||
t.Errorf("%d bytes: err = %v, want ErrDigestMismatch", size, err)
|
||||
}
|
||||
if len(fake.objects) != 0 {
|
||||
t.Errorf("%d bytes: an object was completed: %v", size, fake.objects)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -262,13 +262,6 @@ func TestSaveStateOr(t *testing.T) {
|
||||
if got := NewestState(path, fallback); got != path {
|
||||
t.Fatalf("no fallback yet: NewestState = %q, want the file", got)
|
||||
}
|
||||
// An earlier run wrote the file, minutes before this one. Linux stamps files
|
||||
// from a coarse clock, so two writes a test makes back to back can share an
|
||||
// mtime, and a tie reads the file.
|
||||
earlier := time.Now().Add(-time.Hour)
|
||||
if err := os.Chtimes(path, earlier, earlier); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
mailed := &State{Recipients: []string{"[email protected]"}}
|
||||
run(mailed, Report{Findings: []Finding{finding("memory", Warning, 0)}}, t0)
|
||||
|
||||
@@ -17,8 +17,8 @@ const (
|
||||
LabelManagedBy = "app.kubernetes.io/managed-by"
|
||||
LabelComponent = "app.kubernetes.io/component"
|
||||
LabelServer = "felis.lolicon.best/server"
|
||||
// LabelMode is what the Job sends (ModeWorld, ModeBackup or ModeFiles). A
|
||||
// world or files export holds the world volume; a backup export does not.
|
||||
// LabelMode is what the Job archives (ModeWorld or ModeBackup). A world
|
||||
// export holds the world volume; a backup export does not.
|
||||
LabelMode = "felis.lolicon.best/export-mode"
|
||||
|
||||
managedByValue = "felis-export"
|
||||
@@ -30,12 +30,10 @@ const (
|
||||
felisBinaryPath = "/usr/local/bin/felis"
|
||||
)
|
||||
|
||||
// What an export sends: the whole world as a tar.gz, one stored backup as a
|
||||
// tar.gz, or one file or folder of the world (a folder as a zip).
|
||||
// The two things an export can archive.
|
||||
const (
|
||||
ModeWorld = "world"
|
||||
ModeBackup = "backup"
|
||||
ModeFiles = "files"
|
||||
)
|
||||
|
||||
// TokenEnv carries the one-time upload token into the Pod. It is the only
|
||||
@@ -43,17 +41,6 @@ const (
|
||||
// process listing on the node would show it.
|
||||
const TokenEnv = "FELIS_EXPORT_TOKEN"
|
||||
|
||||
// The Job's PUT goes chunked, so that it can end with a trailer: DigestTrailer
|
||||
// carries the SHA-256 of every byte it sent (sha-256=:<base64>:, RFC 9530),
|
||||
// which felis-api checks before the last of them reaches the browser.
|
||||
// LengthHeader declares the length up front when the Job knows it (one file),
|
||||
// which chunked encoding cannot carry, so the browser still gets a
|
||||
// Content-Length.
|
||||
const (
|
||||
DigestTrailer = "Content-Digest"
|
||||
LengthHeader = "X-Felis-Export-Length"
|
||||
)
|
||||
|
||||
// JobParams are the rendered inputs to an export Job. ExportJob is a pure
|
||||
// function of them, so the Job shape is unit-tested without a cluster.
|
||||
type JobParams struct {
|
||||
@@ -62,19 +49,10 @@ type JobParams struct {
|
||||
// upload path, so two exports of one server never collide.
|
||||
ID string
|
||||
Mode string
|
||||
// WorldPVC is mounted for ModeWorld and ModeFiles, BackupPVC and BackupRef
|
||||
// for ModeBackup.
|
||||
// WorldPVC is mounted for ModeWorld, BackupPVC and BackupRef for ModeBackup.
|
||||
WorldPVC string
|
||||
BackupPVC string
|
||||
BackupRef string
|
||||
// BackupSHA256 is what the stored archive must hash to. The Job checks it
|
||||
// itself, because what it sends is the archive re-written without its
|
||||
// secrets and no longer hashes to anything felis-api knows.
|
||||
BackupSHA256 string
|
||||
// Path is the file or folder a ModeFiles export sends, and Dir whether the
|
||||
// caller saw a folder there.
|
||||
Path string
|
||||
Dir bool
|
||||
// TargetURL is where the Pod PUTs the archive (felis-api's internal face),
|
||||
// and Token the one-time bearer token that opens it.
|
||||
TargetURL string
|
||||
@@ -140,20 +118,11 @@ func ExportJob(p JobParams) (*batchv1.Job, error) {
|
||||
mount corev1.VolumeMount
|
||||
)
|
||||
switch p.Mode {
|
||||
case ModeWorld, ModeFiles:
|
||||
case ModeWorld:
|
||||
if p.WorldPVC == "" {
|
||||
return nil, fmt.Errorf("worldexport: world PVC name is required")
|
||||
}
|
||||
args = append(args, "--worlds-root", p.WorldsRoot)
|
||||
if p.Mode == ModeFiles {
|
||||
if p.Path == "" {
|
||||
return nil, fmt.Errorf("worldexport: a files export needs a path")
|
||||
}
|
||||
args = append(args, "--path", p.Path)
|
||||
if p.Dir {
|
||||
args = append(args, "--dir")
|
||||
}
|
||||
}
|
||||
volume = readOnlyClaim(worldVolume, p.WorldPVC)
|
||||
mount = corev1.VolumeMount{Name: worldVolume, MountPath: p.WorldsRoot, ReadOnly: true}
|
||||
case ModeBackup:
|
||||
@@ -161,9 +130,6 @@ func ExportJob(p JobParams) (*batchv1.Job, error) {
|
||||
return nil, fmt.Errorf("worldexport: backup PVC name and archive ref are required")
|
||||
}
|
||||
args = append(args, "--ref", p.BackupRef, "--backup-root", p.BackupRoot)
|
||||
if p.BackupSHA256 != "" {
|
||||
args = append(args, "--sha256", p.BackupSHA256)
|
||||
}
|
||||
volume = readOnlyClaim(backupVolume, p.BackupPVC)
|
||||
mount = corev1.VolumeMount{Name: backupVolume, MountPath: p.BackupRoot, ReadOnly: true}
|
||||
default:
|
||||
|
||||
@@ -8,11 +8,8 @@ import (
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
k8stesting "k8s.io/client-go/testing"
|
||||
)
|
||||
|
||||
const secretToken = "5ecret5ecret5ecret5ecret5ecret5ecret5ecret5ecret5ecret5ecret5ecr"
|
||||
@@ -21,8 +18,7 @@ func params(mode string) JobParams {
|
||||
return JobParams{
|
||||
Server: "survival", ID: "0011223344556677", Mode: mode,
|
||||
WorldPVC: "world-survival-0", BackupPVC: "felis-backups",
|
||||
BackupRef: "/backups/survival-1.tar.gz", BackupSHA256: strings.Repeat("ab", 32),
|
||||
Path: "plugins/Essentials", Dir: true,
|
||||
BackupRef: "/backups/survival-1.tar.gz",
|
||||
TargetURL: "http://felis-api-internal.felis.svc.cluster.local:8081/api/v1/internal/exports/0011223344556677",
|
||||
Token: secretToken,
|
||||
Namespace: "minecraft", ServiceAccount: "felis-restore", Image: "felis:1",
|
||||
@@ -41,8 +37,7 @@ func TestExportJobIsolation(t *testing.T) {
|
||||
args []string
|
||||
}{
|
||||
{ModeWorld, worldVolume, "world-survival-0", "/world", []string{"--worlds-root", "/world"}},
|
||||
{ModeBackup, backupVolume, "felis-backups", "/backups", []string{"--ref", "/backups/survival-1.tar.gz", "--backup-root", "/backups", "--sha256", strings.Repeat("ab", 32)}},
|
||||
{ModeFiles, worldVolume, "world-survival-0", "/world", []string{"--worlds-root", "/world", "--path", "plugins/Essentials", "--dir"}},
|
||||
{ModeBackup, backupVolume, "felis-backups", "/backups", []string{"--ref", "/backups/survival-1.tar.gz", "--backup-root", "/backups"}},
|
||||
} {
|
||||
job, err := ExportJob(params(tc.mode))
|
||||
if err != nil {
|
||||
@@ -137,40 +132,6 @@ func TestExportJobRefusesIncompleteParams(t *testing.T) {
|
||||
if _, err := ExportJob(p); err == nil {
|
||||
t.Error("a backup export without a ref was accepted")
|
||||
}
|
||||
p = params(ModeFiles)
|
||||
p.Path = ""
|
||||
if _, err := ExportJob(p); err == nil {
|
||||
t.Error("a files export without a path was accepted")
|
||||
}
|
||||
p = params(ModeFiles)
|
||||
p.WorldPVC = ""
|
||||
if _, err := ExportJob(p); err == nil {
|
||||
t.Error("a files export without a world claim was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestExportJobOptionalArgs: a backup with no recorded digest carries no
|
||||
// --sha256, and a file download carries its path and no --dir.
|
||||
func TestExportJobOptionalArgs(t *testing.T) {
|
||||
p := params(ModeBackup)
|
||||
p.BackupSHA256 = ""
|
||||
job, err := ExportJob(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"--ref", "/backups/survival-1.tar.gz", "--backup-root", "/backups"}
|
||||
if args := job.Spec.Template.Spec.Containers[0].Args; !slices.Equal(args[len(args)-len(want):], want) || slices.Contains(args, "--sha256") {
|
||||
t.Errorf("backup args = %v", args)
|
||||
}
|
||||
p = params(ModeFiles)
|
||||
p.Dir = false
|
||||
if job, err = ExportJob(p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want = []string{"--worlds-root", "/world", "--path", "plugins/Essentials"}
|
||||
if args := job.Spec.Template.Spec.Containers[0].Args; !slices.Equal(args[len(args)-len(want):], want) || slices.Contains(args, "--dir") {
|
||||
t.Errorf("file args = %v", args)
|
||||
}
|
||||
}
|
||||
|
||||
// TestExportJobDefaults: a caller that leaves the deadline and the TTL unset
|
||||
@@ -214,59 +175,4 @@ func TestStartCreatesTheJob(t *testing.T) {
|
||||
TargetURL: "http://api:8081/x", Token: secretToken}); err == nil {
|
||||
t.Error("a second Job of the same name was reported as created")
|
||||
}
|
||||
|
||||
// Each mode's own fields reach the Pod's arguments.
|
||||
for _, tc := range []struct {
|
||||
r Request
|
||||
tail []string
|
||||
}{
|
||||
{Request{Server: "survival", Mode: ModeBackup, ID: "1111111111111111", BackupRef: "/backups/a.tar.gz", BackupSHA256: strings.Repeat("cd", 32)},
|
||||
[]string{"--ref", "/backups/a.tar.gz", "--backup-root", "/backups", "--sha256", strings.Repeat("cd", 32)}},
|
||||
{Request{Server: "survival", Mode: ModeFiles, ID: "2222222222222222", Path: "plugins/Essentials", Dir: true},
|
||||
[]string{"--worlds-root", "/world", "--path", "plugins/Essentials", "--dir"}},
|
||||
} {
|
||||
tc.r.TargetURL, tc.r.Token = "http://api:8081/x", secretToken
|
||||
name, err := e.Start(context.Background(), tc.r)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: Start: %v", tc.r.Mode, err)
|
||||
}
|
||||
job, err := cs.BatchV1().Jobs("minecraft").Get(context.Background(), name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if args := job.Spec.Template.Spec.Containers[0].Args; !slices.Equal(args[len(args)-len(tc.tail):], tc.tail) {
|
||||
t.Errorf("%s: args = %v, want them to end %v", tc.r.Mode, args, tc.tail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A Job felis-api gave up on goes with its Pods, and one already gone is no
|
||||
// error: the sweep that stops it may run after the Job's own TTL took it.
|
||||
func TestStopDeletesTheJobAndItsPods(t *testing.T) {
|
||||
cs := fake.NewSimpleClientset()
|
||||
var policy metav1.DeletionPropagation
|
||||
cs.PrependReactor("delete", "jobs", func(a k8stesting.Action) (bool, runtime.Object, error) {
|
||||
if o := a.(k8stesting.DeleteActionImpl).GetDeleteOptions().PropagationPolicy; o != nil {
|
||||
policy = *o
|
||||
}
|
||||
return false, nil, nil
|
||||
})
|
||||
e := New(cs, Config{Image: "felis:1", BackupPVC: "felis-backups"})
|
||||
name, err := e.Start(context.Background(), Request{Server: "survival", Mode: ModeWorld, ID: "0011223344556677",
|
||||
TargetURL: "http://api:8081/x", Token: secretToken})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.Stop(context.Background(), name); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if _, err := cs.BatchV1().Jobs("minecraft").Get(context.Background(), name, metav1.GetOptions{}); !apierrors.IsNotFound(err) {
|
||||
t.Fatalf("the Job is still there: %v", err)
|
||||
}
|
||||
if policy != metav1.DeletePropagationBackground {
|
||||
t.Errorf("propagation = %q, want Background so its Pods go too", policy)
|
||||
}
|
||||
if err := e.Stop(context.Background(), name); err != nil {
|
||||
t.Errorf("stopping a Job already gone: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,11 @@
|
||||
// Package worldexport is the executor behind the world export routes (POST
|
||||
// /servers/{name}/world/export and POST /servers/{name}/backups/{id}/export)
|
||||
// and the file manager's download (POST /servers/{name}/files/download): a
|
||||
// /servers/{name}/world/export and POST /servers/{name}/backups/{id}/export): a
|
||||
// one-shot Job in the minecraft namespace that reads a stopped server's world,
|
||||
// one of its stored archives, or one file or folder of the world, and PUTs it
|
||||
// to felis-api's internal face, which streams it on to the owner's browser as
|
||||
// it arrives (internal/api/exports.go). Nothing is staged on the way: felis-api
|
||||
// never mounts a world or the backup store, and what is sent never lands on a
|
||||
// disk it owns.
|
||||
// or one of its stored archives, and PUTs the tar.gz to felis-api's internal
|
||||
// face, which streams it on to the owner's browser as it arrives
|
||||
// (internal/api/exports.go). Nothing is staged on the way: felis-api never
|
||||
// mounts a world or the backup store, and the archive never lands on a disk it
|
||||
// owns.
|
||||
//
|
||||
// Trust model as in internal/restore: the Pod runs under the weak felis-restore
|
||||
// SA with no API token, mounts one volume read-only, and holds no database URL
|
||||
@@ -20,7 +19,6 @@ import (
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/naming"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
@@ -28,14 +26,9 @@ import (
|
||||
// Request is one export as felis-api admitted it.
|
||||
type Request struct {
|
||||
Server string
|
||||
Mode string // ModeWorld, ModeBackup or ModeFiles
|
||||
// BackupRef is the archive a ModeBackup export reads, and BackupSHA256 what
|
||||
// it must hash to.
|
||||
Mode string // ModeWorld or ModeBackup
|
||||
// BackupRef is the archive a ModeBackup export reads.
|
||||
BackupRef string
|
||||
BackupSHA256 string
|
||||
// Path and Dir are the file or folder a ModeFiles export sends.
|
||||
Path string
|
||||
Dir bool
|
||||
// ID names the export (JobName) and TargetURL/Token are where and how the
|
||||
// Pod hands the archive over.
|
||||
ID string
|
||||
@@ -115,7 +108,6 @@ func (e *Exporter) Start(ctx context.Context, r Request) (string, error) {
|
||||
job, err := ExportJob(JobParams{
|
||||
Server: r.Server, ID: r.ID, Mode: r.Mode,
|
||||
WorldPVC: naming.WorldPVCName(r.Server), BackupPVC: c.BackupPVC, BackupRef: r.BackupRef,
|
||||
BackupSHA256: r.BackupSHA256, Path: r.Path, Dir: r.Dir,
|
||||
TargetURL: r.TargetURL, Token: r.Token,
|
||||
Namespace: c.Namespace, ServiceAccount: c.ServiceAccount, Image: c.Image,
|
||||
BackupRoot: c.BackupRoot, WorldsRoot: c.WorldsRoot, Deadline: c.Deadline,
|
||||
@@ -131,15 +123,3 @@ func (e *Exporter) Start(ctx context.Context, r Request) (string, error) {
|
||||
}
|
||||
return job.Name, nil
|
||||
}
|
||||
|
||||
// Stop deletes an export Job felis-api has given up on, so a Pod that never got
|
||||
// going stops holding the world. Its Pods go with it; one already gone is not
|
||||
// an error.
|
||||
func (e *Exporter) Stop(ctx context.Context, job string) error {
|
||||
bg := metav1.DeletePropagationBackground
|
||||
err := e.cs.BatchV1().Jobs(e.cfg.Namespace).Delete(ctx, job, metav1.DeleteOptions{PropagationPolicy: &bg})
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
return fmt.Errorf("worldexport: delete export job: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Generated
+103
-119
@@ -20,7 +20,7 @@
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
"react-i18next": "^17.0.8",
|
||||
"react-router-dom": "^7.18.4",
|
||||
"react-router-dom": "^6.28.2",
|
||||
"tailwind-merge": "^2.6.0",
|
||||
"three": "^0.172.0"
|
||||
},
|
||||
@@ -41,13 +41,13 @@
|
||||
"globals": "^17.12.0",
|
||||
"jsdom": "^30.1.1",
|
||||
"openapi-typescript": "^7.13.0",
|
||||
"postcss": "^8.5.28",
|
||||
"postcss": "^8.5.1",
|
||||
"tailwindcss": "^3.4.17",
|
||||
"tailwindcss-animate": "^1.0.7",
|
||||
"typescript": "^5.7.3",
|
||||
"typescript-eslint": "^8.70.1",
|
||||
"vite": "^6.0.7",
|
||||
"vitest": "^4.1.11"
|
||||
"vitest": "^4.1.9"
|
||||
}
|
||||
},
|
||||
"node_modules/@alloc/quick-lru": {
|
||||
@@ -2066,6 +2066,15 @@
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/@remix-run/router": {
|
||||
"version": "1.23.3",
|
||||
"resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.3.tgz",
|
||||
"integrity": "sha512-4An71tdz9X8+3sI4Qqqd2LWd9vS39J7sqd9EU4Scw7TJE/qB10Flv/UuqbPVgfQV9XoK8Np6jNquZitnZq5i+Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@rolldown/pluginutils": {
|
||||
"version": "1.0.0-beta.27",
|
||||
"resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz",
|
||||
@@ -2922,16 +2931,16 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/expect": {
|
||||
"version": "4.1.11",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz",
|
||||
"integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==",
|
||||
"version": "4.1.9",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.9.tgz",
|
||||
"integrity": "sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@standard-schema/spec": "^1.1.0",
|
||||
"@types/chai": "^5.2.2",
|
||||
"@vitest/spy": "4.1.11",
|
||||
"@vitest/utils": "4.1.11",
|
||||
"@vitest/spy": "4.1.9",
|
||||
"@vitest/utils": "4.1.9",
|
||||
"chai": "^6.2.2",
|
||||
"tinyrainbow": "^3.1.0"
|
||||
},
|
||||
@@ -2940,13 +2949,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/mocker": {
|
||||
"version": "4.1.11",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz",
|
||||
"integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==",
|
||||
"version": "4.1.9",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.9.tgz",
|
||||
"integrity": "sha512-EVkXzBjrPGM+cK8/ANWgBrkUCfJfb38/EfTSO8h7pWvKkyPkpWxvR7BkD2MyItMF62C97zAEoqdpUixwR/e+Rw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@vitest/spy": "4.1.11",
|
||||
"@vitest/spy": "4.1.9",
|
||||
"estree-walker": "^3.0.3",
|
||||
"magic-string": "^0.30.21"
|
||||
},
|
||||
@@ -2967,9 +2976,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/pretty-format": {
|
||||
"version": "4.1.11",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz",
|
||||
"integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==",
|
||||
"version": "4.1.9",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.9.tgz",
|
||||
"integrity": "sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -2980,13 +2989,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/runner": {
|
||||
"version": "4.1.11",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz",
|
||||
"integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==",
|
||||
"version": "4.1.9",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.9.tgz",
|
||||
"integrity": "sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@vitest/utils": "4.1.11",
|
||||
"@vitest/utils": "4.1.9",
|
||||
"pathe": "^2.0.3"
|
||||
},
|
||||
"funding": {
|
||||
@@ -2994,14 +3003,14 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/snapshot": {
|
||||
"version": "4.1.11",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz",
|
||||
"integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==",
|
||||
"version": "4.1.9",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.9.tgz",
|
||||
"integrity": "sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@vitest/pretty-format": "4.1.11",
|
||||
"@vitest/utils": "4.1.11",
|
||||
"@vitest/pretty-format": "4.1.9",
|
||||
"@vitest/utils": "4.1.9",
|
||||
"magic-string": "^0.30.21",
|
||||
"pathe": "^2.0.3"
|
||||
},
|
||||
@@ -3010,9 +3019,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/spy": {
|
||||
"version": "4.1.11",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz",
|
||||
"integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==",
|
||||
"version": "4.1.9",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.9.tgz",
|
||||
"integrity": "sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
@@ -3020,13 +3029,13 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/utils": {
|
||||
"version": "4.1.11",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz",
|
||||
"integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==",
|
||||
"version": "4.1.9",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.9.tgz",
|
||||
"integrity": "sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@vitest/pretty-format": "4.1.11",
|
||||
"@vitest/pretty-format": "4.1.9",
|
||||
"convert-source-map": "^2.0.0",
|
||||
"tinyrainbow": "^3.1.0"
|
||||
},
|
||||
@@ -3239,9 +3248,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/baseline-browser-mapping": {
|
||||
"version": "2.11.26",
|
||||
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.26.tgz",
|
||||
"integrity": "sha512-GLQdD3y6UF8iVuMJl5fHgE4jdn/ua7n+toKfLgNlg3BqQtOZjpy68T8Tup8/wGWZCDlm7KMg7tPb4MPn7oN0TQ==",
|
||||
"version": "2.10.38",
|
||||
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.38.tgz",
|
||||
"integrity": "sha512-31/02mVB4yuQU6adKk5SlY6m+mxDwUq5KZkyYgnLrrKl7TEm1+3PyDtDBz2kOv/wxZz41GHsvV1A/u6RmiyBvw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
@@ -3301,9 +3310,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/browserslist": {
|
||||
"version": "4.29.2",
|
||||
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.2.tgz",
|
||||
"integrity": "sha512-/u9r8k8ue4ZhHCw9ovDtltpWdXeXzjhdxCGj6vm1RGLPXb8lu33EMdoCHF5Sx8oYBS+Q/FYYNmAqlHncxR3RmA==",
|
||||
"version": "4.28.4",
|
||||
"resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.4.tgz",
|
||||
"integrity": "sha512-MTc8i/x9jBQd1iMw2CFGS+rwMa07eYjLR0CCTLDACl9xhxy+nIs3KeML/biicXtk9JrZ6dnnTatmc7ErPXIxqw==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -3321,11 +3330,11 @@
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"baseline-browser-mapping": "^2.11.26",
|
||||
"caniuse-lite": "^1.0.30001812",
|
||||
"electron-to-chromium": "^1.5.439",
|
||||
"node-releases": "^2.0.57",
|
||||
"update-browserslist-db": "^1.3.3"
|
||||
"baseline-browser-mapping": "^2.10.38",
|
||||
"caniuse-lite": "^1.0.30001799",
|
||||
"electron-to-chromium": "^1.5.376",
|
||||
"node-releases": "^2.0.48",
|
||||
"update-browserslist-db": "^1.2.3"
|
||||
},
|
||||
"bin": {
|
||||
"browserslist": "cli.js"
|
||||
@@ -3359,9 +3368,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/caniuse-lite": {
|
||||
"version": "1.0.30001813",
|
||||
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001813.tgz",
|
||||
"integrity": "sha512-zfjJo4rM0+fUomGDBW/xcDjhIwz/210DGvip2MAMDZ8KHcRPnOHmEgHPZP0UHlZoxr8fYKVJOqcORhYQcG4FKQ==",
|
||||
"version": "1.0.30001799",
|
||||
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001799.tgz",
|
||||
"integrity": "sha512-hG1bReV+OUU+MOqK4t/ZWI0tZOyz3rqS9XuhOUz1cIcbwBKjOyJEJuw9ER5JuNyqxNk8u/JUVbGibBOL1yrjFw==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -3479,19 +3488,6 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/cookie": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz",
|
||||
"integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/express"
|
||||
}
|
||||
},
|
||||
"node_modules/cross-spawn": {
|
||||
"version": "7.0.6",
|
||||
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
|
||||
@@ -3640,9 +3636,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/electron-to-chromium": {
|
||||
"version": "1.5.440",
|
||||
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.440.tgz",
|
||||
"integrity": "sha512-SghDzqdJokdz8zP8YNlvS74+CwLRoUPDGvP/gFA+HrKVw+pOshUAgx8pXR37xl/u16zGae746rFNWvZ/22kU2Q==",
|
||||
"version": "1.5.378",
|
||||
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.378.tgz",
|
||||
"integrity": "sha512-VinvOAuuPmdD1guEgGv5f2Qp7/vlfqOrUOMYNnOD4wj3pit8kRsQHzfIf6teyUGWo15Tg5+bOJaRunvyltpVWQ==",
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
@@ -4703,9 +4699,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/nanoid": {
|
||||
"version": "3.3.19",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz",
|
||||
"integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==",
|
||||
"version": "3.3.15",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz",
|
||||
"integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -4729,9 +4725,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/node-releases": {
|
||||
"version": "2.0.57",
|
||||
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.57.tgz",
|
||||
"integrity": "sha512-kQK9LGGFiHtrWiNhZtA7Qbw17AQz+dmsEKODRIVTXA9+e5MS/2gZEBhYJt13GrAz5/IOZKddH/0Z3TP/Zgo+yw==",
|
||||
"version": "2.0.50",
|
||||
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.50.tgz",
|
||||
"integrity": "sha512-J6l92tKHX6w8Jy5nO1Vuc01NoIiRGi/d6qBKVxh+IQ8Cr3b6HbVNfKiF8ZpFKufTwpwxMmce2W3iQZ861ZRyTg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -4998,9 +4994,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/postcss": {
|
||||
"version": "8.5.28",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz",
|
||||
"integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==",
|
||||
"version": "8.5.15",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz",
|
||||
"integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -5018,7 +5014,7 @@
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"nanoid": "^3.3.18",
|
||||
"nanoid": "^3.3.12",
|
||||
"picocolors": "^1.1.1",
|
||||
"source-map-js": "^1.2.1"
|
||||
},
|
||||
@@ -5353,41 +5349,35 @@
|
||||
}
|
||||
},
|
||||
"node_modules/react-router": {
|
||||
"version": "7.18.4",
|
||||
"resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.4.tgz",
|
||||
"integrity": "sha512-PUPQcMhMGRAslLcvtlPz/kmzBEWPhLdgLFrL7pLNepBL6dX0lWj4WD2cUYVgYCuT3jxvghYFg81cDTj44DhetQ==",
|
||||
"version": "6.30.4",
|
||||
"resolved": "https://registry.npmjs.org/react-router/-/react-router-6.30.4.tgz",
|
||||
"integrity": "sha512-SVUsDe+DybHM/WmYKIVYhZh1o5Dcuf16yM6WjG02Q9XVFMZIJyHYhwrr6bFBXZkVP6z69kNkMyBCujt8FaFLJA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cookie": "^1.0.1",
|
||||
"set-cookie-parser": "^2.6.0"
|
||||
"@remix-run/router": "1.23.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
"node": ">=14.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": ">=18",
|
||||
"react-dom": ">=18"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"react-dom": {
|
||||
"optional": true
|
||||
}
|
||||
"react": ">=16.8"
|
||||
}
|
||||
},
|
||||
"node_modules/react-router-dom": {
|
||||
"version": "7.18.4",
|
||||
"resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.18.4.tgz",
|
||||
"integrity": "sha512-yrfmJHIpDG7taCpqKjT1G5B6q3O2K+RN8/fgNf0lTjCwiPbQ0ei6vXX9ZjQR+7ld8Tr7Z5xmyMnZ8YJrphWQUw==",
|
||||
"version": "6.30.4",
|
||||
"resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.30.4.tgz",
|
||||
"integrity": "sha512-q4HvNl+mmDdkS0g+MqiBZNteQJCuimWoOyHMy4T/RQLAn9Z29+E91QXRaxOujeMl2HTzRSS0KFPd7lxX3PjV0Q==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"react-router": "7.18.4"
|
||||
"@remix-run/router": "1.23.3",
|
||||
"react-router": "6.30.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
"node": ">=14.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"react": ">=18",
|
||||
"react-dom": ">=18"
|
||||
"react": ">=16.8",
|
||||
"react-dom": ">=16.8"
|
||||
}
|
||||
},
|
||||
"node_modules/react-style-singleton": {
|
||||
@@ -5579,12 +5569,6 @@
|
||||
"semver": "bin/semver.js"
|
||||
}
|
||||
},
|
||||
"node_modules/set-cookie-parser": {
|
||||
"version": "2.7.2",
|
||||
"resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz",
|
||||
"integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/shebang-command": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
|
||||
@@ -5841,9 +5825,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/tinyrainbow": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz",
|
||||
"integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==",
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz",
|
||||
"integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
@@ -6017,9 +6001,9 @@
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/update-browserslist-db": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.3.tgz",
|
||||
"integrity": "sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==",
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz",
|
||||
"integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
@@ -6230,19 +6214,19 @@
|
||||
}
|
||||
},
|
||||
"node_modules/vitest": {
|
||||
"version": "4.1.11",
|
||||
"resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz",
|
||||
"integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==",
|
||||
"version": "4.1.9",
|
||||
"resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.9.tgz",
|
||||
"integrity": "sha512-nE3/LEyc0z87uHYLZebqCUOaJr2hdtuPp7BQ4BosVFnfltxgAvMG08NyrSGlPpOUWvR27c5flSmYFTNr78L9GQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@vitest/expect": "4.1.11",
|
||||
"@vitest/mocker": "4.1.11",
|
||||
"@vitest/pretty-format": "4.1.11",
|
||||
"@vitest/runner": "4.1.11",
|
||||
"@vitest/snapshot": "4.1.11",
|
||||
"@vitest/spy": "4.1.11",
|
||||
"@vitest/utils": "4.1.11",
|
||||
"@vitest/expect": "4.1.9",
|
||||
"@vitest/mocker": "4.1.9",
|
||||
"@vitest/pretty-format": "4.1.9",
|
||||
"@vitest/runner": "4.1.9",
|
||||
"@vitest/snapshot": "4.1.9",
|
||||
"@vitest/spy": "4.1.9",
|
||||
"@vitest/utils": "4.1.9",
|
||||
"es-module-lexer": "^2.0.0",
|
||||
"expect-type": "^1.3.0",
|
||||
"magic-string": "^0.30.21",
|
||||
@@ -6270,12 +6254,12 @@
|
||||
"@edge-runtime/vm": "*",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0",
|
||||
"@vitest/browser-playwright": "4.1.11",
|
||||
"@vitest/browser-preview": "4.1.11",
|
||||
"@vitest/browser-webdriverio": "4.1.11",
|
||||
"@vitest/coverage-istanbul": "4.1.11",
|
||||
"@vitest/coverage-v8": "4.1.11",
|
||||
"@vitest/ui": "4.1.11",
|
||||
"@vitest/browser-playwright": "4.1.9",
|
||||
"@vitest/browser-preview": "4.1.9",
|
||||
"@vitest/browser-webdriverio": "4.1.9",
|
||||
"@vitest/coverage-istanbul": "4.1.9",
|
||||
"@vitest/coverage-v8": "4.1.9",
|
||||
"@vitest/ui": "4.1.9",
|
||||
"happy-dom": "*",
|
||||
"jsdom": "*",
|
||||
"vite": "^6.0.0 || ^7.0.0 || ^8.0.0"
|
||||
|
||||
+3
-3
@@ -28,7 +28,7 @@
|
||||
"react": "^18.3.1",
|
||||
"react-dom": "^18.3.1",
|
||||
"react-i18next": "^17.0.8",
|
||||
"react-router-dom": "^7.18.4",
|
||||
"react-router-dom": "^6.28.2",
|
||||
"tailwind-merge": "^2.6.0",
|
||||
"three": "^0.172.0"
|
||||
},
|
||||
@@ -49,12 +49,12 @@
|
||||
"globals": "^17.12.0",
|
||||
"jsdom": "^30.1.1",
|
||||
"openapi-typescript": "^7.13.0",
|
||||
"postcss": "^8.5.28",
|
||||
"postcss": "^8.5.1",
|
||||
"tailwindcss": "^3.4.17",
|
||||
"tailwindcss-animate": "^1.0.7",
|
||||
"typescript": "^5.7.3",
|
||||
"typescript-eslint": "^8.70.1",
|
||||
"vite": "^6.0.7",
|
||||
"vitest": "^4.1.11"
|
||||
"vitest": "^4.1.9"
|
||||
}
|
||||
}
|
||||
@@ -23,8 +23,6 @@ interface Props {
|
||||
/** Runs the action. The dialog closes when it resolves; a rejection is shown in
|
||||
* the dialog, which stays open so the action can be retried or dismissed. */
|
||||
onConfirm: () => Promise<void>;
|
||||
/** Shown under the description, for what a sentence cannot hold (a list). */
|
||||
children?: ReactNode;
|
||||
}
|
||||
|
||||
// ConfirmDialog asks before an action that cannot be taken back, in the panel's
|
||||
@@ -40,7 +38,6 @@ export function ConfirmDialog({
|
||||
cancelLabel,
|
||||
confirmVariant = "destructive",
|
||||
onConfirm,
|
||||
children,
|
||||
}: Props) {
|
||||
const { t } = useTranslation("common");
|
||||
const [busy, setBusy] = useState(false);
|
||||
@@ -72,7 +69,6 @@ export function ConfirmDialog({
|
||||
<DialogTitle>{title}</DialogTitle>
|
||||
{description && <DialogDescription>{description}</DialogDescription>}
|
||||
</DialogHeader>
|
||||
{children}
|
||||
{error && <MessageLine kind="error" message={error} compact />}
|
||||
<ConfirmFooter
|
||||
onCancel={() => setOpen(false)}
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
// @vitest-environment jsdom
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import { fireEvent, render, screen, within } from "@testing-library/react";
|
||||
import i18next from "i18next";
|
||||
import type { FileOp } from "@/lib/types";
|
||||
import { humanizeError } from "@/lib/api";
|
||||
import { FileOps } from "./FileOps";
|
||||
import { opErrorText } from "./opText";
|
||||
|
||||
const t = (key: string, opts?: Record<string, unknown>) => i18next.t(`files:${key}`, opts ?? {});
|
||||
|
||||
function op(over: Partial<FileOp> = {}): FileOp {
|
||||
return { id: "a", op: "unzip", path: "pack.zip", state: "running", started_at: "2026-09-28T00:00:00Z", done: 0, total: 0, ...over };
|
||||
}
|
||||
|
||||
function show(ops: FileOp[], error: unknown = null) {
|
||||
const onDismiss = vi.fn();
|
||||
const onConflicts = vi.fn();
|
||||
const view = render(<FileOps ops={ops} error={error} onDismiss={onDismiss} onConflicts={onConflicts} />);
|
||||
return { ...view, onDismiss, onConflicts };
|
||||
}
|
||||
|
||||
describe("FileOps", () => {
|
||||
it("shows nothing with no ops and nothing to say", () => {
|
||||
const { container } = show([]);
|
||||
expect(container.innerHTML).toBe("");
|
||||
});
|
||||
|
||||
it("counts a running op in whole percents, rounding down and never past 100", () => {
|
||||
show([op({ id: "a", path: "a.zip", done: 999, total: 1000 }), op({ id: "b", path: "b.zip", done: 5, total: 4 })]);
|
||||
|
||||
expect(screen.getByText(t("op_progress", { done: "999 B", total: "1000 B", percent: 99 }))).toBeTruthy();
|
||||
expect(screen.getByText(t("op_progress", { done: "5 B", total: "4 B", percent: 100 }))).toBeTruthy();
|
||||
const bars = screen.getAllByRole("progressbar");
|
||||
expect(bars.map((b) => [b.getAttribute("aria-label"), b.getAttribute("aria-valuenow")])).toEqual([
|
||||
[t("op_progress_label", { path: "a.zip" }), "99"],
|
||||
[t("op_progress_label", { path: "b.zip" }), "100"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("claims no progress before the Job has counted, and offers no dismissal while it runs", () => {
|
||||
show([op()]);
|
||||
|
||||
expect(screen.getByText(t("op_preparing"))).toBeTruthy();
|
||||
expect(screen.getByRole("progressbar").hasAttribute("aria-valuenow")).toBe(false);
|
||||
expect(screen.queryByRole("button")).toBeNull();
|
||||
});
|
||||
|
||||
it("words how each kind of op ended", () => {
|
||||
show([
|
||||
op({ id: "u", op: "upload", path: "big.jar", state: "succeeded" }),
|
||||
op({ id: "z", path: "pack.zip", state: "succeeded", files: 1, bytes: 2048 }),
|
||||
op({ id: "f", op: "upload", path: "lost.jar", state: "failed" }),
|
||||
]);
|
||||
|
||||
expect(screen.getByText(t("op_upload", { path: "big.jar" }))).toBeTruthy();
|
||||
expect(screen.getByText(t("op_upload_done"))).toBeTruthy();
|
||||
expect(screen.getByText(t("op_unzip", { path: "pack.zip" }))).toBeTruthy();
|
||||
expect(screen.getByText(t("op_unzip_done", { count: 1, bytes: "2.0 KiB" }))).toBeTruthy();
|
||||
// A failure the Job left unexplained still says something.
|
||||
expect(screen.getByText(opErrorText("upload", { code: "job_failed", message: "" }))).toBeTruthy();
|
||||
expect(screen.queryByRole("progressbar")).toBeNull();
|
||||
});
|
||||
|
||||
it("offers the conflicts of an extraction only, and dismisses each ended op by its id", () => {
|
||||
const exists = { code: "file_exists", message: "", conflicts: ["a.txt"] };
|
||||
const refused = op({ id: "z", path: "pack.zip", state: "failed", error: exists });
|
||||
const { onDismiss, onConflicts } = show([
|
||||
refused,
|
||||
op({ id: "u", op: "upload", path: "a.txt", state: "failed", error: exists }),
|
||||
op({ id: "bad", path: "bad.zip", state: "failed", error: { code: "archive_invalid", message: "" } }),
|
||||
]);
|
||||
|
||||
const conflicts = screen.getAllByRole("button", { name: t("op_conflicts") });
|
||||
expect(conflicts).toHaveLength(1);
|
||||
fireEvent.click(conflicts[0]);
|
||||
expect(onConflicts.mock.calls).toEqual([[refused]]);
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: t("op_dismiss", { path: "a.txt" }) }));
|
||||
expect(onDismiss.mock.calls).toEqual([["u"]]);
|
||||
});
|
||||
|
||||
it("says why the ops could not be read, over the ones it has", () => {
|
||||
const broke = { status: 500, code: "internal", message: "" };
|
||||
show([op({ state: "succeeded", files: 1, bytes: 8 })], broke);
|
||||
|
||||
const region = screen.getByRole("region", { name: t("ops_label") });
|
||||
expect(within(region).getByText(t("ops_refresh_failed", { reason: humanizeError(broke) }))).toBeTruthy();
|
||||
expect(within(region).getByText(t("op_unzip_done", { count: 1, bytes: "8 B" }))).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -1,134 +0,0 @@
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { AlertCircle, CheckCircle2, FileArchive, Loader2, Upload, X } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { MessageLine } from "@/components/MessageLine";
|
||||
import { humanizeError } from "@/lib/api";
|
||||
import { formatBytes } from "@/lib/format";
|
||||
import type { FileOp } from "@/lib/types";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { opErrorText } from "./opText";
|
||||
|
||||
interface Props {
|
||||
ops: readonly FileOp[];
|
||||
/** Why the latest read of the ops failed, if it did. */
|
||||
error: unknown;
|
||||
onDismiss: (id: string) => void;
|
||||
/** Opens the list of files an extraction stopped short of replacing. */
|
||||
onConflicts: (op: FileOp) => void;
|
||||
}
|
||||
|
||||
// FileOps shows the server's background operations: how far a running one has
|
||||
// got, and how each recent one ended, until it is dismissed.
|
||||
export function FileOps({ ops, error, onDismiss, onConflicts }: Props) {
|
||||
const { t } = useTranslation("files");
|
||||
if (ops.length === 0 && error == null) return null;
|
||||
|
||||
return (
|
||||
<section aria-label={t("ops_label")} className="border-b border-border bg-muted/20">
|
||||
{error != null && (
|
||||
<div className="px-4 pt-3">
|
||||
<MessageLine kind="error" message={t("ops_refresh_failed", { reason: humanizeError(error) })} compact />
|
||||
</div>
|
||||
)}
|
||||
<ul className="divide-y divide-border/60 px-4 py-1">
|
||||
{ops.map((op) => (
|
||||
<OpRow key={op.id} op={op} onDismiss={() => onDismiss(op.id)} onConflicts={() => onConflicts(op)} />
|
||||
))}
|
||||
</ul>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function OpRow({ op, onDismiss, onConflicts }: { op: FileOp; onDismiss: () => void; onConflicts: () => void }) {
|
||||
const { t } = useTranslation("files");
|
||||
const title = t(op.op === "unzip" ? "op_unzip" : "op_upload", { path: op.path });
|
||||
const Kind = op.op === "unzip" ? FileArchive : Upload;
|
||||
const counted = op.total > 0;
|
||||
const percent = counted ? Math.min(100, Math.floor((op.done * 100) / op.total)) : 0;
|
||||
const failure = op.error ?? { code: "job_failed", message: "" };
|
||||
const conflicts = op.state === "failed" && op.op === "unzip" && failure.code === "file_exists";
|
||||
|
||||
let status: string;
|
||||
if (op.state === "running") {
|
||||
status = counted
|
||||
? t("op_progress", { done: formatBytes(op.done), total: formatBytes(op.total), percent })
|
||||
: t("op_preparing");
|
||||
} else if (op.state === "succeeded") {
|
||||
status =
|
||||
op.op === "unzip"
|
||||
? t("op_unzip_done", { count: op.files ?? 0, bytes: formatBytes(op.bytes ?? 0) })
|
||||
: t("op_upload_done");
|
||||
} else {
|
||||
status = opErrorText(op.op, failure);
|
||||
}
|
||||
|
||||
const icon =
|
||||
op.state === "running" ? (
|
||||
<Loader2 className="h-4 w-4 animate-spin text-primary" />
|
||||
) : op.state === "succeeded" ? (
|
||||
<CheckCircle2 className="h-4 w-4 text-emerald-500" />
|
||||
) : (
|
||||
<AlertCircle className="h-4 w-4 text-destructive" />
|
||||
);
|
||||
|
||||
return (
|
||||
<li className="py-2">
|
||||
<div className="flex items-start gap-2.5">
|
||||
<span className="mt-0.5 shrink-0">{icon}</span>
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="flex min-w-0 items-center gap-1.5">
|
||||
<Kind className="h-3.5 w-3.5 shrink-0 text-muted-foreground" />
|
||||
<span className="truncate font-mono text-xs" title={op.path}>
|
||||
{title}
|
||||
</span>
|
||||
</p>
|
||||
<p
|
||||
className={cn(
|
||||
"mt-0.5 text-xs tabular-nums",
|
||||
op.state === "failed" ? "text-destructive" : "text-muted-foreground",
|
||||
)}
|
||||
>
|
||||
{status}
|
||||
</p>
|
||||
{op.state === "running" && (
|
||||
<div
|
||||
role="progressbar"
|
||||
aria-label={t("op_progress_label", { path: op.path })}
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={100}
|
||||
aria-valuenow={counted ? percent : undefined}
|
||||
className="mt-1.5 h-1 w-full overflow-hidden rounded-full bg-muted"
|
||||
>
|
||||
<div
|
||||
className={cn(
|
||||
"h-full rounded-full bg-primary transition-[width] duration-300 ease-out",
|
||||
!counted && "w-full animate-pulse",
|
||||
)}
|
||||
style={counted ? { width: `${percent}%` } : undefined}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{op.state !== "running" && (
|
||||
<div className="flex shrink-0 items-center gap-1">
|
||||
{conflicts && (
|
||||
<Button size="sm" variant="outline" className="h-7 text-xs" onClick={onConflicts}>
|
||||
{t("op_conflicts")}
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
className="h-7 w-7 p-0"
|
||||
onClick={onDismiss}
|
||||
aria-label={t("op_dismiss", { path: op.path })}
|
||||
title={t("op_dismiss", { path: op.path })}
|
||||
>
|
||||
<X className="h-3.5 w-3.5" />
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</li>
|
||||
);
|
||||
}
|
||||
@@ -79,16 +79,10 @@ function UploadRow({
|
||||
}) {
|
||||
const { t } = useTranslation("files");
|
||||
const { file, state } = item;
|
||||
// The body is all sent and the Job is landing it. A file sent in one request
|
||||
// lands in seconds, with no bytes left to count; one sent in parts is fetched
|
||||
// by its Job afresh, which reports how far it has got.
|
||||
const percent = file.size > 0 ? Math.min(100, Math.floor((item.sent * 100) / file.size)) : 100;
|
||||
// The body is all sent and the Job is landing it: seconds more, with no bytes
|
||||
// left to count.
|
||||
const landing = state === "uploading" && item.sent >= file.size;
|
||||
const written = landing && item.landing !== null && item.landing.total > 0 ? item.landing : null;
|
||||
const percent = written
|
||||
? Math.min(100, Math.floor((written.done * 100) / written.total))
|
||||
: file.size > 0
|
||||
? Math.min(100, Math.floor((item.sent * 100) / file.size))
|
||||
: 100;
|
||||
|
||||
const icon = {
|
||||
queued: <Clock className="h-4 w-4 text-muted-foreground" />,
|
||||
@@ -100,7 +94,6 @@ function UploadRow({
|
||||
|
||||
let status: string;
|
||||
if (state === "queued") status = t("upload_queued");
|
||||
else if (written) status = t("upload_landing_progress", { percent });
|
||||
else if (landing) status = t("upload_landing");
|
||||
else if (state === "uploading") status = t("upload_sending", { sent: formatBytes(item.sent), total: formatBytes(file.size), percent });
|
||||
else if (state === "done") status = t("upload_done");
|
||||
@@ -147,7 +140,7 @@ function UploadRow({
|
||||
<div
|
||||
className={cn(
|
||||
"h-full rounded-full bg-primary transition-[width] duration-300 ease-out",
|
||||
landing && !written && "animate-pulse",
|
||||
landing && "animate-pulse",
|
||||
)}
|
||||
style={{ width: `${percent}%` }}
|
||||
/>
|
||||
|
||||
@@ -25,12 +25,6 @@ export function isManaged(path: string): boolean {
|
||||
/** The longest name a Linux filesystem takes, in bytes (NAME_MAX). */
|
||||
const NAME_MAX = 255;
|
||||
|
||||
/** nameTooLong says whether a name is past NAME_MAX. A Mac or Windows disk
|
||||
* counts in characters, so a file picked there can be. */
|
||||
export function nameTooLong(name: string): boolean {
|
||||
return new TextEncoder().encode(name).length > NAME_MAX;
|
||||
}
|
||||
|
||||
export type NameProblem = "name_required" | "name_slash" | "name_dots" | "name_too_long" | "name_taken";
|
||||
|
||||
/** nameProblem says why name cannot be used for a new entry in a folder listing
|
||||
@@ -46,7 +40,7 @@ export function nameProblem(
|
||||
if (name === "") return "name_required";
|
||||
if (name.includes("/")) return "name_slash";
|
||||
if (name === "." || name === "..") return "name_dots";
|
||||
if (nameTooLong(name)) return "name_too_long";
|
||||
if (new TextEncoder().encode(name).length > NAME_MAX) return "name_too_long";
|
||||
if (name !== current && entries?.some((e) => e.name === name)) return "name_taken";
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { humanizeError } from "@/lib/api";
|
||||
import { opErrorText } from "./opText";
|
||||
|
||||
const e = (code: string, over: Record<string, unknown> = {}) => ({ code, message: "raw words", ...over });
|
||||
|
||||
describe("opErrorText", () => {
|
||||
it("tells an upload that found its file there from an extraction that would replace files", () => {
|
||||
expect(opErrorText("upload", e("file_exists"))).toBe(
|
||||
"A file with this name is already here, so nothing was replaced. Upload it again and choose Replace.",
|
||||
);
|
||||
expect(opErrorText("unzip", e("file_exists", { conflicts: ["a", "b"], conflict_count: 250 }))).toBe(
|
||||
"It would replace 250 files already here, so nothing was extracted yet.",
|
||||
);
|
||||
// Without the count, the list is counted.
|
||||
expect(opErrorText("unzip", e("file_exists", { conflicts: ["a"] }))).toBe(
|
||||
"It would replace 1 file already here, so nothing was extracted yet.",
|
||||
);
|
||||
});
|
||||
|
||||
it("names both sizes of a volume too small, and falls back when they are missing", () => {
|
||||
expect(opErrorText("unzip", e("volume_full", { need: 3 * 1024 * 1024, avail: 1024 }))).toBe(
|
||||
"Not enough room on the world volume: 3.0 MiB needed, 1.0 KiB free. Nothing was changed.",
|
||||
);
|
||||
// A full volume: the Job leaves the zero out.
|
||||
expect(opErrorText("upload", e("volume_full", { need: 2048 }))).toBe(
|
||||
"Not enough room on the world volume: 2.0 KiB needed, 0 B free. Nothing was changed.",
|
||||
);
|
||||
expect(opErrorText("upload", e("volume_full"))).toBe(humanizeError({ status: 0, code: "volume_full", message: "raw words" }));
|
||||
});
|
||||
|
||||
it("names the archive entry at fault", () => {
|
||||
expect(opErrorText("unzip", e("archive_invalid"))).toBe("The archive is damaged, or not a zip file. Nothing was changed.");
|
||||
expect(opErrorText("unzip", e("archive_invalid", { entry: "world/level.dat" }))).toBe(
|
||||
"world/level.dat in the archive is damaged (its size or checksum does not match). Nothing was changed.",
|
||||
);
|
||||
expect(opErrorText("unzip", e("archive_unsafe", { entry: "../../etc/passwd" }))).toBe(
|
||||
"../../etc/passwd in the archive would land outside this folder, or is a device file. Nothing in the archive was extracted.",
|
||||
);
|
||||
expect(opErrorText("unzip", e("archive_symlink", { entry: "world/link" }))).toBe(
|
||||
"world/link in the archive is a symbolic link. Nothing in the archive was extracted.",
|
||||
);
|
||||
expect(opErrorText("unzip", e("type_conflict", { entry: "plugins" }))).toBe(
|
||||
"plugins is a file on one side and a folder on the other, which replacing cannot resolve. Rename or delete plugins here, then extract again.",
|
||||
);
|
||||
});
|
||||
|
||||
it("gives a Job that ran out of time its own words", () => {
|
||||
expect(opErrorText("unzip", e("job_failed", { message: "job failed: DeadlineExceeded" }))).toBe(
|
||||
"The background task did not finish within 2 hours and was stopped. Refresh the list to check, then try again.",
|
||||
);
|
||||
expect(opErrorText("upload", e("job_failed", { message: "job failed: BackoffLimitExceeded" }))).toBe(
|
||||
"The background task ended without saying why. Refresh the list to check, then try again.",
|
||||
);
|
||||
});
|
||||
|
||||
it("tells an extraction killed for memory to split the archive", () => {
|
||||
// The message felis-api words a memory kill with.
|
||||
const oom = "the file operation ran out of memory (OOMKilled); an archive of this many files has to be split into smaller ones";
|
||||
expect(opErrorText("unzip", e("job_failed", { message: oom }))).toBe(
|
||||
"The archive holds more files than the extraction task has memory to list, so the system stopped it. The files on the server were not changed. Split it into several smaller zips and extract each one.",
|
||||
);
|
||||
expect(opErrorText("upload", e("job_failed", { message: oom }))).toBe(
|
||||
"The background task ran out of memory and the system stopped it. The files on the server were not changed. Try again.",
|
||||
);
|
||||
});
|
||||
|
||||
it("words any other code as the file routes do", () => {
|
||||
expect(opErrorText("upload", e("file_changed"))).toBe(humanizeError({ status: 0, code: "file_changed", message: "raw words" }));
|
||||
expect(opErrorText("upload", e("file_changed"))).not.toBe("raw words");
|
||||
});
|
||||
});
|
||||
@@ -1,37 +0,0 @@
|
||||
import i18next from "i18next";
|
||||
import { humanizeError } from "@/lib/api";
|
||||
import { formatBytes } from "@/lib/format";
|
||||
import type { FileOp, FileOpError } from "@/lib/types";
|
||||
|
||||
/** opErrorText says why an op ended failed. The codes an extraction refuses an
|
||||
* archive with name the entry at fault; the rest are the codes the file routes
|
||||
* answer, worded as they are there. */
|
||||
export function opErrorText(op: FileOp["op"], e: FileOpError): string {
|
||||
const t = i18next.getFixedT(null, "files");
|
||||
const entry = e.entry ?? "";
|
||||
switch (e.code) {
|
||||
case "file_exists":
|
||||
return op === "upload"
|
||||
? t("op_upload_exists")
|
||||
: t("op_unzip_conflicts", { count: e.conflict_count ?? e.conflicts?.length ?? 0 });
|
||||
// A volume with nothing left leaves avail out of the error, the way the Job
|
||||
// omits a zero.
|
||||
case "volume_full":
|
||||
return e.need !== undefined
|
||||
? t("op_volume_full", { need: formatBytes(e.need), avail: formatBytes(e.avail ?? 0) })
|
||||
: humanizeError({ status: 0, code: e.code, message: e.message });
|
||||
case "archive_invalid":
|
||||
return entry ? t("archive_invalid_entry", { entry }) : t("archive_invalid");
|
||||
case "archive_unsafe":
|
||||
case "archive_symlink":
|
||||
case "type_conflict":
|
||||
return t(e.code, { entry });
|
||||
// The Job's reason rides in the message; a deadline and a memory kill are
|
||||
// the ones worth their own words.
|
||||
case "job_failed":
|
||||
if (e.message.includes("OOMKilled")) return op === "unzip" ? t("job_out_of_memory_unzip") : t("job_out_of_memory");
|
||||
return e.message.includes("DeadlineExceeded") ? t("job_timed_out") : t("job_failed");
|
||||
default:
|
||||
return humanizeError({ status: 0, code: e.code, message: e.message });
|
||||
}
|
||||
}
|
||||
@@ -1,534 +0,0 @@
|
||||
// @vitest-environment jsdom
|
||||
import { createHash } from "node:crypto";
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import type { FileOp, FileUploadSession } from "@/lib/types";
|
||||
import { MAX_ATTEMPTS } from "@/lib/contextUpload";
|
||||
import {
|
||||
MAX_OP_MISSES,
|
||||
OP_POLL_MS,
|
||||
STALE_SESSION_MS,
|
||||
discardSession,
|
||||
forgetSession,
|
||||
sendInParts,
|
||||
watchOp,
|
||||
} from "./sessionUpload";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
getServerFileUpload: vi.fn(),
|
||||
beginServerFileUpload: vi.fn(),
|
||||
putServerFileUploadPart: vi.fn(),
|
||||
deleteServerFileUpload: vi.fn(),
|
||||
commitServerFileUpload: vi.fn(),
|
||||
listServerFileOps: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/api", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@/lib/api")>();
|
||||
return { ...actual, api: { ...actual.api, ...mocks } };
|
||||
});
|
||||
|
||||
const KEY = "felis-file-upload:lobby:world.zip";
|
||||
const PART = 4;
|
||||
const NOW = 1_000_000_000;
|
||||
const BODY = "0123456789";
|
||||
|
||||
// A 10-byte file sent in parts of 4: offsets 0, 4 and 8.
|
||||
function file(body = BODY, modified = 111) {
|
||||
return new File([body], "world.zip", { lastModified: modified });
|
||||
}
|
||||
|
||||
// The parts a server holding the first `received` bytes of body lists, as
|
||||
// they went up: PART bytes each, the last one short, hashed by Node.
|
||||
function partsOf(received: number, body = BODY) {
|
||||
const parts: FileUploadSession["parts"] = [];
|
||||
for (let at = 0; at < received; at += PART) {
|
||||
const end = Math.min(at + PART, received);
|
||||
parts.push({ size: end - at, sha256: createHash("sha256").update(body.slice(at, end)).digest("hex") });
|
||||
}
|
||||
return parts;
|
||||
}
|
||||
|
||||
function session(received: number, over: Partial<FileUploadSession> = {}): FileUploadSession {
|
||||
return { id: "s1", path: "world.zip", size: 10, received, part_max_bytes: PART, parts: partsOf(received), ...over };
|
||||
}
|
||||
|
||||
function op(over: Partial<FileOp> = {}): FileOp {
|
||||
return {
|
||||
id: "op1",
|
||||
op: "upload",
|
||||
path: "world.zip",
|
||||
state: "running",
|
||||
started_at: "2026-09-28T00:00:00Z",
|
||||
done: 0,
|
||||
total: 0,
|
||||
...over,
|
||||
};
|
||||
}
|
||||
|
||||
const netErr = { status: 0, code: "network_error", message: "" };
|
||||
const gone = { status: 404, code: "upload_not_found", message: "no such upload" };
|
||||
const held = { status: 409, code: "maintenance_in_progress", message: "held" };
|
||||
|
||||
// The server takes every part whole and reports where the session stands.
|
||||
function serverTakesParts() {
|
||||
mocks.putServerFileUploadPart.mockImplementation(async (_n: string, id: string, offset: number, part: Blob) =>
|
||||
session(offset + part.size, { id }),
|
||||
);
|
||||
}
|
||||
|
||||
// Pauses at once. A loop that never gives up would never yield either, so
|
||||
// the hundredth pause fails the test instead of hanging it.
|
||||
const sleep = vi.fn(async (_ms: number, _signal?: AbortSignal) => {
|
||||
if (sleep.mock.calls.length > 100) throw new Error("runaway retry loop");
|
||||
});
|
||||
const opts = (over: Partial<Parameters<typeof sendInParts>[3]> = {}) => ({
|
||||
overwrite: false,
|
||||
sleep,
|
||||
now: () => NOW,
|
||||
...over,
|
||||
});
|
||||
const offsets = () => mocks.putServerFileUploadPart.mock.calls.map((c) => c[2]);
|
||||
const stored = () => JSON.parse(localStorage.getItem(KEY) ?? "null");
|
||||
|
||||
beforeEach(() => {
|
||||
for (const m of Object.values(mocks)) m.mockReset();
|
||||
sleep.mockClear();
|
||||
localStorage.clear();
|
||||
mocks.beginServerFileUpload.mockResolvedValue(session(0));
|
||||
mocks.commitServerFileUpload.mockResolvedValue({ op: op() });
|
||||
serverTakesParts();
|
||||
});
|
||||
|
||||
describe("sendInParts", () => {
|
||||
it("sends each part at the offset the server has reached, then commits", async () => {
|
||||
const seen: number[] = [];
|
||||
const sessions: string[] = [];
|
||||
|
||||
const got = await sendInParts("lobby", "world.zip", file(), opts({
|
||||
overwrite: true,
|
||||
onProgress: (n) => seen.push(n),
|
||||
onSession: (id) => sessions.push(id),
|
||||
}));
|
||||
|
||||
expect(got).toEqual(op());
|
||||
expect(mocks.beginServerFileUpload.mock.calls).toEqual([["lobby", "world.zip", 10]]);
|
||||
expect(offsets()).toEqual([0, 4, 8]);
|
||||
expect(mocks.putServerFileUploadPart.mock.calls.map((c) => (c[3] as Blob).size)).toEqual([4, 4, 2]);
|
||||
expect(mocks.commitServerFileUpload.mock.calls).toEqual([["lobby", "s1", true]]);
|
||||
expect(seen).toEqual([0, 4, 8, 10]);
|
||||
expect(sessions).toEqual(["s1"]);
|
||||
});
|
||||
|
||||
it("remembers the session under the server and path, with the file it holds, touched at each part", async () => {
|
||||
const f = file("0123456789", 777);
|
||||
let clock = NOW;
|
||||
let seenMidway: unknown = null;
|
||||
mocks.putServerFileUploadPart.mockImplementation(async (_n: string, id: string, offset: number, part: Blob) => {
|
||||
if (offset === 4) seenMidway = stored();
|
||||
clock += 1000;
|
||||
return session(offset + part.size, { id });
|
||||
});
|
||||
|
||||
await sendInParts("lobby", "world.zip", f, opts({ now: () => clock }));
|
||||
|
||||
expect(seenMidway).toEqual({ server: "lobby", path: "world.zip", id: "s1", size: 10, modified: 777, touched: NOW + 1000 });
|
||||
expect(stored().touched).toBe(NOW + 3000);
|
||||
});
|
||||
|
||||
it("carries on a remembered session from where the server says it stands", async () => {
|
||||
localStorage.setItem(KEY, JSON.stringify({ server: "lobby", path: "world.zip", id: "s7", size: 10, modified: 111, touched: 0 }));
|
||||
mocks.getServerFileUpload.mockResolvedValue(session(8, { id: "s7" }));
|
||||
|
||||
await sendInParts("lobby", "world.zip", file(), opts());
|
||||
|
||||
expect(mocks.getServerFileUpload.mock.calls).toEqual([["lobby", "s7"]]);
|
||||
expect(mocks.beginServerFileUpload).not.toHaveBeenCalled();
|
||||
expect(offsets()).toEqual([8]);
|
||||
expect(mocks.commitServerFileUpload.mock.calls).toEqual([["lobby", "s7", false]]);
|
||||
});
|
||||
|
||||
it("hashes the parts a remembered session holds against the file before carrying on", async () => {
|
||||
localStorage.setItem(KEY, JSON.stringify({ server: "lobby", path: "world.zip", id: "s7", size: 10, modified: 111, touched: 0 }));
|
||||
mocks.getServerFileUpload.mockResolvedValue(session(8, { id: "s7" }));
|
||||
const seen: number[] = [];
|
||||
|
||||
await sendInParts("lobby", "world.zip", file(), opts({ onProgress: (n) => seen.push(n) }));
|
||||
|
||||
// 4 and 8 as each held part checks out, 8 where the session stands, 10 sent.
|
||||
expect(seen).toEqual([4, 8, 8, 10]);
|
||||
expect(mocks.deleteServerFileUpload).not.toHaveBeenCalled();
|
||||
expect(offsets()).toEqual([8]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a part of another file of the same name, size and time", partsOf(8, "abcd4567")],
|
||||
["a later part of another file", partsOf(8, "0123x567")],
|
||||
["fewer parts than the bytes it says it holds", partsOf(4)],
|
||||
])("gives back a remembered session holding %s, and starts the file over", async (_label, parts) => {
|
||||
localStorage.setItem(KEY, JSON.stringify({ server: "lobby", path: "world.zip", id: "s7", size: 10, modified: 111, touched: 0 }));
|
||||
mocks.getServerFileUpload.mockResolvedValue(session(8, { id: "s7", parts }));
|
||||
mocks.deleteServerFileUpload.mockResolvedValue(null);
|
||||
|
||||
await sendInParts("lobby", "world.zip", file(), opts());
|
||||
|
||||
expect(mocks.deleteServerFileUpload.mock.calls).toEqual([["lobby", "s7"]]);
|
||||
expect(mocks.beginServerFileUpload.mock.calls).toEqual([["lobby", "world.zip", 10]]);
|
||||
expect(offsets()).toEqual([0, 4, 8]);
|
||||
expect(mocks.commitServerFileUpload.mock.calls).toEqual([["lobby", "s1", false]]);
|
||||
expect(stored().id).toBe("s1");
|
||||
});
|
||||
|
||||
it("stops while hashing what a remembered session holds, and keeps it for later", async () => {
|
||||
localStorage.setItem(KEY, JSON.stringify({ server: "lobby", path: "world.zip", id: "s7", size: 10, modified: 111, touched: 0 }));
|
||||
const ctrl = new AbortController();
|
||||
mocks.getServerFileUpload.mockImplementation(async () => {
|
||||
ctrl.abort();
|
||||
return session(8, { id: "s7" });
|
||||
});
|
||||
|
||||
await expect(sendInParts("lobby", "world.zip", file(), opts({ signal: ctrl.signal }))).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
|
||||
expect(offsets()).toEqual([]);
|
||||
expect(mocks.deleteServerFileUpload).not.toHaveBeenCalled();
|
||||
expect(stored().id).toBe("s7");
|
||||
});
|
||||
|
||||
it("sends a part changed on the way again, hashing none of the parts already known", async () => {
|
||||
let first = true;
|
||||
mocks.putServerFileUploadPart.mockImplementation(async (_n: string, id: string, offset: number, part: Blob) => {
|
||||
if (offset === 4 && first) {
|
||||
first = false;
|
||||
throw { status: 400, code: "digest_mismatch", message: "" };
|
||||
}
|
||||
return session(offset + part.size, { id });
|
||||
});
|
||||
mocks.getServerFileUpload.mockResolvedValue(session(4));
|
||||
const seen: number[] = [];
|
||||
|
||||
await sendInParts("lobby", "world.zip", file(), opts({ onProgress: (n) => seen.push(n) }));
|
||||
|
||||
expect(offsets()).toEqual([0, 4, 4, 8]);
|
||||
expect(sleep.mock.calls.map((c) => c[0])).toEqual([1000]);
|
||||
// No 4 from hashing the first part again: this run sent it.
|
||||
expect(seen).toEqual([0, 4, 4, 8, 10]);
|
||||
expect(mocks.commitServerFileUpload.mock.calls).toEqual([["lobby", "s1", false]]);
|
||||
});
|
||||
|
||||
it("commits at once when the server already holds the whole file", async () => {
|
||||
localStorage.setItem(KEY, JSON.stringify({ server: "lobby", path: "world.zip", id: "s7", size: 10, modified: 111, touched: 0 }));
|
||||
mocks.getServerFileUpload.mockResolvedValue(session(10, { id: "s7" }));
|
||||
|
||||
await sendInParts("lobby", "world.zip", file(), opts());
|
||||
|
||||
expect(offsets()).toEqual([]);
|
||||
expect(mocks.commitServerFileUpload.mock.calls).toEqual([["lobby", "s7", false]]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a different size", { size: 11, modified: 111 }],
|
||||
["a different modification time", { size: 10, modified: 112 }],
|
||||
])("starts over when the remembered session was for %s", async (_label, held) => {
|
||||
localStorage.setItem(KEY, JSON.stringify({ server: "lobby", path: "world.zip", id: "s7", touched: 0, ...held }));
|
||||
|
||||
await sendInParts("lobby", "world.zip", file(), opts());
|
||||
|
||||
expect(mocks.getServerFileUpload).not.toHaveBeenCalled();
|
||||
expect(mocks.beginServerFileUpload).toHaveBeenCalledTimes(1);
|
||||
expect(offsets()).toEqual([0, 4, 8]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["is gone", () => mocks.getServerFileUpload.mockRejectedValue(gone)],
|
||||
["now stands for another file", () => mocks.getServerFileUpload.mockResolvedValue(session(8, { id: "s7", size: 99 }))],
|
||||
])("begins a new session when the remembered one %s", async (_label, arrange) => {
|
||||
localStorage.setItem(KEY, JSON.stringify({ server: "lobby", path: "world.zip", id: "s7", size: 10, modified: 111, touched: 0 }));
|
||||
arrange();
|
||||
|
||||
await sendInParts("lobby", "world.zip", file(), opts());
|
||||
|
||||
expect(mocks.beginServerFileUpload).toHaveBeenCalledTimes(1);
|
||||
expect(offsets()).toEqual([0, 4, 8]);
|
||||
expect(stored().id).toBe("s1");
|
||||
expect(sleep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("gives back only the stale sessions this browser left behind when there are too many, then begins", async () => {
|
||||
const old = { server: "lobby", path: "old.zip", id: "old", size: 5, modified: 1, touched: NOW - STALE_SESSION_MS };
|
||||
const fresh = { server: "lobby", path: "fresh.zip", id: "fresh", size: 5, modified: 1, touched: NOW - STALE_SESSION_MS + 1 };
|
||||
localStorage.setItem("felis-file-upload:lobby:old.zip", JSON.stringify(old));
|
||||
localStorage.setItem("felis-file-upload:lobby:fresh.zip", JSON.stringify(fresh));
|
||||
localStorage.setItem("unrelated", JSON.stringify({ id: "x", server: "lobby", path: "x" }));
|
||||
mocks.beginServerFileUpload
|
||||
.mockRejectedValueOnce({ status: 429, code: "too_many_uploads", message: "" })
|
||||
.mockResolvedValueOnce(session(0));
|
||||
mocks.deleteServerFileUpload.mockResolvedValue(null);
|
||||
|
||||
await sendInParts("lobby", "world.zip", file(), opts());
|
||||
|
||||
expect(mocks.deleteServerFileUpload.mock.calls).toEqual([["lobby", "old"]]);
|
||||
expect(localStorage.getItem("felis-file-upload:lobby:old.zip")).toBeNull();
|
||||
expect(localStorage.getItem("felis-file-upload:lobby:fresh.zip")).not.toBeNull();
|
||||
expect(mocks.beginServerFileUpload).toHaveBeenCalledTimes(2);
|
||||
expect(offsets()).toEqual([0, 4, 8]);
|
||||
});
|
||||
|
||||
it("reports too many uploads when none of this browser's sessions could be given back", async () => {
|
||||
const refusal = { status: 429, code: "too_many_uploads", message: "" };
|
||||
const old = { server: "lobby", path: "old.zip", id: "old", size: 5, modified: 1, touched: 0 };
|
||||
localStorage.setItem("felis-file-upload:lobby:old.zip", JSON.stringify(old));
|
||||
mocks.beginServerFileUpload.mockRejectedValue(refusal);
|
||||
mocks.deleteServerFileUpload.mockRejectedValue(gone);
|
||||
|
||||
await expect(sendInParts("lobby", "world.zip", file(), opts())).rejects.toBe(refusal);
|
||||
|
||||
expect(mocks.beginServerFileUpload).toHaveBeenCalledTimes(1);
|
||||
expect(offsets()).toEqual([]);
|
||||
});
|
||||
|
||||
it("asks where the session stands after a dropped part, and resends from there", async () => {
|
||||
let first = true;
|
||||
mocks.putServerFileUploadPart.mockImplementation(async (_n: string, id: string, offset: number, part: Blob) => {
|
||||
if (offset === 4 && first) {
|
||||
first = false;
|
||||
throw netErr;
|
||||
}
|
||||
return session(offset + part.size, { id });
|
||||
});
|
||||
// Half of the dropped part had arrived.
|
||||
mocks.getServerFileUpload.mockResolvedValue(session(6));
|
||||
|
||||
await sendInParts("lobby", "world.zip", file(), opts());
|
||||
|
||||
expect(mocks.getServerFileUpload.mock.calls).toEqual([["lobby", "s1"]]);
|
||||
expect(offsets()).toEqual([0, 4, 6]);
|
||||
expect(sleep.mock.calls.map((c) => c[0])).toEqual([1000]);
|
||||
});
|
||||
|
||||
it("begins a new session when the one in use went away mid-upload", async () => {
|
||||
mocks.putServerFileUploadPart
|
||||
.mockImplementationOnce(async (_n: string, id: string, offset: number, part: Blob) => session(offset + part.size, { id }))
|
||||
.mockRejectedValueOnce(gone);
|
||||
mocks.beginServerFileUpload.mockResolvedValueOnce(session(0)).mockResolvedValueOnce(session(0, { id: "s2" }));
|
||||
|
||||
await sendInParts("lobby", "world.zip", file(), opts());
|
||||
|
||||
expect(mocks.getServerFileUpload).not.toHaveBeenCalled();
|
||||
expect(mocks.beginServerFileUpload).toHaveBeenCalledTimes(2);
|
||||
expect(offsets()).toEqual([0, 4, 0, 4, 8]);
|
||||
expect(mocks.commitServerFileUpload.mock.calls).toEqual([["lobby", "s2", false]]);
|
||||
});
|
||||
|
||||
it("stops at a refusal sending again cannot change", async () => {
|
||||
const full = { status: 507, code: "upload_staging_full", message: "" };
|
||||
mocks.putServerFileUploadPart.mockRejectedValue(full);
|
||||
|
||||
await expect(sendInParts("lobby", "world.zip", file(), opts())).rejects.toBe(full);
|
||||
|
||||
expect(offsets()).toEqual([0]);
|
||||
expect(sleep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it(`gives up after ${MAX_ATTEMPTS} dropped attempts in a row`, async () => {
|
||||
mocks.putServerFileUploadPart.mockRejectedValue(netErr);
|
||||
mocks.getServerFileUpload.mockResolvedValue(session(0));
|
||||
|
||||
await expect(sendInParts("lobby", "world.zip", file(), opts())).rejects.toBe(netErr);
|
||||
|
||||
expect(offsets()).toHaveLength(MAX_ATTEMPTS);
|
||||
});
|
||||
|
||||
it("counts only drops in a row", async () => {
|
||||
let received = 0;
|
||||
let drops = 0;
|
||||
mocks.putServerFileUploadPart.mockImplementation(async (_n: string, id: string, offset: number, part: Blob) => {
|
||||
// Each part lands only after MAX_ATTEMPTS - 1 drops.
|
||||
if (drops++ < MAX_ATTEMPTS - 1) throw netErr;
|
||||
drops = 0;
|
||||
received = offset + part.size;
|
||||
return session(received, { id });
|
||||
});
|
||||
mocks.getServerFileUpload.mockImplementation(async () => session(received));
|
||||
|
||||
await sendInParts("lobby", "world.zip", file(), opts());
|
||||
|
||||
expect(offsets()).toHaveLength(3 * MAX_ATTEMPTS);
|
||||
expect(mocks.commitServerFileUpload).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps the session remembered when stopped, even in its first part, for a later resume", async () => {
|
||||
const ctrl = new AbortController();
|
||||
mocks.putServerFileUploadPart.mockImplementation(async (_n: string, id: string, offset: number, part: Blob) => {
|
||||
if (offset === 0) {
|
||||
ctrl.abort();
|
||||
throw new DOMException("cancelled", "AbortError");
|
||||
}
|
||||
return session(offset + part.size, { id });
|
||||
});
|
||||
|
||||
await expect(sendInParts("lobby", "world.zip", file(), opts({ signal: ctrl.signal }))).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
|
||||
expect(mocks.putServerFileUploadPart.mock.calls[0][4]).toMatchObject({ signal: ctrl.signal });
|
||||
expect(stored().id).toBe("s1");
|
||||
expect(mocks.deleteServerFileUpload).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe("a commit whose answer was lost", () => {
|
||||
it("takes the running upload of this path as its answer when the world is held", async () => {
|
||||
const landing = op({ id: "op9" });
|
||||
mocks.commitServerFileUpload.mockRejectedValueOnce(netErr).mockRejectedValueOnce(held);
|
||||
mocks.listServerFileOps.mockResolvedValue({
|
||||
ops: [op({ id: "other", path: "else.zip" }), op({ id: "old", state: "failed" }), landing],
|
||||
});
|
||||
|
||||
expect(await sendInParts("lobby", "world.zip", file(), opts())).toEqual(landing);
|
||||
expect(mocks.commitServerFileUpload).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("takes the upload of this path in any state when the session is already gone", async () => {
|
||||
const landed = op({ id: "op9", state: "succeeded" });
|
||||
mocks.commitServerFileUpload.mockRejectedValueOnce(netErr).mockRejectedValueOnce(gone);
|
||||
mocks.listServerFileOps.mockResolvedValue({ ops: [op({ id: "u", op: "unzip" }), landed] });
|
||||
|
||||
expect(await sendInParts("lobby", "world.zip", file(), opts())).toEqual(landed);
|
||||
});
|
||||
|
||||
it("reports the session gone when no upload of this path is there", async () => {
|
||||
mocks.commitServerFileUpload.mockRejectedValueOnce(netErr).mockRejectedValueOnce(gone);
|
||||
mocks.listServerFileOps.mockResolvedValue({ ops: [op({ path: "else.zip" })] });
|
||||
|
||||
await expect(sendInParts("lobby", "world.zip", file(), opts())).rejects.toBe(gone);
|
||||
});
|
||||
|
||||
it("asks again while the world is held by something else", async () => {
|
||||
mocks.commitServerFileUpload
|
||||
.mockRejectedValueOnce(netErr)
|
||||
.mockRejectedValueOnce(held)
|
||||
.mockResolvedValueOnce({ op: op({ id: "op2" }) });
|
||||
mocks.listServerFileOps.mockResolvedValue({ ops: [op({ id: "done", state: "succeeded" })] });
|
||||
|
||||
expect(await sendInParts("lobby", "world.zip", file(), opts())).toEqual(op({ id: "op2" }));
|
||||
expect(mocks.commitServerFileUpload).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
|
||||
it(`gives up a commit after ${MAX_ATTEMPTS} attempts that did not get through`, async () => {
|
||||
mocks.commitServerFileUpload.mockRejectedValue(netErr);
|
||||
|
||||
await expect(sendInParts("lobby", "world.zip", file(), opts())).rejects.toBe(netErr);
|
||||
|
||||
expect(mocks.commitServerFileUpload).toHaveBeenCalledTimes(MAX_ATTEMPTS);
|
||||
});
|
||||
|
||||
it("reads a held world on the first commit as the refusal it is", async () => {
|
||||
mocks.commitServerFileUpload.mockRejectedValue(held);
|
||||
|
||||
await expect(sendInParts("lobby", "world.zip", file(), opts())).rejects.toBe(held);
|
||||
|
||||
expect(mocks.commitServerFileUpload).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.listServerFileOps).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("watchOp", () => {
|
||||
const watch = (over: Parameters<typeof watchOp>[2] = {}) => watchOp("lobby", "op1", { sleep, ...over });
|
||||
|
||||
it("reads every OP_POLL_MS, reports progress, and answers the op once it ends", async () => {
|
||||
const seen: number[] = [];
|
||||
mocks.listServerFileOps
|
||||
.mockResolvedValueOnce({ ops: [op({ done: 3, total: 10 })] })
|
||||
.mockResolvedValueOnce({ ops: [op({ id: "x", state: "succeeded" }), op({ done: 7, total: 10 })] })
|
||||
.mockResolvedValueOnce({ ops: [op({ state: "succeeded", done: 10, total: 10 })] });
|
||||
|
||||
const end = await watch({ onProgress: (o) => seen.push(o.done) });
|
||||
|
||||
expect(end).toEqual(op({ state: "succeeded", done: 10, total: 10 }));
|
||||
expect(seen).toEqual([3, 7]);
|
||||
expect(sleep.mock.calls.map((c) => c[0])).toEqual([OP_POLL_MS, OP_POLL_MS, OP_POLL_MS]);
|
||||
});
|
||||
|
||||
it("backs off after a read that did not get through, and carries on", async () => {
|
||||
mocks.listServerFileOps
|
||||
.mockRejectedValueOnce(netErr)
|
||||
.mockRejectedValueOnce(netErr)
|
||||
.mockRejectedValueOnce(netErr)
|
||||
.mockResolvedValueOnce({ ops: [op()] })
|
||||
.mockResolvedValueOnce({ ops: [op({ state: "failed" })] });
|
||||
|
||||
expect((await watch()).state).toBe("failed");
|
||||
expect(sleep.mock.calls.map((c) => c[0])).toEqual([OP_POLL_MS, 1000, 2000, 4000, OP_POLL_MS]);
|
||||
});
|
||||
|
||||
it(`gives up after ${MAX_ATTEMPTS} reads in a row did not get through`, async () => {
|
||||
mocks.listServerFileOps.mockRejectedValue(netErr);
|
||||
|
||||
await expect(watch()).rejects.toBe(netErr);
|
||||
expect(mocks.listServerFileOps).toHaveBeenCalledTimes(MAX_ATTEMPTS);
|
||||
});
|
||||
|
||||
it("stops at a read refused for good", async () => {
|
||||
const refused = { status: 403, code: "forbidden", message: "" };
|
||||
mocks.listServerFileOps.mockRejectedValue(refused);
|
||||
|
||||
await expect(watch()).rejects.toBe(refused);
|
||||
expect(mocks.listServerFileOps).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it(`reports the op lost after ${MAX_OP_MISSES} reads in a row without it`, async () => {
|
||||
mocks.listServerFileOps.mockResolvedValue({ ops: [op({ id: "other" })] });
|
||||
|
||||
await expect(watch()).rejects.toMatchObject({ code: "op_lost" });
|
||||
expect(mocks.listServerFileOps).toHaveBeenCalledTimes(MAX_OP_MISSES);
|
||||
});
|
||||
|
||||
it("counts only misses in a row", async () => {
|
||||
const without = { ops: [] };
|
||||
for (let i = 0; i < MAX_OP_MISSES - 1; i++) mocks.listServerFileOps.mockResolvedValueOnce(without);
|
||||
mocks.listServerFileOps.mockResolvedValueOnce({ ops: [op()] });
|
||||
for (let i = 0; i < MAX_OP_MISSES - 1; i++) mocks.listServerFileOps.mockResolvedValueOnce(without);
|
||||
mocks.listServerFileOps.mockResolvedValueOnce({ ops: [op({ state: "succeeded" })] });
|
||||
|
||||
expect((await watch()).state).toBe("succeeded");
|
||||
});
|
||||
});
|
||||
|
||||
describe("discardSession and forgetSession", () => {
|
||||
it("forgets the session and cancels it, waiting out a part still arriving", async () => {
|
||||
localStorage.setItem(KEY, "{}");
|
||||
mocks.deleteServerFileUpload
|
||||
.mockRejectedValueOnce({ status: 409, code: "upload_busy", message: "" })
|
||||
.mockResolvedValueOnce(null);
|
||||
|
||||
await discardSession("lobby", "world.zip", "s1", sleep);
|
||||
|
||||
expect(localStorage.getItem(KEY)).toBeNull();
|
||||
expect(mocks.deleteServerFileUpload.mock.calls).toEqual([
|
||||
["lobby", "s1"],
|
||||
["lobby", "s1"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("tries three times at most, and never past another refusal", async () => {
|
||||
mocks.deleteServerFileUpload.mockRejectedValue({ status: 409, code: "upload_busy", message: "" });
|
||||
await discardSession("lobby", "world.zip", "s1", sleep);
|
||||
expect(mocks.deleteServerFileUpload).toHaveBeenCalledTimes(3);
|
||||
|
||||
mocks.deleteServerFileUpload.mockReset();
|
||||
mocks.deleteServerFileUpload.mockRejectedValue(gone);
|
||||
await discardSession("lobby", "world.zip", "s1", sleep);
|
||||
expect(mocks.deleteServerFileUpload).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("forgetSession drops only the one path", () => {
|
||||
localStorage.setItem(KEY, "{}");
|
||||
localStorage.setItem("felis-file-upload:lobby:other.zip", "{}");
|
||||
|
||||
forgetSession("lobby", "world.zip");
|
||||
|
||||
expect(localStorage.getItem(KEY)).toBeNull();
|
||||
expect(localStorage.getItem("felis-file-upload:lobby:other.zip")).toBe("{}");
|
||||
});
|
||||
});
|
||||
@@ -1,336 +0,0 @@
|
||||
import { api, clientError } from "@/lib/api";
|
||||
import { sha256Of } from "@/lib/digest";
|
||||
import { MAX_ATTEMPTS, isTransient, retryDelay, wait } from "@/lib/contextUpload";
|
||||
import type { FileOp, FileUploadSession } from "@/lib/types";
|
||||
|
||||
// A file bigger than one request carries goes up as a session (api
|
||||
// beginServerFileUpload): parts of at most part_max_bytes, each at the byte
|
||||
// offset the session has reached, so a dropped connection resumes where the
|
||||
// server says it stands rather than starting over. The commit answers at once
|
||||
// with the op landing the file, and watchOp follows that op to its end.
|
||||
//
|
||||
// The session is also remembered in this browser, under the server and path it
|
||||
// lands at, with the file's size and modification time. Choosing the same file
|
||||
// again after a reload, a closed tab or a lost connection carries on from the
|
||||
// bytes already sent, once those have been hashed again against the file: the
|
||||
// session lists each part it holds with its SHA-256, and a session holding
|
||||
// anything else (another file of the same name, size and time) is given back
|
||||
// and the file starts over. A remembered session no page has touched for a
|
||||
// while is what a refusal for too many sessions gives back first.
|
||||
//
|
||||
// Every part goes with its SHA-256, and one the server hashes differently
|
||||
// (digest_mismatch: changed on the way) is sent again.
|
||||
|
||||
type Sleep = (ms: number, signal?: AbortSignal) => Promise<void>;
|
||||
|
||||
/** How often a running op is read. */
|
||||
export const OP_POLL_MS = 2000;
|
||||
/** Reads of the ops list that may miss an op before it counts as lost. */
|
||||
export const MAX_OP_MISSES = 5;
|
||||
/** How long a remembered session must sit untouched before it is given back
|
||||
* to make room for a new one; one another tab is still sending is younger. */
|
||||
export const STALE_SESSION_MS = 5 * 60 * 1000;
|
||||
|
||||
const KEY_PREFIX = "felis-file-upload:";
|
||||
|
||||
interface Remembered {
|
||||
server: string;
|
||||
path: string;
|
||||
id: string;
|
||||
size: number;
|
||||
modified: number;
|
||||
touched: number;
|
||||
}
|
||||
|
||||
const code = (e: unknown) => (e as { code?: unknown } | null)?.code;
|
||||
|
||||
// Browser storage can be missing or refuse (a private window, blocked site
|
||||
// data); a session is then simply not remembered.
|
||||
function storage(): Storage | null {
|
||||
try {
|
||||
return window.localStorage;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const keyOf = (server: string, path: string) => `${KEY_PREFIX}${server}:${path}`;
|
||||
|
||||
function remember(r: Remembered) {
|
||||
try {
|
||||
storage()?.setItem(keyOf(r.server, r.path), JSON.stringify(r));
|
||||
} catch {
|
||||
/* not remembered: a reload starts this file over */
|
||||
}
|
||||
}
|
||||
|
||||
function recalled(server: string, path: string, file: File): Remembered | null {
|
||||
try {
|
||||
const raw = storage()?.getItem(keyOf(server, path));
|
||||
const r = raw ? (JSON.parse(raw) as Remembered) : null;
|
||||
return r && r.id && r.size === file.size && r.modified === file.lastModified ? r : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function rememberedAll(): Remembered[] {
|
||||
const s = storage();
|
||||
const out: Remembered[] = [];
|
||||
try {
|
||||
for (let i = 0; s && i < s.length; i++) {
|
||||
const k = s.key(i);
|
||||
if (!k?.startsWith(KEY_PREFIX)) continue;
|
||||
const r = JSON.parse(s.getItem(k) ?? "null") as Remembered | null;
|
||||
if (r?.id && r.server && typeof r.path === "string") out.push(r);
|
||||
}
|
||||
} catch {
|
||||
/* whatever was read so far */
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** forgetSession stops remembering the session for path on server, once its
|
||||
* file has landed. */
|
||||
export function forgetSession(server: string, path: string) {
|
||||
try {
|
||||
storage()?.removeItem(keyOf(server, path));
|
||||
} catch {
|
||||
/* nothing to forget */
|
||||
}
|
||||
}
|
||||
|
||||
/** discardSession cancels a session and forgets it. A part still arriving holds
|
||||
* it briefly (upload_busy), so that is waited out a few times; whatever still
|
||||
* fails is left for the server to drop after 6 idle hours. */
|
||||
export async function discardSession(server: string, path: string, id: string, sleep: Sleep = wait) {
|
||||
forgetSession(server, path);
|
||||
for (let n = 1; n <= 3; n++) {
|
||||
try {
|
||||
await api.deleteServerFileUpload(server, id);
|
||||
return;
|
||||
} catch (e) {
|
||||
if (code(e) !== "upload_busy") return;
|
||||
await sleep(retryDelay(n));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dropStale cancels the remembered sessions nothing has sent a part to lately
|
||||
// and reports whether any of them was still held.
|
||||
async function dropStale(now: number): Promise<boolean> {
|
||||
let freed = false;
|
||||
for (const r of rememberedAll()) {
|
||||
if (now - r.touched < STALE_SESSION_MS) continue;
|
||||
forgetSession(r.server, r.path);
|
||||
try {
|
||||
await api.deleteServerFileUpload(r.server, r.id);
|
||||
freed = true;
|
||||
} catch {
|
||||
/* gone already, or someone else's */
|
||||
}
|
||||
}
|
||||
return freed;
|
||||
}
|
||||
|
||||
export interface SendOptions {
|
||||
overwrite: boolean;
|
||||
/** Bytes of the file the server holds so far, plus the part in flight. */
|
||||
onProgress?: (sent: number) => void;
|
||||
/** The session the file goes up in, once there is one. */
|
||||
onSession?: (id: string) => void;
|
||||
/** Stops sending; the session stays, remembered, for a later resume. */
|
||||
signal?: AbortSignal;
|
||||
/** Test seam for the pause between attempts. */
|
||||
sleep?: Sleep;
|
||||
/** Test seam for the clock the remembered sessions are aged by. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
/** sendInParts sends file to path on server as an upload session, commits it,
|
||||
* and answers the op landing it. */
|
||||
export async function sendInParts(server: string, path: string, file: File, opts: SendOptions): Promise<FileOp> {
|
||||
const { signal, onProgress } = opts;
|
||||
const sleep = opts.sleep ?? wait;
|
||||
const now = opts.now ?? Date.now;
|
||||
const size = file.size;
|
||||
|
||||
let id: string | null = recalled(server, path, file)?.id ?? null;
|
||||
// offset is where the next part starts; null means "ask the server first".
|
||||
let offset: number | null = null;
|
||||
let partMax = 0;
|
||||
let failures = 0;
|
||||
// known is how many bytes at the head of the session this run has seen to be
|
||||
// the file's own: parts it sent, or held parts it hashed again. A session
|
||||
// begun during the run holds only parts the run sent, so a new one needs no
|
||||
// reset.
|
||||
let known = 0;
|
||||
for (;;) {
|
||||
try {
|
||||
if (offset === null) {
|
||||
const at = await standing(server, path, file, id, now);
|
||||
id = at.id;
|
||||
opts.onSession?.(id);
|
||||
if (!(await holdsTheFile(file, at, known, onProgress, signal))) {
|
||||
await discardSession(server, path, at.id, sleep);
|
||||
id = null;
|
||||
continue;
|
||||
}
|
||||
known = at.received;
|
||||
partMax = at.part_max_bytes;
|
||||
offset = at.received;
|
||||
onProgress?.(offset);
|
||||
}
|
||||
if (offset >= size) break;
|
||||
const start = offset;
|
||||
const part = file.slice(start, Math.min(start + partMax, size));
|
||||
const at = await api.putServerFileUploadPart(server, id!, start, part, {
|
||||
signal,
|
||||
onProgress: (sent) => onProgress?.(start + sent),
|
||||
});
|
||||
offset = at.received;
|
||||
known = at.received;
|
||||
failures = 0;
|
||||
remember({ server, path, id: id!, size, modified: file.lastModified, touched: now() });
|
||||
onProgress?.(offset);
|
||||
} catch (e) {
|
||||
// The session went away under the upload (felis-api restarted, or it sat
|
||||
// idle too long): the next pass begins a new one. A part changed on the
|
||||
// way was not kept, so the next pass sends it again (isTransient).
|
||||
if (code(e) === "upload_not_found") {
|
||||
id = null;
|
||||
} else if (!isTransient(e)) {
|
||||
throw e;
|
||||
}
|
||||
if (++failures >= MAX_ATTEMPTS) throw e;
|
||||
await sleep(retryDelay(failures), signal);
|
||||
offset = null;
|
||||
}
|
||||
}
|
||||
return commit(server, path, id!, opts.overwrite, sleep, signal);
|
||||
}
|
||||
|
||||
// standing answers the session to carry on with: id's when it still stands for
|
||||
// this file, else a new one. Four sessions per account is the server's bound;
|
||||
// when that refuses, the sessions this browser left behind are given back
|
||||
// first.
|
||||
async function standing(
|
||||
server: string,
|
||||
path: string,
|
||||
file: File,
|
||||
id: string | null,
|
||||
now: () => number,
|
||||
): Promise<FileUploadSession> {
|
||||
if (id) {
|
||||
try {
|
||||
const at = await api.getServerFileUpload(server, id);
|
||||
if (at.path === path && at.size === file.size) return at;
|
||||
} catch (e) {
|
||||
if (code(e) !== "upload_not_found") throw e;
|
||||
}
|
||||
}
|
||||
let at: FileUploadSession;
|
||||
try {
|
||||
at = await api.beginServerFileUpload(server, path, file.size);
|
||||
} catch (e) {
|
||||
if (code(e) !== "too_many_uploads" || !(await dropStale(now()))) throw e;
|
||||
at = await api.beginServerFileUpload(server, path, file.size);
|
||||
}
|
||||
remember({ server, path, id: at.id, size: file.size, modified: file.lastModified, touched: now() });
|
||||
return at;
|
||||
}
|
||||
|
||||
// holdsTheFile hashes the parts session at holds past the first known bytes
|
||||
// against the same ranges of file, and answers whether all of them match.
|
||||
// onProgress walks up through the parts as they check out.
|
||||
async function holdsTheFile(
|
||||
file: File,
|
||||
at: FileUploadSession,
|
||||
known: number,
|
||||
onProgress?: (sent: number) => void,
|
||||
signal?: AbortSignal,
|
||||
): Promise<boolean> {
|
||||
let start = 0;
|
||||
for (const part of at.parts) {
|
||||
const end = start + part.size;
|
||||
if (end > known) {
|
||||
if (signal?.aborted) throw new DOMException("The upload was cancelled", "AbortError");
|
||||
if ((await sha256Of(file.slice(start, end))).hex !== part.sha256) return false;
|
||||
onProgress?.(end);
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
return start === at.received;
|
||||
}
|
||||
|
||||
// commit lands the session. A commit whose answer was lost may still have
|
||||
// started the Job, so asking again is read in that light: the world held
|
||||
// (maintenance_in_progress) by an upload of this path running now, or the
|
||||
// session gone because that Job reported the file landed, means the first
|
||||
// commit went through, and its op is the answer.
|
||||
async function commit(
|
||||
server: string,
|
||||
path: string,
|
||||
id: string,
|
||||
overwrite: boolean,
|
||||
sleep: Sleep,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FileOp> {
|
||||
let lost = false;
|
||||
for (let failures = 1; ; failures++) {
|
||||
try {
|
||||
return (await api.commitServerFileUpload(server, id, overwrite)).op;
|
||||
} catch (e) {
|
||||
const c = code(e);
|
||||
if (lost && (c === "maintenance_in_progress" || c === "upload_not_found")) {
|
||||
const { ops } = await api.listServerFileOps(server);
|
||||
const op = ops.find(
|
||||
(o) => o.op === "upload" && o.path === path && (c === "upload_not_found" || o.state === "running"),
|
||||
);
|
||||
if (op) return op;
|
||||
if (c === "upload_not_found") throw e;
|
||||
} else if (isTransient(e)) {
|
||||
lost = true;
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
if (failures >= MAX_ATTEMPTS) throw e;
|
||||
await sleep(retryDelay(failures), signal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface WatchOptions {
|
||||
signal?: AbortSignal;
|
||||
sleep?: Sleep;
|
||||
/** Each read of the op while it runs. */
|
||||
onProgress?: (op: FileOp) => void;
|
||||
}
|
||||
|
||||
/** watchOp reads server's ops until the op id has ended, and answers it. A read
|
||||
* that did not get through is tried again, backing off; an op missing from
|
||||
* MAX_OP_MISSES reads in a row is reported as op_lost. */
|
||||
export async function watchOp(server: string, id: string, opts: WatchOptions = {}): Promise<FileOp> {
|
||||
const sleep = opts.sleep ?? wait;
|
||||
let failures = 0;
|
||||
let misses = 0;
|
||||
for (;;) {
|
||||
await sleep(failures > 0 ? retryDelay(failures) : OP_POLL_MS, opts.signal);
|
||||
let ops: FileOp[];
|
||||
try {
|
||||
ops = (await api.listServerFileOps(server)).ops;
|
||||
failures = 0;
|
||||
} catch (e) {
|
||||
if (!isTransient(e) || ++failures >= MAX_ATTEMPTS) throw e;
|
||||
continue;
|
||||
}
|
||||
const op = ops.find((o) => o.id === id);
|
||||
if (!op) {
|
||||
if (++misses >= MAX_OP_MISSES) throw clientError("op_lost");
|
||||
continue;
|
||||
}
|
||||
misses = 0;
|
||||
if (op.state !== "running") return op;
|
||||
opts.onProgress?.(op);
|
||||
}
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
// @vitest-environment jsdom
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { act, renderHook } from "@testing-library/react";
|
||||
import type { FileOp } from "@/lib/types";
|
||||
import { OP_POLL_MS } from "./sessionUpload";
|
||||
import { useFileOps } from "./useFileOps";
|
||||
|
||||
const mocks = vi.hoisted(() => ({ listServerFileOps: vi.fn() }));
|
||||
|
||||
vi.mock("@/lib/api", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("@/lib/api")>();
|
||||
return { ...actual, api: { ...actual.api, ...mocks } };
|
||||
});
|
||||
|
||||
function op(id: string, over: Partial<FileOp> = {}): FileOp {
|
||||
return { id, op: "unzip", path: `${id}.zip`, state: "running", started_at: "2026-09-28T00:00:00Z", done: 0, total: 0, ...over };
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (v: T) => void;
|
||||
const promise = new Promise<T>((res) => (resolve = res));
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
const ids = (ops: readonly FileOp[]) => ops.map((o) => `${o.id}:${o.state}`);
|
||||
const tick = () => act(() => vi.advanceTimersByTimeAsync(OP_POLL_MS));
|
||||
|
||||
beforeEach(() => {
|
||||
mocks.listServerFileOps.mockReset();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
describe("useFileOps", () => {
|
||||
it("reads nothing until enabled, then once, and polls no further while nothing runs", async () => {
|
||||
mocks.listServerFileOps.mockResolvedValue({ ops: [op("a", { state: "succeeded" })] });
|
||||
const { result, rerender } = renderHook(({ on }) => useFileOps("lobby", on, () => {}), { initialProps: { on: false } });
|
||||
await tick();
|
||||
expect(mocks.listServerFileOps).not.toHaveBeenCalled();
|
||||
|
||||
rerender({ on: true });
|
||||
await act(async () => {});
|
||||
expect(mocks.listServerFileOps.mock.calls).toEqual([["lobby"]]);
|
||||
expect(ids(result.current.ops)).toEqual(["a:succeeded"]);
|
||||
expect(result.current.running).toBe(false);
|
||||
|
||||
await tick();
|
||||
await tick();
|
||||
expect(mocks.listServerFileOps).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("polls while one runs, and tells of each op it saw running once that op ends", async () => {
|
||||
const ended = vi.fn();
|
||||
mocks.listServerFileOps
|
||||
.mockResolvedValueOnce({ ops: [op("a", { done: 1, total: 4 }), op("old", { state: "failed" })] })
|
||||
.mockResolvedValueOnce({ ops: [op("a", { done: 3, total: 4 }), op("old", { state: "failed" })] })
|
||||
.mockResolvedValue({ ops: [op("a", { state: "succeeded" }), op("old", { state: "failed" })] });
|
||||
const { result } = renderHook(() => useFileOps("lobby", true, ended));
|
||||
await act(async () => {});
|
||||
expect(result.current.running).toBe(true);
|
||||
|
||||
await tick();
|
||||
expect(result.current.ops[0].done).toBe(3);
|
||||
expect(ended).not.toHaveBeenCalled();
|
||||
|
||||
await tick();
|
||||
expect(result.current.running).toBe(false);
|
||||
// Only the op seen running here: "old" had ended before the page looked.
|
||||
expect(ended.mock.calls.map(([o]) => o.id)).toEqual(["a"]);
|
||||
|
||||
await tick();
|
||||
await tick();
|
||||
expect(mocks.listServerFileOps).toHaveBeenCalledTimes(3);
|
||||
expect(ended).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("shows an op this page started at once, over a read already on its way", async () => {
|
||||
const slow = deferred<{ ops: FileOp[] }>();
|
||||
mocks.listServerFileOps.mockReturnValueOnce(slow.promise);
|
||||
const { result } = renderHook(() => useFileOps("lobby", true, () => {}));
|
||||
|
||||
act(() => result.current.started(op("new")));
|
||||
expect(ids(result.current.ops)).toEqual(["new:running"]);
|
||||
await act(async () => slow.resolve({ ops: [] }));
|
||||
|
||||
expect(ids(result.current.ops)).toEqual(["new:running"]);
|
||||
expect(result.current.running).toBe(true);
|
||||
});
|
||||
|
||||
it("tells of the end of an op this page started even when no read saw it running", async () => {
|
||||
const ended = vi.fn();
|
||||
mocks.listServerFileOps.mockResolvedValueOnce({ ops: [] }).mockResolvedValue({ ops: [op("new", { state: "failed" })] });
|
||||
const { result } = renderHook(() => useFileOps("lobby", true, ended));
|
||||
await act(async () => {});
|
||||
|
||||
act(() => result.current.started(op("new")));
|
||||
await tick();
|
||||
|
||||
expect(ended.mock.calls.map(([o]) => `${o.id}:${o.state}`)).toEqual(["new:failed"]);
|
||||
});
|
||||
|
||||
it("starts no read over one still on its way", async () => {
|
||||
const slow = deferred<{ ops: FileOp[] }>();
|
||||
mocks.listServerFileOps.mockResolvedValueOnce({ ops: [op("a")] }).mockReturnValueOnce(slow.promise);
|
||||
renderHook(() => useFileOps("lobby", true, () => {}));
|
||||
await act(async () => {});
|
||||
|
||||
await tick();
|
||||
await tick();
|
||||
await tick();
|
||||
|
||||
expect(mocks.listServerFileOps).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("keeps an ignored op out from then on, and never tells of its end", async () => {
|
||||
const ended = vi.fn();
|
||||
mocks.listServerFileOps
|
||||
.mockResolvedValueOnce({ ops: [op("mine", { op: "upload" }), op("theirs")] })
|
||||
.mockResolvedValue({ ops: [op("mine", { op: "upload", state: "succeeded" }), op("theirs", { state: "succeeded" })] });
|
||||
const { result } = renderHook(() => useFileOps("lobby", true, ended));
|
||||
await act(async () => {});
|
||||
|
||||
act(() => result.current.ignore("mine"));
|
||||
expect(ids(result.current.ops)).toEqual(["theirs:running"]);
|
||||
await tick();
|
||||
|
||||
expect(ids(result.current.ops)).toEqual(["theirs:succeeded"]);
|
||||
expect(ended.mock.calls.map(([o]) => o.id)).toEqual(["theirs"]);
|
||||
});
|
||||
|
||||
it("an ignored op that runs alone holds nothing", async () => {
|
||||
mocks.listServerFileOps.mockResolvedValue({ ops: [op("mine", { op: "upload" })] });
|
||||
const { result } = renderHook(() => useFileOps("lobby", true, () => {}));
|
||||
await act(async () => {});
|
||||
expect(result.current.running).toBe(true);
|
||||
|
||||
act(() => result.current.ignore("mine"));
|
||||
|
||||
expect(result.current.running).toBe(false);
|
||||
});
|
||||
|
||||
it("hides a dismissed op", async () => {
|
||||
mocks.listServerFileOps.mockResolvedValue({ ops: [op("a", { state: "failed" }), op("b", { state: "succeeded" })] });
|
||||
const { result } = renderHook(() => useFileOps("lobby", true, () => {}));
|
||||
await act(async () => {});
|
||||
|
||||
act(() => result.current.dismiss("a"));
|
||||
|
||||
expect(ids(result.current.ops)).toEqual(["b:succeeded"]);
|
||||
});
|
||||
|
||||
it("says why a read failed, and clears it once one gets through", async () => {
|
||||
const refused = { status: 500, code: "internal", message: "" };
|
||||
mocks.listServerFileOps.mockRejectedValueOnce(refused).mockResolvedValueOnce({ ops: [] });
|
||||
const { result } = renderHook(() => useFileOps("lobby", true, () => {}));
|
||||
await act(async () => {});
|
||||
expect(result.current.error).toBe(refused);
|
||||
|
||||
await act(async () => result.current.refresh());
|
||||
|
||||
expect(result.current.error).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -1,72 +0,0 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { api } from "@/lib/api";
|
||||
import { usePolling } from "@/lib/hooks";
|
||||
import type { FileOp } from "@/lib/types";
|
||||
import { OP_POLL_MS } from "./sessionUpload";
|
||||
|
||||
// useFileOps follows a server's background ops (an extraction, or the landing of
|
||||
// a file sent in parts): read once when `enabled` turns on, then again every
|
||||
// OP_POLL_MS while one runs. They carry on with the page closed, so a visit
|
||||
// after one began still shows it and how it ended. onEnded runs once for each op
|
||||
// seen running here that has since ended.
|
||||
export function useFileOps(server: string, enabled: boolean, onEnded: (op: FileOp) => void) {
|
||||
const [ops, setOps] = useState<FileOp[]>([]);
|
||||
const [error, setError] = useState<unknown>(null);
|
||||
const [dismissed, setDismissed] = useState<ReadonlySet<string>>(new Set());
|
||||
const seenRunning = useRef(new Set<string>());
|
||||
const ended = useRef(onEnded);
|
||||
ended.current = onEnded;
|
||||
// Only the newest read lands, and a read is not started over one in flight:
|
||||
// a slow answer must not undo a newer one, nor pile up behind the interval.
|
||||
const seq = useRef(0);
|
||||
const inFlight = useRef(false);
|
||||
// Ops the upload queue follows itself (the landing of a file it sent in
|
||||
// parts): its own row shows them, so they are left out here.
|
||||
const ignored = useRef(new Set<string>());
|
||||
|
||||
const read = useCallback(async () => {
|
||||
if (inFlight.current) return;
|
||||
inFlight.current = true;
|
||||
const ticket = ++seq.current;
|
||||
try {
|
||||
const r = await api.listServerFileOps(server);
|
||||
if (ticket !== seq.current) return;
|
||||
const all = (r.ops ?? []).filter((op) => !ignored.current.has(op.id));
|
||||
setError(null);
|
||||
setOps(all);
|
||||
for (const op of all) {
|
||||
if (op.state === "running") seenRunning.current.add(op.id);
|
||||
else if (seenRunning.current.delete(op.id)) ended.current(op);
|
||||
}
|
||||
} catch (e) {
|
||||
if (ticket === seq.current) setError(e);
|
||||
} finally {
|
||||
inFlight.current = false;
|
||||
}
|
||||
}, [server]);
|
||||
|
||||
useEffect(() => {
|
||||
if (enabled) void read();
|
||||
}, [enabled, read]);
|
||||
|
||||
const running = ops.some((op) => op.state === "running");
|
||||
const poll = useCallback(() => void read(), [read]);
|
||||
usePolling(poll, enabled && running ? OP_POLL_MS : null);
|
||||
|
||||
/** started shows an op this page just began at once, ahead of the next read,
|
||||
* and watches it from there. */
|
||||
const started = useCallback((op: FileOp) => {
|
||||
if (op.state === "running") seenRunning.current.add(op.id);
|
||||
seq.current++; // a read already in flight predates it
|
||||
setOps((all) => [op, ...all.filter((o) => o.id !== op.id)]);
|
||||
}, []);
|
||||
|
||||
const dismiss = useCallback((id: string) => setDismissed((s) => new Set(s).add(id)), []);
|
||||
/** ignore leaves the op id out from now on. */
|
||||
const ignore = useCallback((id: string) => {
|
||||
ignored.current.add(id);
|
||||
setOps((all) => all.filter((o) => o.id !== id));
|
||||
}, []);
|
||||
|
||||
return { ops: ops.filter((op) => !dismissed.has(op.id)), running, error, refresh: poll, started, dismiss, ignore };
|
||||
}
|
||||
@@ -2,15 +2,12 @@ import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import i18next from "i18next";
|
||||
import { api, humanizeError } from "@/lib/api";
|
||||
import { formatBytes } from "@/lib/format";
|
||||
import type { FileOpError, ServerFileEntry } from "@/lib/types";
|
||||
import { joinPath, nameTooLong } from "./names";
|
||||
import { opErrorText } from "./opText";
|
||||
import { discardSession, forgetSession, sendInParts, watchOp } from "./sessionUpload";
|
||||
import type { ServerFileEntry } from "@/lib/types";
|
||||
import { joinPath } from "./names";
|
||||
|
||||
/** The most one upload request carries, mirrored from fileedit.MaxUploadBytes
|
||||
* (413 above it). A larger file goes up in parts instead (sessionUpload.ts),
|
||||
* with no ceiling but the room on the world volume. */
|
||||
export const ONE_REQUEST_BYTES = 64 * 1024 * 1024;
|
||||
/** The upload ceiling, mirrored from fileedit.MaxUploadBytes (server truth, 413
|
||||
* above it). Checked here so a file too large is refused before it is sent. */
|
||||
export const MAX_UPLOAD_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
/** queued waits its turn; exists found a file already at its path and waits for
|
||||
* replace or skip; failed stays until it is retried or dismissed. */
|
||||
@@ -25,50 +22,21 @@ export interface UploadItem {
|
||||
sent: number;
|
||||
overwrite: boolean;
|
||||
error: string | null;
|
||||
/** false for a refusal sending again cannot change (a folder there). */
|
||||
/** false for a refusal sending again cannot change (too large, a folder there). */
|
||||
retryable: boolean;
|
||||
/** How far the Job landing a file sent in parts has got, once it reports. */
|
||||
landing: { done: number; total: number } | null;
|
||||
}
|
||||
|
||||
/** An op that ended failed, thrown so the queue settles it like any refusal. */
|
||||
interface OpFailure {
|
||||
status: 0;
|
||||
code: string;
|
||||
message: string;
|
||||
opError: FileOpError;
|
||||
}
|
||||
|
||||
interface Options {
|
||||
/** Runs after each upload that landed, with the folder it landed in. */
|
||||
onLanded: (dir: string) => void;
|
||||
/** Runs with the id of each op a file sent in parts is landed by. */
|
||||
onOp?: (id: string) => void;
|
||||
/** Holds the queue: nothing new starts while it is true. */
|
||||
hold?: boolean;
|
||||
/** Bytes free on the world volume, from the latest listing. */
|
||||
free?: number | null;
|
||||
}
|
||||
|
||||
// useUploads runs a queue of uploads into a server's world volume, one at a time:
|
||||
// each is a file Job that holds the world lock, so a second one sent alongside
|
||||
// would only be refused with 409 maintenance_in_progress.
|
||||
export function useUploads(server: string, { onLanded, onOp, hold = false, free = null }: Options) {
|
||||
// would only be refused with 409 maintenance_in_progress. onLanded runs after
|
||||
// each upload that landed, with the folder it landed in.
|
||||
export function useUploads(server: string, onLanded: (dir: string) => void) {
|
||||
const [items, setItems] = useState<UploadItem[]>([]);
|
||||
const current = useRef(items);
|
||||
current.current = items;
|
||||
const nextId = useRef(1);
|
||||
const running = useRef<number | null>(null);
|
||||
const abort = useRef<AbortController | null>(null);
|
||||
// An upload cancelled on purpose gives its session back; one stopped because
|
||||
// the page went away keeps it for a resume.
|
||||
const cancelled = useRef<number | null>(null);
|
||||
// The session each file sent in parts went up in, by item.
|
||||
const sessions = useRef(new Map<number, { path: string; id: string }>());
|
||||
const hooks = useRef({ onLanded, onOp });
|
||||
hooks.current = { onLanded, onOp };
|
||||
const room = useRef(free);
|
||||
room.current = free;
|
||||
const landed = useRef(onLanded);
|
||||
landed.current = onLanded;
|
||||
|
||||
const patch = useCallback((id: number, change: Partial<UploadItem>) => {
|
||||
setItems((all) => all.map((it) => (it.id === id ? { ...it, ...change } : it)));
|
||||
@@ -76,175 +44,81 @@ export function useUploads(server: string, { onLanded, onOp, hold = false, free
|
||||
const drop = useCallback((id: number) => {
|
||||
setItems((all) => all.filter((it) => it.id !== id));
|
||||
}, []);
|
||||
// discard gives back the session an item holds, if any.
|
||||
const discard = useCallback(
|
||||
(id: number) => {
|
||||
const s = sessions.current.get(id);
|
||||
sessions.current.delete(id);
|
||||
if (s) void discardSession(server, s.path, s.id);
|
||||
},
|
||||
[server],
|
||||
);
|
||||
|
||||
// Leaving the page stops the upload in flight; the queued ones were never sent.
|
||||
useEffect(() => () => abort.current?.abort(), []);
|
||||
|
||||
// sendLarge sends a file in parts and follows the op landing it to its end.
|
||||
const sendLarge = useCallback(
|
||||
async (it: UploadItem, path: string, signal: AbortSignal) => {
|
||||
const op = await sendInParts(server, path, it.file, {
|
||||
overwrite: it.overwrite,
|
||||
signal,
|
||||
onProgress: (sent) => patch(it.id, { sent }),
|
||||
onSession: (id) => sessions.current.set(it.id, { path, id }),
|
||||
});
|
||||
hooks.current.onOp?.(op.id);
|
||||
patch(it.id, { sent: it.file.size });
|
||||
const end =
|
||||
op.state === "running"
|
||||
? await watchOp(server, op.id, {
|
||||
signal,
|
||||
onProgress: (o) => patch(it.id, { landing: { done: o.done, total: o.total } }),
|
||||
})
|
||||
: op;
|
||||
if (end.state === "failed") {
|
||||
const opError = end.error ?? { code: "job_failed", message: "" };
|
||||
const failure: OpFailure = { status: 0, code: opError.code, message: opError.message, opError };
|
||||
throw failure;
|
||||
}
|
||||
sessions.current.delete(it.id);
|
||||
forgetSession(server, path);
|
||||
},
|
||||
[server, patch],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (hold || running.current !== null) return;
|
||||
if (running.current !== null) return;
|
||||
const next = items.find((it) => it.state === "queued");
|
||||
if (!next) return;
|
||||
running.current = next.id;
|
||||
const ctrl = new AbortController();
|
||||
abort.current = ctrl;
|
||||
patch(next.id, { state: "uploading", sent: 0, error: null, landing: null });
|
||||
patch(next.id, { state: "uploading", sent: 0, error: null });
|
||||
const settle = () => {
|
||||
running.current = null;
|
||||
abort.current = null;
|
||||
};
|
||||
const path = joinPath(next.dir, next.file.name);
|
||||
const sending =
|
||||
next.file.size > ONE_REQUEST_BYTES
|
||||
? sendLarge(next, path, ctrl.signal)
|
||||
: api.uploadServerFile(server, path, next.file, next.overwrite, {
|
||||
api
|
||||
.uploadServerFile(server, joinPath(next.dir, next.file.name), next.file, next.overwrite, {
|
||||
signal: ctrl.signal,
|
||||
onProgress: (sent) => patch(next.id, { sent }),
|
||||
});
|
||||
sending.then(
|
||||
})
|
||||
.then(
|
||||
() => {
|
||||
settle();
|
||||
patch(next.id, { state: "done", sent: next.file.size });
|
||||
hooks.current.onLanded(next.dir);
|
||||
landed.current(next.dir);
|
||||
},
|
||||
(e: unknown) => {
|
||||
settle();
|
||||
if (e instanceof DOMException && e.name === "AbortError") {
|
||||
if (cancelled.current === next.id) discard(next.id);
|
||||
cancelled.current = null;
|
||||
drop(next.id);
|
||||
} else if ((e as { code?: string }).code === "file_exists") {
|
||||
// Someone put a file there since the listing: ask, as for one listed.
|
||||
// A file sent in parts keeps its session, so replacing it lands the
|
||||
// bytes already sent.
|
||||
patch(next.id, { state: "exists", sent: 0, landing: null });
|
||||
patch(next.id, { state: "exists", sent: 0 });
|
||||
} else {
|
||||
const opError = (e as Partial<OpFailure>).opError;
|
||||
patch(next.id, {
|
||||
state: "failed",
|
||||
sent: 0,
|
||||
landing: null,
|
||||
error: opError ? opErrorText("upload", opError) : humanizeError(e),
|
||||
retryable: true,
|
||||
});
|
||||
patch(next.id, { state: "failed", sent: 0, error: humanizeError(e), retryable: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
}, [items, hold, server, patch, drop, discard, sendLarge]);
|
||||
}, [items, server, patch, drop]);
|
||||
|
||||
/** add queues files for dir. `entries` is dir's listing: a file of the same
|
||||
* name there waits for replace or skip instead of being sent to be refused.
|
||||
* Files the world volume has no room for are refused before any is sent,
|
||||
* counting the ones ahead of them in the same batch. */
|
||||
* name there waits for replace or skip instead of being sent to be refused. */
|
||||
const add = useCallback((files: readonly File[], dir: string, entries: readonly ServerFileEntry[] | null) => {
|
||||
const t = i18next.getFixedT(null, "files");
|
||||
let budget = room.current;
|
||||
const added = files.map((file): UploadItem => {
|
||||
const base = {
|
||||
id: nextId.current++,
|
||||
file,
|
||||
dir,
|
||||
sent: 0,
|
||||
overwrite: false,
|
||||
error: null,
|
||||
retryable: false,
|
||||
landing: null,
|
||||
};
|
||||
// The server would refuse it only after the bytes were sent.
|
||||
if (nameTooLong(file.name)) {
|
||||
return { ...base, state: "failed", error: t("upload_name_too_long") };
|
||||
}
|
||||
const base = { id: nextId.current++, file, dir, sent: 0, overwrite: false, error: null, retryable: false };
|
||||
const there = entries?.find((e) => e.name === file.name);
|
||||
if (file.size > MAX_UPLOAD_BYTES) {
|
||||
return { ...base, state: "failed", error: t("upload_too_large", { limit: formatBytes(MAX_UPLOAD_BYTES) }) };
|
||||
}
|
||||
if (there?.is_dir) {
|
||||
return { ...base, state: "failed", error: t("upload_folder_there") };
|
||||
}
|
||||
if (budget !== null && file.size > budget) {
|
||||
return { ...base, state: "failed", error: noRoom(file, budget), retryable: true };
|
||||
}
|
||||
if (budget !== null) budget -= file.size;
|
||||
return { ...base, state: there ? "exists" : "queued" };
|
||||
});
|
||||
setItems((all) => [...all, ...added]);
|
||||
}, []);
|
||||
|
||||
// requeue sends the items picked again once the volume has room for each by
|
||||
// the latest listing; one it has none for stays failed and says so again.
|
||||
const requeue = useCallback((pick: (it: UploadItem) => boolean, change: Partial<UploadItem>) => {
|
||||
const free = room.current;
|
||||
setItems((all) =>
|
||||
all.map((it) => {
|
||||
if (!pick(it)) return it;
|
||||
if (free !== null && it.file.size > free) {
|
||||
return { ...it, state: "failed", error: noRoom(it.file, free), retryable: true };
|
||||
}
|
||||
return { ...it, ...change, state: "queued", error: null };
|
||||
}),
|
||||
);
|
||||
}, []);
|
||||
|
||||
const replace = useCallback((id: number) => requeue((it) => it.id === id, { overwrite: true }), [requeue]);
|
||||
const retry = useCallback((id: number) => requeue((it) => it.id === id, {}), [requeue]);
|
||||
const replace = useCallback((id: number) => patch(id, { state: "queued", overwrite: true }), [patch]);
|
||||
const retry = useCallback((id: number) => patch(id, { state: "queued", error: null }), [patch]);
|
||||
/** remove cancels an upload in flight, or takes any other one off the list. */
|
||||
const remove = useCallback(
|
||||
(id: number) => {
|
||||
if (running.current === id) {
|
||||
cancelled.current = id;
|
||||
abort.current?.abort();
|
||||
} else {
|
||||
discard(id);
|
||||
drop(id);
|
||||
}
|
||||
if (running.current === id) abort.current?.abort();
|
||||
else drop(id);
|
||||
},
|
||||
[drop, discard],
|
||||
[drop],
|
||||
);
|
||||
const replaceAll = useCallback(() => requeue((it) => it.state === "exists", { overwrite: true }), [requeue]);
|
||||
const skipAll = useCallback(() => {
|
||||
for (const it of current.current) if (it.state === "exists") discard(it.id);
|
||||
setItems((all) => all.filter((it) => it.state !== "exists"));
|
||||
}, [discard]);
|
||||
const replaceAll = useCallback(() => {
|
||||
setItems((all) => all.map((it) => (it.state === "exists" ? { ...it, state: "queued", overwrite: true } : it)));
|
||||
}, []);
|
||||
const skipAll = useCallback(() => setItems((all) => all.filter((it) => it.state !== "exists")), []);
|
||||
const clearDone = useCallback(() => setItems((all) => all.filter((it) => it.state !== "done")), []);
|
||||
|
||||
const busy = items.some((it) => it.state === "queued" || it.state === "uploading");
|
||||
return { items, busy, add, replace, retry, remove, replaceAll, skipAll, clearDone };
|
||||
}
|
||||
|
||||
function noRoom(file: File, free: number): string {
|
||||
return i18next.t("files:upload_no_room", { free: formatBytes(free), size: formatBytes(file.size) });
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { api } from "@/lib/api";
|
||||
import { usePolling } from "@/lib/hooks";
|
||||
import type { ServerJob } from "@/lib/types";
|
||||
import { OP_POLL_MS } from "./sessionUpload";
|
||||
|
||||
/** holdsWorld says whether a Job keeps the world from being changed, as the
|
||||
* server counts it (maintenance.JobKind): a running backup, restore, world
|
||||
* export or file download. A backup export reads only the backup store. A
|
||||
* safety snapshot whose restore has yet to start holds it for that restore. */
|
||||
export function holdsWorld(j: ServerJob): boolean {
|
||||
return j.then_restore === "pending" || (j.state === "running" && j.kind !== "export_backup");
|
||||
}
|
||||
|
||||
/** restores says whether the holder is (or leads to) a restore, which replaces
|
||||
* the files the page lists. */
|
||||
function restores(j: ServerJob): boolean {
|
||||
return j.kind === "restore" || j.then_restore === "pending";
|
||||
}
|
||||
|
||||
/** holderText is the files key naming what holds the world and what to wait
|
||||
* for. */
|
||||
export function holderText(j: ServerJob): string {
|
||||
if (restores(j)) return "wait_for_restore";
|
||||
switch (j.kind) {
|
||||
case "backup":
|
||||
return "wait_for_backup";
|
||||
case "export_world":
|
||||
return "wait_for_world_export";
|
||||
default:
|
||||
return "wait_for_file_download";
|
||||
}
|
||||
}
|
||||
|
||||
// useWorldJobs follows the Jobs that hold a server's world besides the file
|
||||
// manager's own ops (useFileOps): read once when `enabled` turns on, then every
|
||||
// OP_POLL_MS while one holds it. onRestored runs when a restore seen here ends.
|
||||
export function useWorldJobs(server: string, enabled: boolean, onRestored: () => void) {
|
||||
const [holder, setHolder] = useState<ServerJob | null>(null);
|
||||
const restoring = useRef(false);
|
||||
const restored = useRef(onRestored);
|
||||
restored.current = onRestored;
|
||||
// As in useFileOps: only the newest read lands, and none starts over another.
|
||||
const seq = useRef(0);
|
||||
const inFlight = useRef(false);
|
||||
|
||||
const read = useCallback(async () => {
|
||||
if (inFlight.current) return;
|
||||
inFlight.current = true;
|
||||
const ticket = ++seq.current;
|
||||
try {
|
||||
const jobs = await api.serverJobs(server);
|
||||
if (ticket !== seq.current) return;
|
||||
const h = jobs.find(holdsWorld) ?? null;
|
||||
setHolder(h);
|
||||
const now = h !== null && restores(h);
|
||||
if (restoring.current && !now) restored.current();
|
||||
restoring.current = now;
|
||||
} catch {
|
||||
// A list that fails leaves the page as it was: a change the world cannot
|
||||
// take is still refused, in the server's words.
|
||||
} finally {
|
||||
inFlight.current = false;
|
||||
}
|
||||
}, [server]);
|
||||
|
||||
useEffect(() => {
|
||||
if (enabled) void read();
|
||||
else setHolder(null);
|
||||
}, [enabled, read]);
|
||||
|
||||
const poll = useCallback(() => void read(), [read]);
|
||||
usePolling(poll, enabled && holder !== null ? OP_POLL_MS : null);
|
||||
|
||||
return { holder, refresh: poll };
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// @vitest-environment jsdom
|
||||
import { describe, it, expect, vi, afterEach, beforeEach } from "vitest";
|
||||
import { describe, it, expect, vi, afterEach } from "vitest";
|
||||
import i18next from "i18next";
|
||||
import LanguageDetector from "i18next-browser-languagedetector";
|
||||
import { i18nOptions } from "./index";
|
||||
@@ -15,13 +15,6 @@ async function boot(languages: string[]) {
|
||||
return instance;
|
||||
}
|
||||
|
||||
// Importing ./index boots the app's own instance, which on some Node versions (22,
|
||||
// which CI runs) caches jsdom's en-US in storage before the first test; a first
|
||||
// visit starts from empty storage, and no test may inherit another's choice.
|
||||
beforeEach(() => {
|
||||
localStorage.clear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
@@ -43,7 +43,6 @@
|
||||
"job_scheduled": "Scheduled backup",
|
||||
"job_export_world": "World export",
|
||||
"job_export_backup": "Backup download",
|
||||
"job_export_files": "File download",
|
||||
"job_running": "Running",
|
||||
"job_succeeded": "Succeeded",
|
||||
"job_failed": "Failed",
|
||||
|
||||
@@ -25,14 +25,14 @@
|
||||
"no_backup": "There's no restorable backup for this server yet.",
|
||||
"backup_corrupt": "This backup failed its read-back check and can't be restored intact — pick another backup.",
|
||||
"not_stopped": "Stop the server completely first — this changes its world volume, which the running server holds.",
|
||||
"maintenance_in_progress": "This server's world is busy with a restore, backup, world export, file download, file change or idle reclaim. Try again once it finishes. A restore, backup or file change usually takes a minute or two; an export or a download lasts until the browser has all of it, and an idle reclaim can take longer on a large world.",
|
||||
"maintenance_in_progress": "This server's world is busy with a restore, backup, world export, file write or idle reclaim — try again once it finishes. A restore, backup or file write usually takes a minute or two; a world export lasts until its download ends, and an idle reclaim can take longer on a large world.",
|
||||
"no_world_volume": "This server has no world volume yet — start it once so it is created, then retry.",
|
||||
"file_changed": "This file changed after you opened it (another manager saved it, or the server rewrote it on its last run), so your save was not written, to keep that change.",
|
||||
"volume_full": "The server's volume is full, so the change was not written and the files there are unchanged. Delete files it no longer needs, or ask an admin to grow its volume.",
|
||||
"backup_cooldown": "This server was backed up moments ago, and manual backups have a cooldown — try again in a few minutes.",
|
||||
"backup_store_full": "The backup store is full, so manual backups are paused — ask an administrator to free space.",
|
||||
"restore_unavailable": "Restore isn't available right now — try again later.",
|
||||
"export_busy": "Too many exports are being prepared: one at a time per person, two at a time across the platform, and six an hour per person. Try again in a few minutes.",
|
||||
"export_busy": "Too many downloads are being prepared right now (one at a time per person, six an hour) — try again in a few minutes.",
|
||||
"export_expired": "This download has expired or was already used — start the export again.",
|
||||
"export_not_ready": "The download isn't ready yet — wait a moment and try again.",
|
||||
"export_unavailable": "Downloading worlds and backups isn't set up on this deployment — ask an administrator.",
|
||||
@@ -71,14 +71,6 @@
|
||||
"upload_staging_full": "The panel's upload space is nearly full right now, so the file was not passed on. Try again later, or ask an admin to free space on the uploads volume.",
|
||||
"upload_incomplete": "The upload stopped before the whole file arrived, so nothing was changed. Try again.",
|
||||
"length_required": "The upload did not say how large it is, so it was refused. Upload it again from the panel.",
|
||||
"digest_mismatch": "The file was changed on its way to the server, so it was refused and nothing was written. Try again.",
|
||||
"digest_required": "The request came without a checksum, so it was refused. Reload the panel and try again.",
|
||||
"bad_digest": "The request's checksum was malformed, so it was refused. Reload the panel and try again.",
|
||||
"file_unreadable": "The file could not be read: it was changed, moved or deleted after it was picked. Pick it again and upload.",
|
||||
"read_damaged": "The file arrived damaged, so it was not opened: saving it would write the damage back. Open it again.",
|
||||
"upload_not_found": "This upload is gone: it was cancelled, already landed, sat idle for 6 hours, or the panel service restarted. Upload the file again.",
|
||||
"too_many_uploads": "You already have 4 large uploads in progress. Wait for one to finish, or cancel one, and try again.",
|
||||
"op_lost": "The operation's progress can no longer be read. Refresh the list to see whether the file landed.",
|
||||
"jobs_unavailable": "The background job service isn't available right now.",
|
||||
"already_terminal": "This build already finished — there is nothing to cancel.",
|
||||
"build_unavailable": "Image builds aren't available right now.",
|
||||
@@ -90,7 +82,7 @@
|
||||
"submission_cooldown": "Too many submission requests — try again shortly.",
|
||||
"submissions_unavailable": "Submissions aren't available right now.",
|
||||
"uploads_unavailable": "Uploads aren't available right now.",
|
||||
"upload_busy": "Another part of this upload is still being sent — wait a moment and try again.",
|
||||
"upload_busy": "Another upload of this submission is still running — wait a moment and try again.",
|
||||
"upload_offset_mismatch": "The upload fell out of step with the server — try again to pick up where it stopped.",
|
||||
"uploads_store_unavailable": "The uploads store did not answer — try again in a moment; what was already sent is kept.",
|
||||
"part_too_large": "A piece of the upload was larger than the server accepts.",
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
"new_file": "New file",
|
||||
"new_folder": "New folder",
|
||||
"upload": "Upload",
|
||||
"upload_hint": "Upload files into this folder, or drop them on the list. Zip a folder before uploading it",
|
||||
"upload_hint": "Upload files into this folder (up to {{limit}} each), or drop them onto the list",
|
||||
"in_folder": "In {{dir}}",
|
||||
"name_label": "Name",
|
||||
"create": "Create",
|
||||
@@ -60,8 +60,8 @@
|
||||
"delete_folder_body": "The folder and everything in it are deleted for good. Take a backup first if you may want any of it back.",
|
||||
"secret_config_unreadable": "config/paper-global.yml holds the proxy forwarding secret every server shares, so the editor does not open it.",
|
||||
"drop_here": "Drop to upload into {{dir}}",
|
||||
"upload_no_folders_one": "Folders cannot be uploaded, so one was skipped: loose files cut off halfway leave a broken save behind. Zip it, upload the .zip, then choose Extract here on it.",
|
||||
"upload_no_folders_other": "Folders cannot be uploaded, so {{count}} were skipped: loose files cut off halfway leave a broken save behind. Zip them, upload the .zip, then choose Extract here on it.",
|
||||
"upload_no_folders_one": "A folder cannot be uploaded, so it was skipped. Make the folder here, then upload the files inside it.",
|
||||
"upload_no_folders_other": "Folders cannot be uploaded, so {{count}} were skipped. Make the folders here, then upload the files inside them.",
|
||||
"uploads_label": "Uploads",
|
||||
"uploads_title": "Uploads · {{done}} of {{total}} done",
|
||||
"upload_queued": "Waiting",
|
||||
@@ -78,56 +78,6 @@
|
||||
"upload_dismiss": "Dismiss {{name}}",
|
||||
"upload_clear_done": "Clear finished",
|
||||
"upload_progress_label": "Uploading {{name}}",
|
||||
"upload_folder_there": "A folder with this name is already here.",
|
||||
"upload_name_too_long": "This file's name is longer than the 255 bytes a file name on the server can hold. Rename it shorter, then upload it.",
|
||||
"upload_no_room": "The world volume has {{free}} free, not enough for this {{size}} file. Delete files you do not need, then retry.",
|
||||
"upload_landing_progress": "Writing it to the server… {{percent}}%",
|
||||
"wait_for_op": "Wait for the background operation to finish first.",
|
||||
"wait_for_download": "Wait until the download is ready first.",
|
||||
"wait_for_backup": "A backup of this world is running. Change files once it finishes.",
|
||||
"wait_for_restore": "This world is being restored. Change files once the restore finishes.",
|
||||
"wait_for_world_export": "This world is being exported. Change files once that download ends.",
|
||||
"wait_for_file_download": "A download is reading this world. Change files once it has finished.",
|
||||
"unzip_item": "Extract {{name}} here",
|
||||
"unzip_conflicts_title_one": "Extracting {{name}} replaces {{count}} file",
|
||||
"unzip_conflicts_title_other": "Extracting {{name}} replaces {{count}} files",
|
||||
"unzip_conflicts_body": "These files in the archive are already here. Confirm to replace them with the archive's versions; everything else extracts as usual. Take a backup first if you may want the old versions back.",
|
||||
"unzip_conflicts_more": "{{count}} more not listed.",
|
||||
"unzip_overwrite": "Replace and extract",
|
||||
"download_item": "Download {{name}}",
|
||||
"download_folder_item": "Download the folder {{name}} as a .zip",
|
||||
"download_preparing": "Preparing the download of {{name}}…",
|
||||
"download_started": "Downloading {{filename}}.",
|
||||
"download_started_props": "Downloading {{filename}}. Its rcon.password is redacted.",
|
||||
"download_started_config": "Downloading {{filename}}. paper-global.yml, the proxy forwarding secret every server shares, is left out.",
|
||||
"download_failed": "The download could not be prepared.",
|
||||
"download_failed_because": "The download could not be prepared: {{reason}}",
|
||||
"download_out_of_memory": "This folder holds more files than the zipping task has memory to list, so the system stopped it and nothing was downloaded. Download its folders one by one instead.",
|
||||
"download_busy": "Too many file downloads are being prepared: two at a time per person, four at a time across the platform, and thirty an hour per person. Try again in a few minutes.",
|
||||
"secret_config_no_download": "config/paper-global.yml holds the proxy forwarding secret every server shares, so it cannot be downloaded.",
|
||||
"ops_label": "Background operations",
|
||||
"ops_refresh_failed": "Could not read the background operations: {{reason}}",
|
||||
"op_unzip": "Extract {{path}}",
|
||||
"op_upload": "Write {{path}}",
|
||||
"op_preparing": "Getting ready…",
|
||||
"op_progress": "{{done}} of {{total}} · {{percent}}%",
|
||||
"op_progress_label": "Progress of {{path}}",
|
||||
"op_unzip_done_one": "Extracted {{count}} file, {{bytes}} in all.",
|
||||
"op_unzip_done_other": "Extracted {{count}} files, {{bytes}} in all.",
|
||||
"op_upload_done": "Written.",
|
||||
"op_dismiss": "Dismiss the result for {{path}}",
|
||||
"op_conflicts": "Review conflicts",
|
||||
"op_upload_exists": "A file with this name is already here, so nothing was replaced. Upload it again and choose Replace.",
|
||||
"op_unzip_conflicts_one": "It would replace {{count}} file already here, so nothing was extracted yet.",
|
||||
"op_unzip_conflicts_other": "It would replace {{count}} files already here, so nothing was extracted yet.",
|
||||
"op_volume_full": "Not enough room on the world volume: {{need}} needed, {{avail}} free. Nothing was changed.",
|
||||
"archive_invalid": "The archive is damaged, or not a zip file. Nothing was changed.",
|
||||
"archive_invalid_entry": "{{entry}} in the archive is damaged (its size or checksum does not match). Nothing was changed.",
|
||||
"archive_unsafe": "{{entry}} in the archive would land outside this folder, or is a device file. Nothing in the archive was extracted.",
|
||||
"archive_symlink": "{{entry}} in the archive is a symbolic link. Nothing in the archive was extracted.",
|
||||
"type_conflict": "{{entry}} is a file on one side and a folder on the other, which replacing cannot resolve. Rename or delete {{entry}} here, then extract again.",
|
||||
"job_failed": "The background task ended without saying why. Refresh the list to check, then try again.",
|
||||
"job_timed_out": "The background task did not finish within 2 hours and was stopped. Refresh the list to check, then try again.",
|
||||
"job_out_of_memory": "The background task ran out of memory and the system stopped it. The files on the server were not changed. Try again.",
|
||||
"job_out_of_memory_unzip": "The archive holds more files than the extraction task has memory to list, so the system stopped it. The files on the server were not changed. Split it into several smaller zips and extract each one."
|
||||
"upload_too_large": "Larger than the {{limit}} upload limit.",
|
||||
"upload_folder_there": "A folder with this name is already here."
|
||||
}
|
||||
@@ -43,7 +43,6 @@
|
||||
"job_scheduled": "定时备份",
|
||||
"job_export_world": "世界导出",
|
||||
"job_export_backup": "备份下载",
|
||||
"job_export_files": "文件下载",
|
||||
"job_running": "进行中",
|
||||
"job_succeeded": "成功",
|
||||
"job_failed": "失败",
|
||||
|
||||
@@ -25,14 +25,14 @@
|
||||
"no_backup": "这台服务器暂时没有可回档的备份。",
|
||||
"backup_corrupt": "这份备份回读校验未通过，已无法完整恢复——请选择另一份备份。",
|
||||
"not_stopped": "请先把服务器完全停止——这项操作要改动世界存储卷，运行中的服务器独占着它。",
|
||||
"maintenance_in_progress": "这台服务器的世界正在回档、备份、导出、下载文件、改动文件或闲置回收，等它完成后再试。回档、备份和改动文件通常一两分钟，导出和下载要等浏览器下载完，闲置回收视世界大小可能更久。",
|
||||
"maintenance_in_progress": "这台服务器的世界正在回档、备份、导出、写入文件或闲置回收——等它完成后再试。回档、备份和写文件通常一两分钟，导出要等下载结束，闲置回收视世界大小可能更久。",
|
||||
"no_world_volume": "这台服务器还没有世界卷——先启动一次让它创建，然后再试。",
|
||||
"file_changed": "这个文件在你打开之后被改过了（另一位管理者保存过，或者服务器上次运行时改写了它）。为了不覆盖那次修改，这次保存没有写入。",
|
||||
"volume_full": "服务器的存储卷已满，这次改动没有写入，原有文件保持不变。删掉用不着的文件，或者请管理员给它扩容。",
|
||||
"backup_cooldown": "这台服务器刚备份过，手动备份之间有冷却时间——请过几分钟再试。",
|
||||
"backup_store_full": "备份存储已满，暂时无法手动备份——请联系管理员清理空间。",
|
||||
"restore_unavailable": "回档功能当前不可用，请稍后再试。",
|
||||
"export_busy": "正在准备的导出太多了：每人同时一个，整个平台同时最多两个，每人每小时最多六个。请过几分钟再试。",
|
||||
"export_busy": "现在正在准备的下载太多了（每人同时一个、每小时最多六个）——请过几分钟再试。",
|
||||
"export_expired": "这个下载已过期或已经用过了——请重新导出。",
|
||||
"export_not_ready": "下载还没准备好——请稍等片刻再试。",
|
||||
"export_unavailable": "当前部署没有开通世界和备份下载——请联系管理员。",
|
||||
@@ -71,14 +71,6 @@
|
||||
"upload_staging_full": "面板的上传暂存空间快满了，这个文件没有转存过去。稍后再试，或者请管理员清理上传卷。",
|
||||
"upload_incomplete": "文件还没传完上传就中断了，什么都没有改动。请重试。",
|
||||
"length_required": "这次上传没有声明文件大小，被拒绝了。请从面板重新上传。",
|
||||
"digest_mismatch": "文件在传输途中被改动了，服务器已拒收，什么都没有写入。请重试。",
|
||||
"digest_required": "这次请求没有附带校验值，被拒绝了。请刷新面板后再试。",
|
||||
"bad_digest": "这次请求附带的校验值格式不对，被拒绝了。请刷新面板后再试。",
|
||||
"file_unreadable": "读不出这个文件：它在选中之后被改动、移走或删除了。请重新选择文件再上传。",
|
||||
"read_damaged": "文件传过来时损坏了，所以没有打开：保存它会把损坏写回去。请重新打开。",
|
||||
"upload_not_found": "这次分片上传已经不在了（取消过、已经写入、闲置超过 6 小时，或者面板服务重启过）。请重新上传。",
|
||||
"too_many_uploads": "你同时进行的大文件上传已经有 4 个了。等其中一个完成，或者取消一个再试。",
|
||||
"op_lost": "看不到这次操作的进度了。刷新列表看看文件有没有写入。",
|
||||
"jobs_unavailable": "后台任务服务当前不可用。",
|
||||
"already_terminal": "该构建已经结束，无法重复取消。",
|
||||
"build_unavailable": "构建功能当前不可用。",
|
||||
@@ -90,7 +82,7 @@
|
||||
"submission_cooldown": "操作太频繁——请稍后再试。",
|
||||
"submissions_unavailable": "提交流程当前不可用。",
|
||||
"uploads_unavailable": "上传功能当前不可用。",
|
||||
"upload_busy": "这次上传的另一部分仍在发送——请稍等片刻再试。",
|
||||
"upload_busy": "这个投稿的另一次上传仍在进行——请稍等片刻再试。",
|
||||
"upload_offset_mismatch": "上传进度与服务器对不上——重试即可从中断处接着传。",
|
||||
"uploads_store_unavailable": "上传存储暂时没有响应——稍后重试即可，已传的部分会保留。",
|
||||
"part_too_large": "上传的某一片超过了服务器接受的大小。",
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
"new_file": "新建文件",
|
||||
"new_folder": "新建文件夹",
|
||||
"upload": "上传",
|
||||
"upload_hint": "上传文件到这个文件夹，也可以直接拖到列表上。文件夹请先压成 .zip 再上传",
|
||||
"upload_hint": "上传文件到这个文件夹（每个最大 {{limit}}），也可以直接拖到列表上",
|
||||
"in_folder": "位置：{{dir}}",
|
||||
"name_label": "名称",
|
||||
"create": "创建",
|
||||
@@ -60,7 +60,7 @@
|
||||
"delete_folder_body": "文件夹和里面的所有内容都会被永久删除。之后可能还要用的话，先做一次备份。",
|
||||
"secret_config_unreadable": "config/paper-global.yml 里有所有服务器共用的代理转发密钥，编辑器不打开它。",
|
||||
"drop_here": "松开即可上传到 {{dir}}",
|
||||
"upload_no_folders": "不支持上传文件夹（已跳过 {{count}} 个）：散文件传到一半断掉会留下残缺存档。请先压成 .zip 上传，再对它点「解压到此处」。",
|
||||
"upload_no_folders": "文件夹没法直接上传，已跳过 {{count}} 个。先在这里新建同名文件夹，再上传里面的文件。",
|
||||
"uploads_label": "上传",
|
||||
"uploads_title": "上传 · 已完成 {{done}}/{{total}}",
|
||||
"upload_queued": "等待中",
|
||||
@@ -77,53 +77,6 @@
|
||||
"upload_dismiss": "移除 {{name}}",
|
||||
"upload_clear_done": "清除已完成",
|
||||
"upload_progress_label": "正在上传 {{name}}",
|
||||
"upload_folder_there": "这里已经有同名文件夹。",
|
||||
"upload_name_too_long": "这个文件名超过 255 字节，服务器上的文件名存不下。先改短再上传。",
|
||||
"upload_no_room": "世界卷只剩 {{free}}，放不下这个 {{size}} 的文件。先删掉些用不着的文件再重试。",
|
||||
"upload_landing_progress": "正在写入服务器… {{percent}}%",
|
||||
"wait_for_op": "等后台操作完成后再改动。",
|
||||
"wait_for_download": "等下载准备好后再操作。",
|
||||
"wait_for_backup": "正在备份这台服务器的世界，备份结束后才能改动文件。",
|
||||
"wait_for_restore": "正在回档这台服务器的世界，回档结束后才能改动文件。",
|
||||
"wait_for_world_export": "正在导出这台服务器的世界，那边的下载结束后才能改动文件。",
|
||||
"wait_for_file_download": "有一个下载正在读取这台服务器的世界，传完后才能改动文件。",
|
||||
"unzip_item": "把 {{name}} 解压到此处",
|
||||
"unzip_conflicts_title": "解压 {{name}} 会覆盖 {{count}} 个文件",
|
||||
"unzip_conflicts_body": "压缩包里的这些文件在这里已经存在。确认后会用压缩包里的版本替换它们，其余文件照常解压。旧版本之后可能还要用的话，先做一次备份。",
|
||||
"unzip_conflicts_more": "还有 {{count}} 个没有列出。",
|
||||
"unzip_overwrite": "覆盖并解压",
|
||||
"download_item": "下载 {{name}}",
|
||||
"download_folder_item": "把文件夹 {{name}} 打包成 .zip 下载",
|
||||
"download_preparing": "正在准备 {{name}} 的下载…",
|
||||
"download_started": "已开始下载 {{filename}}。",
|
||||
"download_started_props": "已开始下载 {{filename}}。里面的 rcon.password 已隐去。",
|
||||
"download_started_config": "已开始下载 {{filename}}。所有服务器共用的代理转发密钥 paper-global.yml 不在里面。",
|
||||
"download_failed": "下载没有准备好。",
|
||||
"download_failed_because": "下载没有准备好：{{reason}}",
|
||||
"download_out_of_memory": "这个文件夹里的文件太多，打包任务的内存装不下它的文件清单，被系统停掉了，什么都没下载。请分成几个小一些的文件夹分别下载。",
|
||||
"download_busy": "正在准备的文件下载太多了：每人同时最多两个，整个平台同时最多四个，每人每小时最多三十个。请过几分钟再试。",
|
||||
"secret_config_no_download": "config/paper-global.yml 里有所有服务器共用的代理转发密钥，不能下载。",
|
||||
"ops_label": "后台操作",
|
||||
"ops_refresh_failed": "读取后台操作失败：{{reason}}",
|
||||
"op_unzip": "解压 {{path}}",
|
||||
"op_upload": "写入 {{path}}",
|
||||
"op_preparing": "准备中…",
|
||||
"op_progress": "{{done}} / {{total}} · {{percent}}%",
|
||||
"op_progress_label": "{{path}} 的进度",
|
||||
"op_unzip_done": "已解压 {{count}} 个文件，共 {{bytes}}。",
|
||||
"op_upload_done": "已写入。",
|
||||
"op_dismiss": "关闭 {{path}} 的结果",
|
||||
"op_conflicts": "查看冲突",
|
||||
"op_upload_exists": "这里已经有同名文件，没有替换。重新上传时选「替换」即可。",
|
||||
"op_unzip_conflicts": "会覆盖 {{count}} 个已有文件，还没有解压。",
|
||||
"op_volume_full": "世界卷空间不够：需要 {{need}}，只剩 {{avail}}。什么都没有改动。",
|
||||
"archive_invalid": "这个压缩包损坏了，或者不是 zip 文件。什么都没有改动。",
|
||||
"archive_invalid_entry": "压缩包里的 {{entry}} 损坏了（大小或校验和对不上）。什么都没有改动。",
|
||||
"archive_unsafe": "压缩包里的 {{entry}} 会写到这个文件夹外面，或者是设备文件。整个压缩包都没有解压。",
|
||||
"archive_symlink": "压缩包里的 {{entry}} 是符号链接。整个压缩包都没有解压。",
|
||||
"type_conflict": "压缩包里的 {{entry}} 和这里已有的同名项一个是文件、一个是文件夹，覆盖解决不了。先把这里的 {{entry}} 改名或删掉再解压。",
|
||||
"job_failed": "后台任务没说明原因就结束了。刷新列表确认一下，再试一次。",
|
||||
"job_timed_out": "后台任务 2 小时还没做完，被停下了。刷新列表确认一下，再试一次。",
|
||||
"job_out_of_memory": "后台任务用完了内存，被系统停掉了。服务器上的文件没有被改动，再试一次。",
|
||||
"job_out_of_memory_unzip": "压缩包里的文件太多，解压任务的内存装不下它的文件清单，被系统停掉了。服务器上的文件没有被改动。把它拆成几个小一些的 zip，分别上传解压。"
|
||||
"upload_too_large": "超过 {{limit}} 的上传上限。",
|
||||
"upload_folder_there": "这里已经有同名文件夹。"
|
||||
}
|
||||
+23
-155
@@ -1,7 +1,3 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
// hexOf is the SHA-256 (hex) of the bytes a file editor call carries.
|
||||
const hexOf = (bytes: string | Uint8Array) => createHash("sha256").update(bytes).digest("hex");
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
|
||||
// Pin the GET /me wire shape. is_admin crosses an untyped fetch().json() boundary
|
||||
@@ -662,6 +658,20 @@ describe("image whitelist and builds wire shapes", () => {
|
||||
expect(JSON.parse((opts as RequestInit).body as string)).toEqual({ display_name: "new submission" });
|
||||
});
|
||||
|
||||
it("uploadSubmissionContext POSTs Blob to /me/submissions/{id}/context", async () => {
|
||||
const sub = { id: "sub-3", display_name: "new submission", status: "pending_review" };
|
||||
const fetchSpy = fakeFetch(sub);
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
const blob = new Blob(["test"], { type: "application/x-gzip" });
|
||||
const res = await api.uploadSubmissionContext("sub-3", blob);
|
||||
expect(res).toEqual(sub);
|
||||
const [url, opts] = (fetchSpy as unknown as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
expect(String(url)).toBe("/me/submissions/sub-3/context");
|
||||
expect((opts as RequestInit).method).toBe("POST");
|
||||
expect((opts as RequestInit).body).toBe(blob);
|
||||
expect((opts as RequestInit).headers).toEqual({ "Content-Type": "application/x-gzip" });
|
||||
});
|
||||
|
||||
// The lane's two throttled outcomes (a spent allowance, a closed cooldown)
|
||||
// must surface as their own copy, not the generic forbidden/error text.
|
||||
it("maps the submission quota/cooldown codes to stable human copy", async () => {
|
||||
@@ -856,29 +866,15 @@ describe("image whitelist and builds wire shapes", () => {
|
||||
});
|
||||
|
||||
it("readServerFile GETs /servers/{name}/file and passes base64 through", async () => {
|
||||
const content_sha256 = hexOf(new Uint8Array([0, 1, 2]));
|
||||
const fetchSpy = fakeFetch({ path: "world/level.dat", content: "AAEC", sha256: "a".repeat(64), content_sha256 });
|
||||
const fetchSpy = fakeFetch({ path: "world/level.dat", content: "AAEC" });
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
const res = await api.readServerFile("survival", "world/level.dat");
|
||||
expect(res).toEqual({ path: "world/level.dat", content: "AAEC", sha256: "a".repeat(64), content_sha256 });
|
||||
expect(res.content).toBe("AAEC");
|
||||
const [url, opts] = (fetchSpy as unknown as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
expect(String(url)).toBe("/servers/survival/file?path=world%2Flevel.dat");
|
||||
expect((opts as RequestInit).method).toBe("GET");
|
||||
});
|
||||
|
||||
// An editor that opened a damaged read would save the damage back.
|
||||
it.each([
|
||||
["hash otherwise", "AAED"],
|
||||
["are not base64 at all", "AA=E"],
|
||||
])("readServerFile refuses content whose bytes %s", async (_, content) => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
fakeFetch({ path: "world/level.dat", content, sha256: "a".repeat(64), content_sha256: hexOf(new Uint8Array([0, 1, 2])) }),
|
||||
);
|
||||
await expect(api.readServerFile("survival", "world/level.dat")).rejects.toMatchObject({ code: "read_damaged" });
|
||||
expect(humanizeError({ code: "read_damaged" })).toMatch(/arrived damaged/);
|
||||
});
|
||||
|
||||
it("writeServerFile PUTs {content} — an explicit \"\" is a deliberate truncate, not an omitted field", async () => {
|
||||
const fetchSpy = fakeFetch({ path: "a.txt", status: "written" });
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
@@ -887,7 +883,7 @@ describe("image whitelist and builds wire shapes", () => {
|
||||
const [url, opts] = (fetchSpy as unknown as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
expect(String(url)).toBe("/servers/survival/file?path=a.txt");
|
||||
expect((opts as RequestInit).method).toBe("PUT");
|
||||
expect((opts as RequestInit).body).toBe(JSON.stringify({ content: "", content_sha256: hexOf("") }));
|
||||
expect((opts as RequestInit).body).toBe(JSON.stringify({ content: "" }));
|
||||
});
|
||||
|
||||
it("writeServerFile sends the hash the read returned as expect_sha256", async () => {
|
||||
@@ -897,7 +893,7 @@ describe("image whitelist and builds wire shapes", () => {
|
||||
expect(res.sha256).toBe("b".repeat(64));
|
||||
const [, opts] = (fetchSpy as unknown as ReturnType<typeof vi.fn>).mock.calls[0];
|
||||
expect((opts as RequestInit).body).toBe(
|
||||
JSON.stringify({ content: "aGk=", content_sha256: hexOf("hi"), expect_sha256: "a".repeat(64) }),
|
||||
JSON.stringify({ content: "aGk=", expect_sha256: "a".repeat(64) }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1218,10 +1214,6 @@ async function sentXHR(): Promise<FakeXHR> {
|
||||
return FakeXHR.last as FakeXHR;
|
||||
}
|
||||
|
||||
// What felis-api's parseContentDigest takes: the body's SHA-256, base64, in
|
||||
// RFC 9530's sha-256=:…: form. Hashed here by Node, apart from the panel's own.
|
||||
const contentDigest = (body: string) => `sha-256=:${createHash("sha256").update(body).digest("base64")}:`;
|
||||
|
||||
describe("chunked context upload", () => {
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
@@ -1251,7 +1243,7 @@ describe("chunked context upload", () => {
|
||||
expect((opts as RequestInit).method).toBe("POST");
|
||||
});
|
||||
|
||||
it("putContextPart PUTs the part at its offset with the session cookie, its SHA-256 and reports progress", async () => {
|
||||
it("putContextPart PUTs the part at its offset with the session cookie and reports progress", async () => {
|
||||
const part = new Blob(["abcd"]);
|
||||
const seen: number[] = [];
|
||||
const done = api.putContextPart("sub-3", 8, part, { onProgress: (n) => seen.push(n) });
|
||||
@@ -1259,7 +1251,7 @@ describe("chunked context upload", () => {
|
||||
expect(xhr.method).toBe("PUT");
|
||||
expect(xhr.url).toBe("/me/submissions/sub-3/context/upload?offset=8");
|
||||
expect(xhr.withCredentials).toBe(true);
|
||||
expect(xhr.headers).toEqual({ "Content-Type": "application/octet-stream", "Content-Digest": contentDigest("abcd") });
|
||||
expect(xhr.headers).toEqual({ "Content-Type": "application/octet-stream" });
|
||||
expect(xhr.body).toBe(part);
|
||||
xhr.upload.onprogress?.({ loaded: 3 });
|
||||
xhr.respond(200, JSON.stringify(progress));
|
||||
@@ -1356,28 +1348,6 @@ describe("server file manager wire shapes", () => {
|
||||
return [String(url), opts as RequestInit];
|
||||
}
|
||||
|
||||
it("a file the browser can no longer read is refused before anything is sent", async () => {
|
||||
const changed = Object.assign(new Blob(["jar bytes"]), {
|
||||
arrayBuffer: () => Promise.reject(new DOMException("the file changed on disk", "NotReadableError")),
|
||||
});
|
||||
await expect(api.uploadServerFile("survival", "plugins/a.jar", changed, false)).rejects.toEqual({
|
||||
status: 0,
|
||||
code: "file_unreadable",
|
||||
message: "the file changed on disk",
|
||||
});
|
||||
await expect(api.putServerFileUploadPart("survival", "s1", 0, changed)).rejects.toMatchObject({
|
||||
code: "file_unreadable",
|
||||
});
|
||||
expect(FakeXHR.last).toBeUndefined();
|
||||
expect(humanizeError({ code: "file_unreadable" })).toMatch(/changed, moved or deleted/);
|
||||
});
|
||||
|
||||
it("the upload refusals over a checksum read as what to do next", () => {
|
||||
expect(humanizeError({ code: "digest_mismatch" })).toMatch(/changed on its way/);
|
||||
expect(humanizeError({ code: "digest_required" })).toMatch(/without a checksum/);
|
||||
expect(humanizeError({ code: "bad_digest" })).toMatch(/checksum was malformed/);
|
||||
});
|
||||
|
||||
it("createServerFile PUTs the content with create_only, so nothing already there is replaced", async () => {
|
||||
const fetchSpy = fakeFetch({ path: "plugins/new.yml", status: "written", sha256: "c".repeat(64) });
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
@@ -1389,7 +1359,7 @@ describe("server file manager wire shapes", () => {
|
||||
const [url, opts] = sent(fetchSpy);
|
||||
expect(url).toBe("/servers/survival/file?path=plugins%2Fnew.yml");
|
||||
expect(opts.method).toBe("PUT");
|
||||
expect(opts.body).toBe(JSON.stringify({ content: "", content_sha256: hexOf(""), create_only: true }));
|
||||
expect(opts.body).toBe(JSON.stringify({ content: "", create_only: true }));
|
||||
});
|
||||
|
||||
it("deleteServerFile DELETEs /servers/{name}/file with no body", async () => {
|
||||
@@ -1436,7 +1406,7 @@ describe("server file manager wire shapes", () => {
|
||||
expect(xhr.method).toBe("PUT");
|
||||
expect(xhr.url).toBe("/servers/survival/files/upload?path=plugins%2FChunky%201.4.jar");
|
||||
expect(xhr.withCredentials).toBe(true);
|
||||
expect(xhr.headers).toEqual({ "Content-Type": "application/octet-stream", "Content-Digest": contentDigest("jar bytes") });
|
||||
expect(xhr.headers).toEqual({ "Content-Type": "application/octet-stream" });
|
||||
expect(xhr.body).toBe(file);
|
||||
xhr.upload.onprogress?.({ loaded: 4 });
|
||||
xhr.respond(200, JSON.stringify({ path: "plugins/Chunky 1.4.jar", status: "uploaded", sha256: "d".repeat(64), size: 9 }));
|
||||
@@ -1470,108 +1440,6 @@ describe("server file manager wire shapes", () => {
|
||||
expect(fetchSpy).not.toHaveBeenCalled();
|
||||
expect(FakeXHR.last).toBeUndefined();
|
||||
});
|
||||
|
||||
const session = { id: "s1", path: "worlds/big world.zip", size: 100_000_000, received: 0, part_max_bytes: 33_554_432 };
|
||||
const op = {
|
||||
id: "op1",
|
||||
op: "upload",
|
||||
path: "worlds/big world.zip",
|
||||
state: "running",
|
||||
started_at: "2026-09-28T00:00:00Z",
|
||||
done: 0,
|
||||
total: 0,
|
||||
};
|
||||
|
||||
it("beginServerFileUpload POSTs the size, with the path in the query", async () => {
|
||||
const fetchSpy = fakeFetch(session);
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
expect(await api.beginServerFileUpload("survival", "worlds/big world.zip", 100_000_000)).toEqual(session);
|
||||
const [url, opts] = sent(fetchSpy);
|
||||
expect(url).toBe("/servers/survival/files/uploads?path=worlds%2Fbig%20world.zip");
|
||||
expect(opts.method).toBe("POST");
|
||||
expect(opts.body).toBe(JSON.stringify({ size: 100_000_000 }));
|
||||
});
|
||||
|
||||
it("getServerFileUpload and deleteServerFileUpload name the session in the path", async () => {
|
||||
const fetchSpy = fakeFetch(session);
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
expect(await api.getServerFileUpload("survival", "s1")).toEqual(session);
|
||||
await api.deleteServerFileUpload("survival", "s1");
|
||||
const calls = (fetchSpy as unknown as ReturnType<typeof vi.fn>).mock.calls.map(([u, o]) => [
|
||||
String(u),
|
||||
(o as RequestInit).method,
|
||||
]);
|
||||
expect(calls).toEqual([
|
||||
["/servers/survival/files/uploads/s1", "GET"],
|
||||
["/servers/survival/files/uploads/s1", "DELETE"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("putServerFileUploadPart PUTs the part's raw bytes at its offset and reports progress", async () => {
|
||||
const part = new Blob(["part bytes"]);
|
||||
const seen: number[] = [];
|
||||
const done = api.putServerFileUploadPart("survival", "s1", 33_554_432, part, { onProgress: (n) => seen.push(n) });
|
||||
const xhr = await sentXHR();
|
||||
expect(xhr.method).toBe("PUT");
|
||||
expect(xhr.url).toBe("/servers/survival/files/uploads/s1?offset=33554432");
|
||||
expect(xhr.withCredentials).toBe(true);
|
||||
expect(xhr.headers).toEqual({ "Content-Type": "application/octet-stream", "Content-Digest": contentDigest("part bytes") });
|
||||
expect(xhr.body).toBe(part);
|
||||
xhr.upload.onprogress?.({ loaded: 3 });
|
||||
xhr.respond(200, JSON.stringify({ ...session, received: 33_554_442 }));
|
||||
expect(await done).toEqual({ ...session, received: 33_554_442 });
|
||||
expect(seen).toEqual([3]);
|
||||
});
|
||||
|
||||
it("commitServerFileUpload POSTs overwrite and answers the op landing the file", async () => {
|
||||
const fetchSpy = fakeFetch({ op });
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
expect(await api.commitServerFileUpload("survival", "s1", true)).toEqual({ op });
|
||||
const [url, opts] = sent(fetchSpy);
|
||||
expect(url).toBe("/servers/survival/files/uploads/s1/commit");
|
||||
expect(opts.method).toBe("POST");
|
||||
expect(opts.body).toBe(JSON.stringify({ overwrite: true }));
|
||||
});
|
||||
|
||||
it("unzipServerFile POSTs overwrite with the archive in the query", async () => {
|
||||
const unzip = { ...op, op: "unzip", path: "maps/Spawn 2.zip" };
|
||||
const fetchSpy = fakeFetch({ op: unzip });
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
expect(await api.unzipServerFile("survival", "maps/Spawn 2.zip", false)).toEqual({ op: unzip });
|
||||
const [url, opts] = sent(fetchSpy);
|
||||
expect(url).toBe("/servers/survival/files/unzip?path=maps%2FSpawn%202.zip");
|
||||
expect(opts.method).toBe("POST");
|
||||
expect(opts.body).toBe(JSON.stringify({ overwrite: false }));
|
||||
});
|
||||
|
||||
it("listServerFileOps GETs the server's ops", async () => {
|
||||
const fetchSpy = fakeFetch({ ops: [op] });
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
expect(await api.listServerFileOps("survival")).toEqual({ ops: [op] });
|
||||
const [url, opts] = sent(fetchSpy);
|
||||
expect(url).toBe("/servers/survival/files/ops");
|
||||
expect(opts.method).toBe("GET");
|
||||
});
|
||||
|
||||
it.each([
|
||||
[false, "server.properties", "/servers/survival/files/download?path=server.properties&dir=false"],
|
||||
[true, "world/data", "/servers/survival/files/download?path=world%2Fdata&dir=true"],
|
||||
])("downloadServerFile POSTs the path and whether it is a folder (dir=%s)", async (dir, path, want) => {
|
||||
const ticket = { ticket: "t1", state: "pending", filename: "x" };
|
||||
const fetchSpy = fakeFetch(ticket);
|
||||
vi.stubGlobal("fetch", fetchSpy);
|
||||
expect(await api.downloadServerFile("survival", path, dir)).toEqual(ticket);
|
||||
const [url, opts] = sent(fetchSpy);
|
||||
expect(url).toBe(want);
|
||||
expect(opts.method).toBe("POST");
|
||||
expect(opts.body).toBeUndefined();
|
||||
});
|
||||
|
||||
it("words the upload session codes in the panel's own copy", () => {
|
||||
expect(humanizeError({ status: 404, code: "upload_not_found", message: "raw" })).toMatch(/^This upload is gone/);
|
||||
expect(humanizeError({ status: 429, code: "too_many_uploads", message: "raw" })).toMatch(/4 large uploads in progress/);
|
||||
expect(humanizeError({ status: 0, code: "op_lost", message: "" })).toMatch(/progress can no longer be read/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("scheduled task wire shapes", () => {
|
||||
@@ -1788,7 +1656,7 @@ describe("world export wire shapes", () => {
|
||||
|
||||
it("words the export refusals itself", () => {
|
||||
expect(humanizeError({ status: 429, code: "export_busy", message: "raw" })).toBe(
|
||||
"Too many exports are being prepared: one at a time per person, two at a time across the platform, and six an hour per person. Try again in a few minutes.",
|
||||
"Too many downloads are being prepared right now (one at a time per person, six an hour) — try again in a few minutes.",
|
||||
);
|
||||
expect(humanizeError({ status: 410, code: "export_expired", message: "raw" })).toBe(
|
||||
"This download has expired or was already used — start the export again.",
|
||||
|
||||
+31
-142
@@ -12,8 +12,6 @@ import type {
|
||||
CreateUserRequest,
|
||||
ExportStatus,
|
||||
ExportTicket,
|
||||
FileOp,
|
||||
FileUploadSession,
|
||||
FleetServer,
|
||||
Identity,
|
||||
KickResult,
|
||||
@@ -45,7 +43,6 @@ import type {
|
||||
UpdateReport,
|
||||
} from "./types";
|
||||
import { loadConfig } from "./config";
|
||||
import { base64Sha256, sha256Of } from "./digest";
|
||||
import i18next from "i18next";
|
||||
|
||||
// Typed client for the felis-api external face (spec §7). Credentials are sent so
|
||||
@@ -232,19 +229,28 @@ function request<T>(method: string, path: string, body?: unknown): Promise<T> {
|
||||
});
|
||||
}
|
||||
|
||||
function requestRaw<T>(
|
||||
method: string,
|
||||
path: string,
|
||||
body: Blob,
|
||||
headers?: Record<string, string>,
|
||||
): Promise<T> {
|
||||
return send<T>(path, { method, headers, body });
|
||||
}
|
||||
|
||||
// sendWithProgress sends body by XMLHttpRequest, the one browser API that
|
||||
// reports how much of a request body has gone out (fetch has no upload
|
||||
// progress), and settles the way send does: the parsed 2xx body, or the same
|
||||
// ApiError fetchOK would throw. onProgress gets the bytes of body sent so far;
|
||||
// signal aborts the request with an AbortError; headers go along with it.
|
||||
// signal aborts the request with an AbortError.
|
||||
async function sendWithProgress<T>(
|
||||
method: string,
|
||||
path: string,
|
||||
body: Blob,
|
||||
opts: { onProgress?: (sent: number) => void; signal?: AbortSignal; headers?: Record<string, string> } = {},
|
||||
opts: { onProgress?: (sent: number) => void; signal?: AbortSignal } = {},
|
||||
): Promise<T> {
|
||||
const { apiBase } = await loadConfig();
|
||||
const { onProgress, signal, headers } = opts;
|
||||
const { onProgress, signal } = opts;
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
const onAbort = () => xhr.abort();
|
||||
@@ -252,7 +258,6 @@ async function sendWithProgress<T>(
|
||||
xhr.open(method, `${apiBase}${path}`);
|
||||
xhr.withCredentials = true;
|
||||
xhr.setRequestHeader("Content-Type", "application/octet-stream");
|
||||
for (const [k, v] of Object.entries(headers ?? {})) xhr.setRequestHeader(k, v);
|
||||
if (onProgress) xhr.upload.onprogress = (e) => onProgress(e.loaded);
|
||||
xhr.onabort = () => {
|
||||
settle();
|
||||
@@ -290,19 +295,6 @@ async function sendWithProgress<T>(
|
||||
});
|
||||
}
|
||||
|
||||
// sendDigested sends body as sendWithProgress does, with its SHA-256 in
|
||||
// Content-Digest: felis-api hashes what arrives and keeps none of it on a
|
||||
// mismatch (400 digest_mismatch), so bytes changed on the way never land.
|
||||
async function sendDigested<T>(
|
||||
method: string,
|
||||
path: string,
|
||||
body: Blob,
|
||||
opts: { onProgress?: (sent: number) => void; signal?: AbortSignal } = {},
|
||||
): Promise<T> {
|
||||
const { header } = await sha256Of(body);
|
||||
return sendWithProgress<T>(method, path, body, { ...opts, headers: { "Content-Digest": header } });
|
||||
}
|
||||
|
||||
// rejectingSync turns a synchronous throw inside an api method (urlPath refusing
|
||||
// a segment) into a rejected promise, so every caller handles it the way it
|
||||
// handles any failed call.
|
||||
@@ -756,11 +748,9 @@ export const api = rejectingSync({
|
||||
// volume is RWO), so callers gate on phase === "Stopped". The path travels as a
|
||||
// query parameter — a file path contains "/" and never round-trips through a
|
||||
// path segment. Content is []byte on the wire, which Go's encoding/json renders
|
||||
// as base64, so it is binary-safe in both directions. A listing also says how
|
||||
// much room the world volume has (free_bytes), so an upload too big for it is
|
||||
// refused before it is sent.
|
||||
// as base64, so it is binary-safe in both directions.
|
||||
listServerFiles: (name: string, path: string) =>
|
||||
request<{ path: string; entries: ServerFileEntry[]; truncated: boolean; free_bytes: number | null }>(
|
||||
request<{ path: string; entries: ServerFileEntry[]; truncated: boolean }>(
|
||||
"GET",
|
||||
urlPath`/servers/${name}/files` + `?path=${encodeURIComponent(path)}`,
|
||||
),
|
||||
@@ -768,49 +758,31 @@ export const api = rejectingSync({
|
||||
// readServerFile returns one file's bytes (base64) and the sha256 of the file
|
||||
// as stored. A file over the read ceiling is a 413, never a silent truncation,
|
||||
// because a later save of a truncated body would destroy the rest of the file.
|
||||
// The content must hash to content_sha256, the digest of the bytes as sent: a
|
||||
// read damaged on the way is refused (read_damaged) rather than opened, since
|
||||
// saving it would write the damage back.
|
||||
readServerFile: async (name: string, path: string) => {
|
||||
const r = await request<{ path: string; content: string; sha256: string; content_sha256: string }>(
|
||||
readServerFile: (name: string, path: string) =>
|
||||
request<{ path: string; content: string; sha256: string }>(
|
||||
"GET",
|
||||
urlPath`/servers/${name}/file` + `?path=${encodeURIComponent(path)}`,
|
||||
);
|
||||
let got = "";
|
||||
try {
|
||||
got = await base64Sha256(r.content);
|
||||
} catch {
|
||||
/* not base64 at all: damaged too */
|
||||
}
|
||||
if (got !== r.content_sha256) throw clientError("read_damaged");
|
||||
return r;
|
||||
},
|
||||
),
|
||||
|
||||
// writeServerFile atomically replaces a file's contents (creating it if
|
||||
// absent). Sending an explicit "" is a deliberate truncate; the wire field is
|
||||
// required, but that is enforced by the caller (this method always sends one).
|
||||
// With expectSha256 (the hash the read returned) a file someone changed since
|
||||
// is refused with 409 file_changed; without it the write is unconditional.
|
||||
// The content goes with its SHA-256 (content_sha256), so content changed on
|
||||
// the way is refused (digest_mismatch) and nothing is written.
|
||||
writeServerFile: async (name: string, path: string, content: string, expectSha256?: string) =>
|
||||
writeServerFile: (name: string, path: string, content: string, expectSha256?: string) =>
|
||||
request<{ path: string; status: string; sha256: string }>(
|
||||
"PUT",
|
||||
urlPath`/servers/${name}/file` + `?path=${encodeURIComponent(path)}`,
|
||||
{
|
||||
content,
|
||||
content_sha256: await base64Sha256(content),
|
||||
...(expectSha256 ? { expect_sha256: expectSha256 } : {}),
|
||||
},
|
||||
expectSha256 ? { content, expect_sha256: expectSha256 } : { content },
|
||||
),
|
||||
|
||||
// createServerFile makes a new file with content, and only if nothing is at the
|
||||
// path yet: something that appeared meanwhile is 409 file_exists, never replaced.
|
||||
createServerFile: async (name: string, path: string, content: string) =>
|
||||
createServerFile: (name: string, path: string, content: string) =>
|
||||
request<{ path: string; status: string; sha256: string }>(
|
||||
"PUT",
|
||||
urlPath`/servers/${name}/file` + `?path=${encodeURIComponent(path)}`,
|
||||
{ content, content_sha256: await base64Sha256(content), create_only: true },
|
||||
{ content, create_only: true },
|
||||
),
|
||||
|
||||
// deleteServerFile deletes a file, a link (never what it names) or a folder with
|
||||
@@ -839,9 +811,8 @@ export const api = rejectingSync({
|
||||
),
|
||||
|
||||
// uploadServerFile sends a file's raw bytes (the browser sets Content-Length
|
||||
// from the Blob) and their SHA-256, with progress. Without overwrite an
|
||||
// existing file is 409 file_exists; with it the file is replaced whole or not
|
||||
// at all.
|
||||
// from the Blob) with progress. Without overwrite an existing file is 409
|
||||
// file_exists; with it the file is replaced whole or not at all.
|
||||
uploadServerFile: (
|
||||
name: string,
|
||||
path: string,
|
||||
@@ -849,7 +820,7 @@ export const api = rejectingSync({
|
||||
overwrite: boolean,
|
||||
opts?: { onProgress?: (sent: number) => void; signal?: AbortSignal },
|
||||
) =>
|
||||
sendDigested<{ path: string; status: string; sha256: string; size: number }>(
|
||||
sendWithProgress<{ path: string; status: string; sha256: string; size: number }>(
|
||||
"PUT",
|
||||
urlPath`/servers/${name}/files/upload` +
|
||||
`?path=${encodeURIComponent(path)}` +
|
||||
@@ -858,68 +829,6 @@ export const api = rejectingSync({
|
||||
opts,
|
||||
),
|
||||
|
||||
// A file too big for one request goes up in parts (components/files/
|
||||
// sessionUpload.ts drives it): begin a session for its path and size, which
|
||||
// reserves room for all of it; put each part at its byte offset, with its
|
||||
// SHA-256; then commit, which answers at once with the op landing it (watch
|
||||
// listServerFileOps). A session answers only the account and server it was
|
||||
// begun for, and lists the parts it holds with their SHA-256. It stays until
|
||||
// the Job landing its file reports it landed, so a landing that fails can be
|
||||
// committed again, and is dropped after 6 hours idle.
|
||||
beginServerFileUpload: (name: string, path: string, size: number) =>
|
||||
request<FileUploadSession>(
|
||||
"POST",
|
||||
urlPath`/servers/${name}/files/uploads` + `?path=${encodeURIComponent(path)}`,
|
||||
{ size },
|
||||
),
|
||||
|
||||
getServerFileUpload: (name: string, id: string) =>
|
||||
request<FileUploadSession>("GET", urlPath`/servers/${name}/files/uploads/${id}`),
|
||||
|
||||
putServerFileUploadPart: (
|
||||
name: string,
|
||||
id: string,
|
||||
offset: number,
|
||||
part: Blob,
|
||||
opts?: { onProgress?: (sent: number) => void; signal?: AbortSignal },
|
||||
) =>
|
||||
sendDigested<FileUploadSession>(
|
||||
"PUT",
|
||||
urlPath`/servers/${name}/files/uploads/${id}` + `?offset=${offset}`,
|
||||
part,
|
||||
opts,
|
||||
),
|
||||
|
||||
deleteServerFileUpload: (name: string, id: string) =>
|
||||
request<null>("DELETE", urlPath`/servers/${name}/files/uploads/${id}`),
|
||||
|
||||
commitServerFileUpload: (name: string, id: string, overwrite: boolean) =>
|
||||
request<{ op: FileOp }>("POST", urlPath`/servers/${name}/files/uploads/${id}/commit`, { overwrite }),
|
||||
|
||||
// unzipServerFile extracts a .zip into the folder holding it, in the
|
||||
// background. Without overwrite an archive that would replace files fails
|
||||
// file_exists and lists them, to be confirmed and run again with overwrite.
|
||||
unzipServerFile: (name: string, path: string, overwrite: boolean) =>
|
||||
request<{ op: FileOp }>(
|
||||
"POST",
|
||||
urlPath`/servers/${name}/files/unzip` + `?path=${encodeURIComponent(path)}`,
|
||||
{ overwrite },
|
||||
),
|
||||
|
||||
// listServerFileOps is the server's running op, if any, and those that ended
|
||||
// within the last 30 minutes, newest first.
|
||||
listServerFileOps: (name: string) =>
|
||||
request<{ ops: FileOp[] }>("GET", urlPath`/servers/${name}/files/ops`),
|
||||
|
||||
// downloadServerFile starts an export of one file, or of a folder as a zip,
|
||||
// and answers its ticket (exportStatus, then exportDownloadURL). The world is
|
||||
// held until the download ends.
|
||||
downloadServerFile: (name: string, path: string, dir: boolean) =>
|
||||
request<ExportTicket>(
|
||||
"POST",
|
||||
urlPath`/servers/${name}/files/download` + `?path=${encodeURIComponent(path)}&dir=${dir}`,
|
||||
),
|
||||
|
||||
// Account linking (spec §10). Both are POST: start reports status from the
|
||||
// session principal (no body, side-effect-free), verify consumes a code the
|
||||
// player was shown in-game. The panel can never mint a code — that is the
|
||||
@@ -1066,10 +975,13 @@ export const api = rejectingSync({
|
||||
createSubmission: (displayName: string) =>
|
||||
request<Submission>("POST", "/me/submissions", { display_name: displayName }),
|
||||
|
||||
uploadSubmissionContext: (id: string, file: Blob) =>
|
||||
requestRaw<Submission>("POST", urlPath`/me/submissions/${id}/context`, file, {
|
||||
"Content-Type": "application/x-gzip",
|
||||
}),
|
||||
|
||||
// A chunked context upload (lib/contextUpload.ts drives it): ask where the
|
||||
// staged upload stands, send each part at its byte offset, then store it. Each
|
||||
// part carries its SHA-256, and one changed on the way is refused
|
||||
// (digest_mismatch) and sent again.
|
||||
// staged upload stands, send each part at its byte offset, then store it.
|
||||
getContextUpload: (id: string) =>
|
||||
request<ContextUploadProgress>("GET", urlPath`/me/submissions/${id}/context/upload`),
|
||||
|
||||
@@ -1079,7 +991,7 @@ export const api = rejectingSync({
|
||||
part: Blob,
|
||||
opts?: { onProgress?: (sent: number) => void; signal?: AbortSignal },
|
||||
) =>
|
||||
sendDigested<ContextUploadProgress>(
|
||||
sendWithProgress<ContextUploadProgress>(
|
||||
"PUT",
|
||||
urlPath`/me/submissions/${id}/context/upload` + `?offset=${offset}`,
|
||||
part,
|
||||
@@ -1395,29 +1307,6 @@ export function humanizeError(e: unknown): string {
|
||||
return t("upload_incomplete");
|
||||
case "length_required":
|
||||
return t("length_required");
|
||||
// Every upload body carries its SHA-256 (sendDigested): bytes that hash
|
||||
// differently on arrival are refused, and a file the browser can no longer
|
||||
// read (changed on disk since it was picked) is never sent.
|
||||
case "digest_mismatch":
|
||||
return t("digest_mismatch");
|
||||
case "digest_required":
|
||||
return t("digest_required");
|
||||
case "bad_digest":
|
||||
return t("bad_digest");
|
||||
case "file_unreadable":
|
||||
return t("file_unreadable");
|
||||
// A file opened in the editor whose bytes arrived damaged (readServerFile).
|
||||
case "read_damaged":
|
||||
return t("read_damaged");
|
||||
// An upload sent in parts: the session is gone (cancelled, landed, idle for
|
||||
// 6 hours, or felis-api restarted), or the account holds four already.
|
||||
case "upload_not_found":
|
||||
return t("upload_not_found");
|
||||
case "too_many_uploads":
|
||||
return t("too_many_uploads");
|
||||
// Client-side: the op being watched dropped out of the ops list.
|
||||
case "op_lost":
|
||||
return t("op_lost");
|
||||
case "jobs_unavailable":
|
||||
return t("jobs_unavailable");
|
||||
// Builds, uploads and review: terminal-state conflicts and unwired subsystems.
|
||||
|
||||
Loaded 100 of 117 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user