feat(files): 大文件分片上传、停服解压 zip 先列冲突再覆盖、文件和文件夹可下载；导出和下载不再带出 RCON 密码与转发密钥
This commit is contained in:
76 files changed
+10542
-569
No files matched your search
+298
-32
@@ -2,13 +2,21 @@ package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
@@ -67,15 +75,51 @@ func tarEntries(t *testing.T, archive []byte) map[string]string {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCmdExportWorld(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.MkdirAll(filepath.Join(root, "world", "region"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, body := range map[string]string{"server.properties": "motd=hi\n", "world/region/r.0.0.mca": "chunks"} {
|
||||
if err := os.WriteFile(filepath.Join(root, name), []byte(body), 0o600); err != nil {
|
||||
// writeTree writes name → body under root, making the folders on the way.
|
||||
func writeTree(t *testing.T, root string, files map[string]string) {
|
||||
t.Helper()
|
||||
for name, body := range files {
|
||||
p := filepath.Join(root, name)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The two secrets a world holds, and what server.properties reads as once
|
||||
// redacted.
|
||||
const (
|
||||
secretProps = "motd=hi\nrcon.password=hunter2\n"
|
||||
redactedProps = "motd=hi\nrcon.password=<redacted by felis>\n"
|
||||
forwardingKey = "secret: aVeryRealForwardingKey\n"
|
||||
)
|
||||
|
||||
// secretWorld is a world holding both secrets, with a hard link to the
|
||||
// forwarding secret under a name nothing would guard by.
|
||||
func secretWorld(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
writeTree(t, root, map[string]string{
|
||||
"server.properties": secretProps,
|
||||
"config/paper-global.yml": forwardingKey,
|
||||
"world/region/r.0.0.mca": "chunks",
|
||||
})
|
||||
if err := os.MkdirAll(filepath.Join(root, "plugins"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Link(filepath.Join(root, "config/paper-global.yml"), filepath.Join(root, "plugins/copy.yml")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func TestCmdExportWorld(t *testing.T) {
|
||||
root := secretWorld(t)
|
||||
if err := os.Symlink("server.properties", filepath.Join(root, "props-link")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rcv := receiveExport(t, noContent)
|
||||
t.Setenv(worldexport.TokenEnv, "tok")
|
||||
@@ -90,61 +134,202 @@ func TestCmdExportWorld(t *testing.T) {
|
||||
r.Header.Get("Content-Type") != "application/gzip" || r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
|
||||
t.Fatalf("request = %s %s, headers %v, length %d, encoding %v", r.Method, r.URL.Path, r.Header, r.ContentLength, r.TransferEncoding)
|
||||
}
|
||||
got := tarEntries(t, rcv.body)
|
||||
want := map[string]string{"server.properties": "motd=hi\n", "world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("archive holds %v, want %v", got, want)
|
||||
want := map[string]string{
|
||||
"server.properties": redactedProps, "config/": "", "plugins/": "",
|
||||
"world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks",
|
||||
}
|
||||
for name, body := range want {
|
||||
if b, ok := got[name]; !ok || b != body {
|
||||
t.Errorf("%s = %q (present %v), want %q", name, b, ok, body)
|
||||
}
|
||||
if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("archive holds %v\nwant %v", got, want)
|
||||
}
|
||||
if !strings.Contains(stdout.String(), "server=survival mode=world downloaded") {
|
||||
t.Errorf("stdout = %q", stdout.String())
|
||||
want2 := "felis export: left out 1 entries a tar cannot hold (symbolic links, devices, sockets)\n" +
|
||||
"felis export: left out 2 files that hold platform secrets\n" +
|
||||
"felis export: server=survival mode=world downloaded\n"
|
||||
if stdout.String() != want2 {
|
||||
t.Errorf("stdout = %q, want %q", stdout.String(), want2)
|
||||
}
|
||||
}
|
||||
|
||||
// A world that cannot be read must never reach felis-api as a complete body:
|
||||
// the chunked upload is cut off, so felis-api aborts the browser's download.
|
||||
func TestCmdExportWorldReadErrorAbortsTheUpload(t *testing.T) {
|
||||
// A world root that cannot be opened fails before anything reaches felis-api.
|
||||
func TestCmdExportWorldUnreadable(t *testing.T) {
|
||||
rcv := receiveExport(t, noContent)
|
||||
t.Setenv(worldexport.TokenEnv, "tok")
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := cmdExport([]string{"--mode", "world", "--target-url", rcv.srv.URL, "--worlds-root", filepath.Join(t.TempDir(), "missing")}, &stdout, &stderr)
|
||||
if code != 1 {
|
||||
t.Fatalf("exit %d, want 1", code)
|
||||
if code != 1 || rcv.hits.Load() != 0 {
|
||||
t.Fatalf("exit %d with %d requests, want 1 and none", code, rcv.hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// An export that fails part-way must never reach felis-api as a complete body:
|
||||
// the chunked upload is cut off, so felis-api aborts the browser's download,
|
||||
// and the failure itself is what the Job reports.
|
||||
func TestStreamExportWriteErrorAbortsTheUpload(t *testing.T) {
|
||||
broken := errors.New("disk read failed")
|
||||
rcv := receiveExport(t, noContent)
|
||||
err := streamExport(context.Background(), rcv.srv.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
|
||||
if _, err := w.Write(bytes.Repeat([]byte("x"), 100_000)); err != nil {
|
||||
return err
|
||||
}
|
||||
return broken
|
||||
})
|
||||
if err != broken {
|
||||
t.Fatalf("err = %v, want the write's own error, unwrapped", err)
|
||||
}
|
||||
select {
|
||||
case <-rcv.served:
|
||||
if rcv.readErr == nil {
|
||||
t.Fatalf("felis-api read a complete %d-byte body from an unreadable world", len(rcv.body))
|
||||
t.Fatalf("felis-api read a complete %d-byte body from a failed export", len(rcv.body))
|
||||
}
|
||||
case <-time.After(2 * time.Second): // the request never reached the handler
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the request never reached felis-api")
|
||||
}
|
||||
}
|
||||
|
||||
// When felis-api refuses first, its reason is reported, not the closed pipe
|
||||
// that then stops the writer.
|
||||
func TestStreamExportRefusalStopsTheWriter(t *testing.T) {
|
||||
refusing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer refusing.Close()
|
||||
stopped := make(chan error, 1)
|
||||
err := streamExport(context.Background(), refusing.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
|
||||
for {
|
||||
if _, err := w.Write(make([]byte, 32<<10)); err != nil {
|
||||
stopped <- err
|
||||
return err
|
||||
}
|
||||
}
|
||||
})
|
||||
if err == nil || err.Error() != "felis-api answered 404 Not Found" {
|
||||
t.Fatalf("err = %v, want felis-api's answer", err)
|
||||
}
|
||||
if werr := <-stopped; !errors.Is(werr, io.ErrClosedPipe) {
|
||||
t.Fatalf("the writer stopped on %v, want the closed pipe", werr)
|
||||
}
|
||||
|
||||
// A PUT that never starts leaves no transport to close the body: the writer
|
||||
// is still stopped, and the export fails rather than hangs.
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- streamExport(context.Background(), "http://[::1", "tok", "application/gzip", -1, func(w io.Writer) error {
|
||||
_, err := w.Write([]byte("x"))
|
||||
return err
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil || !strings.Contains(err.Error(), "missing ']'") {
|
||||
t.Fatalf("err = %v, want the bad URL", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("an export whose PUT never started hung")
|
||||
}
|
||||
}
|
||||
|
||||
// storedBackup writes, at path, a gzip+tar like one the backup store holds:
|
||||
// the world whole, both secrets included, and a region file that does not
|
||||
// compress. It returns the archive's sha256.
|
||||
func storedBackup(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
region := make([]byte, 64<<10)
|
||||
if _, err := rand.Read(region); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
for _, e := range []struct{ name, body string }{
|
||||
{"server.properties", secretProps},
|
||||
{"config/paper-global.yml", forwardingKey},
|
||||
{"world/level.dat", "level"},
|
||||
{"world/region/r.0.0.mca", string(region)},
|
||||
} {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: e.name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(e.body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := io.WriteString(tw, e.body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(buf.Bytes())
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func TestCmdExportBackup(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
archive := bytes.Repeat([]byte("felis"), 10_000)
|
||||
ref := filepath.Join(root, "survival-1.tar.gz")
|
||||
if err := os.WriteFile(ref, archive, 0o600); err != nil {
|
||||
sum := storedBackup(t, ref)
|
||||
stored, err := os.ReadFile(ref)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
region := tarEntries(t, stored)["world/region/r.0.0.mca"]
|
||||
args := func(url, ref string) []string {
|
||||
return []string{"--mode", "backup", "--server", "survival", "--target-url", url, "--ref", ref, "--backup-root", root}
|
||||
}
|
||||
|
||||
t.Run("hands the archive over with its length", func(t *testing.T) {
|
||||
for name, extra := range map[string][]string{
|
||||
"no digest recorded": nil,
|
||||
"recorded digest matches": {"--sha256", sum},
|
||||
} {
|
||||
t.Run(name+": re-streamed through the guards", func(t *testing.T) {
|
||||
rcv := receiveExport(t, noContent)
|
||||
t.Setenv(worldexport.TokenEnv, "tok")
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := cmdExport(append(args(rcv.srv.URL, ref), extra...), &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("exit %d, stderr %q", code, stderr.String())
|
||||
}
|
||||
r := rcv.req
|
||||
if r.ContentLength != -1 || r.Header.Get("Content-Type") != "application/gzip" || r.Header.Get("Authorization") != "Bearer tok" {
|
||||
t.Fatalf("length %d, headers %v", r.ContentLength, r.Header)
|
||||
}
|
||||
want := map[string]string{"server.properties": redactedProps, "world/level.dat": "level", "world/region/r.0.0.mca": region}
|
||||
if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("archive holds %d entries, want exactly the redacted properties, level.dat and the region file", len(got))
|
||||
}
|
||||
if want := "felis export: left out 1 files that hold platform secrets\nfelis export: server=survival mode=backup downloaded\n"; stdout.String() != want {
|
||||
t.Errorf("stdout = %q, want %q", stdout.String(), want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The stored bytes are checked as they stream, and the archive the Job sends
|
||||
// is only closed once they are all read: a mismatch cuts the upload off
|
||||
// short of its end, so felis-api never passes on a complete-looking copy.
|
||||
t.Run("a digest mismatch cuts the upload off before its end", func(t *testing.T) {
|
||||
rcv := receiveExport(t, noContent)
|
||||
t.Setenv(worldexport.TokenEnv, "tok")
|
||||
var stdout, stderr bytes.Buffer
|
||||
if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 0 {
|
||||
t.Fatalf("exit %d, stderr %q", code, stderr.String())
|
||||
if code := cmdExport(append(args(rcv.srv.URL, ref), "--sha256", strings.Repeat("ab", 32)), &stdout, &stderr); code != 1 {
|
||||
t.Fatalf("exit %d, want 1", code)
|
||||
}
|
||||
if rcv.req.ContentLength != int64(len(archive)) || !bytes.Equal(rcv.body, archive) || rcv.req.Header.Get("Authorization") != "Bearer tok" {
|
||||
t.Fatalf("got %d bytes (length %d, auth %q), want the %d archive bytes",
|
||||
len(rcv.body), rcv.req.ContentLength, rcv.req.Header.Get("Authorization"), len(archive))
|
||||
if want := "felis export: the backup archive does not match the sha256 recorded when it was written\n"; stderr.String() != want {
|
||||
t.Fatalf("stderr = %q, want %q", stderr.String(), want)
|
||||
}
|
||||
select {
|
||||
case <-rcv.served:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the upload never reached felis-api")
|
||||
}
|
||||
if rcv.readErr == nil {
|
||||
t.Fatalf("felis-api read a complete %d-byte body", len(rcv.body))
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(rcv.body))
|
||||
if err == nil {
|
||||
_, err = io.ReadAll(zr)
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatal("what felis-api got is a complete archive")
|
||||
}
|
||||
})
|
||||
|
||||
@@ -204,6 +389,86 @@ func TestCmdExportBackup(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCmdExportFiles(t *testing.T) {
|
||||
root := secretWorld(t)
|
||||
writeTree(t, root, map[string]string{"plugins/Essentials/config.yml": "x: 1"})
|
||||
if err := os.Symlink("config.yml", filepath.Join(root, "plugins/Essentials/link.yml")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
export := func(t *testing.T, path string, dir bool) (*exportReceiver, int, string, string) {
|
||||
t.Helper()
|
||||
rcv := receiveExport(t, noContent)
|
||||
t.Setenv(worldexport.TokenEnv, "tok")
|
||||
args := []string{"--mode", "files", "--server", "survival", "--target-url", rcv.srv.URL, "--worlds-root", root, "--path", path}
|
||||
if dir {
|
||||
args = append(args, "--dir")
|
||||
}
|
||||
var stdout, stderr bytes.Buffer
|
||||
code := cmdExport(args, &stdout, &stderr)
|
||||
return rcv, code, stdout.String(), stderr.String()
|
||||
}
|
||||
|
||||
for path, want := range map[string]string{
|
||||
"world/region/r.0.0.mca": "chunks",
|
||||
"server.properties": redactedProps,
|
||||
} {
|
||||
t.Run("a file goes with its exact length: "+path, func(t *testing.T) {
|
||||
rcv, code, stdout, stderr := export(t, path, false)
|
||||
if code != 0 || stdout != "felis export: server=survival mode=files downloaded\n" {
|
||||
t.Fatalf("exit %d, stdout %q, stderr %q", code, stdout, stderr)
|
||||
}
|
||||
if string(rcv.body) != want || rcv.req.ContentLength != int64(len(want)) || rcv.req.Header.Get("Content-Type") != "application/octet-stream" {
|
||||
t.Fatalf("body %q, length %d, type %q; want %q", rcv.body, rcv.req.ContentLength, rcv.req.Header.Get("Content-Type"), want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a folder goes as a zip, guarded", func(t *testing.T) {
|
||||
rcv, code, stdout, stderr := export(t, "plugins", true)
|
||||
if code != 0 {
|
||||
t.Fatalf("exit %d, stderr %q", code, stderr)
|
||||
}
|
||||
if rcv.req.ContentLength != -1 || rcv.req.Header.Get("Content-Type") != "application/zip" {
|
||||
t.Fatalf("length %d, type %q", rcv.req.ContentLength, rcv.req.Header.Get("Content-Type"))
|
||||
}
|
||||
zr, err := zip.NewReader(bytes.NewReader(rcv.body), int64(len(rcv.body)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for _, f := range zr.File {
|
||||
names = append(names, f.Name)
|
||||
}
|
||||
if want := []string{"plugins/", "plugins/Essentials/", "plugins/Essentials/config.yml"}; !slices.Equal(names, want) {
|
||||
t.Fatalf("zip holds %v, want %v", names, want)
|
||||
}
|
||||
want := "felis export: left out 1 entries a zip does not carry (symbolic links, devices, sockets)\n" +
|
||||
"felis export: left out 1 files that hold platform secrets\n" +
|
||||
"felis export: server=survival mode=files downloaded\n"
|
||||
if stdout != want {
|
||||
t.Errorf("stdout = %q, want %q", stdout, want)
|
||||
}
|
||||
})
|
||||
|
||||
for _, c := range []struct {
|
||||
path string
|
||||
dir bool
|
||||
want string
|
||||
}{
|
||||
{"config/paper-global.yml", false, "forwarding secret"},
|
||||
{"plugins/copy.yml", false, "forwarding secret"},
|
||||
{"plugins", false, "is a folder now"},
|
||||
{"server.properties", true, "is not a folder now"},
|
||||
} {
|
||||
t.Run("refused before any request: "+c.path, func(t *testing.T) {
|
||||
rcv, code, _, stderr := export(t, c.path, c.dir)
|
||||
if code != 1 || rcv.hits.Load() != 0 || !strings.Contains(stderr, c.want) {
|
||||
t.Fatalf("exit %d, %d requests, stderr %q; want 1, none, and %q", code, rcv.hits.Load(), stderr, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCmdExportUsage(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
token string
|
||||
@@ -213,6 +478,7 @@ func TestCmdExportUsage(t *testing.T) {
|
||||
"no target": {"tok", []string{"--mode", "world"}},
|
||||
"unknown mode": {"tok", []string{"--mode", "both", "--target-url", "http://api/x"}},
|
||||
"backup without ref": {"tok", []string{"--mode", "backup", "--target-url", "http://api/x"}},
|
||||
"files without path": {"tok", []string{"--mode", "files", "--target-url", "http://api/x"}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Setenv(worldexport.TokenEnv, tc.token)
|
||||
|
||||
Reference in new issue
Block a user