Felis/cmd/felis/export_test.go

package main

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/gzip"
	"context"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"reflect"
	"slices"
	"strconv"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"felis.lolicon.best/internal/config"
	"felis.lolicon.best/internal/worldexport"
)

// exportReceiver stands in for felis-api's internal upload route: it records
// the PUT it gets (or the error reading it ended on) and answers with reply.
type exportReceiver struct {
	srv     *httptest.Server
	hits    atomic.Int32
	req     *http.Request
	body    []byte
	readErr error
	served  chan struct{} // one send per request, once it is answered
}

func receiveExport(t *testing.T, reply func(w http.ResponseWriter)) *exportReceiver {
	t.Helper()
	rcv := &exportReceiver{served: make(chan struct{}, 1)}
	rcv.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		rcv.hits.Add(1)
		rcv.req = r
		rcv.body, rcv.readErr = io.ReadAll(r.Body)
		reply(w)
		rcv.served <- struct{}{}
	}))
	t.Cleanup(rcv.srv.Close)
	return rcv
}

func noContent(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) }

// sentWhole fails unless the upload rcv got ended with the Content-Digest
// trailer of its own bytes, and declared length as its size (-1: none).
func sentWhole(t *testing.T, rcv *exportReceiver, length int64) {
	t.Helper()
	sum := sha256.Sum256(rcv.body)
	want := "sha-256=:" + base64.StdEncoding.EncodeToString(sum[:]) + ":"
	wantLength := ""
	if length >= 0 {
		wantLength = strconv.FormatInt(length, 10)
	}
	r := rcv.req
	if rcv.readErr != nil || r.Trailer.Get(worldexport.DigestTrailer) != want || r.Header.Get(worldexport.LengthHeader) != wantLength ||
		r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
		t.Fatalf("upload read %v, trailer %v, %s %q, length %d, encoding %v; want trailer %q and %s %q, chunked",
			rcv.readErr, r.Trailer, worldexport.LengthHeader, r.Header.Get(worldexport.LengthHeader), r.ContentLength, r.TransferEncoding,
			want, worldexport.LengthHeader, wantLength)
	}
}

func tarEntries(t *testing.T, archive []byte) map[string]string {
	t.Helper()
	gz, err := gzip.NewReader(bytes.NewReader(archive))
	if err != nil {
		t.Fatalf("not gzip: %v", err)
	}
	out := map[string]string{}
	tr := tar.NewReader(gz)
	for {
		h, err := tr.Next()
		if err == io.EOF {
			return out
		}
		if err != nil {
			t.Fatalf("tar: %v", err)
		}
		b, _ := io.ReadAll(tr)
		out[h.Name] = string(b)
	}
}

// writeTree writes name → body under root, making the folders on the way.
func writeTree(t *testing.T, root string, files map[string]string) {
	t.Helper()
	for name, body := range files {
		p := filepath.Join(root, name)
		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
			t.Fatal(err)
		}
	}
}

// The two secrets a world holds, and what server.properties reads as once
// redacted.
const (
	secretProps   = "motd=hi\nrcon.password=hunter2\n"
	redactedProps = "motd=hi\nrcon.password=<redacted by felis>\n"
	forwardingKey = "secret: aVeryRealForwardingKey\n"
)

// secretWorld is a world holding both secrets, with a hard link to the
// forwarding secret under a name nothing would guard by.
func secretWorld(t *testing.T) string {
	t.Helper()
	root := t.TempDir()
	writeTree(t, root, map[string]string{
		"server.properties":       secretProps,
		"config/paper-global.yml": forwardingKey,
		"world/region/r.0.0.mca":  "chunks",
	})
	if err := os.MkdirAll(filepath.Join(root, "plugins"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.Link(filepath.Join(root, "config/paper-global.yml"), filepath.Join(root, "plugins/copy.yml")); err != nil {
		t.Fatal(err)
	}
	return root
}

func TestCmdExportWorld(t *testing.T) {
	root := secretWorld(t)
	if err := os.Symlink("server.properties", filepath.Join(root, "props-link")); err != nil {
		t.Fatal(err)
	}
	rcv := receiveExport(t, noContent)
	t.Setenv(worldexport.TokenEnv, "tok")
	var stdout, stderr bytes.Buffer
	code := cmdExport([]string{"--mode", "world", "--server", "survival", "--target-url", rcv.srv.URL + "/api/v1/internal/exports/ab",
		"--worlds-root", root}, &stdout, &stderr)
	if code != 0 {
		t.Fatalf("exit %d, stderr %q", code, stderr.String())
	}
	r := rcv.req
	if r.Method != http.MethodPut || r.URL.Path != "/api/v1/internal/exports/ab" || r.Header.Get("Authorization") != "Bearer tok" ||
		r.Header.Get("Content-Type") != "application/gzip" || r.ContentLength != -1 || strings.Join(r.TransferEncoding, ",") != "chunked" {
		t.Fatalf("request = %s %s, headers %v, length %d, encoding %v", r.Method, r.URL.Path, r.Header, r.ContentLength, r.TransferEncoding)
	}
	want := map[string]string{
		"server.properties": redactedProps, "config/": "", "plugins/": "",
		"world/": "", "world/region/": "", "world/region/r.0.0.mca": "chunks",
	}
	if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
		t.Fatalf("archive holds %v\nwant %v", got, want)
	}
	sentWhole(t, rcv, -1)
	want2 := "felis export: left out 1 entries a tar cannot hold (symbolic links, devices, sockets)\n" +
		"felis export: left out 2 files that hold platform secrets\n" +
		"felis export: server=survival mode=world downloaded\n"
	if stdout.String() != want2 {
		t.Errorf("stdout = %q, want %q", stdout.String(), want2)
	}
}

// A world root that cannot be opened fails before anything reaches felis-api.
func TestCmdExportWorldUnreadable(t *testing.T) {
	rcv := receiveExport(t, noContent)
	t.Setenv(worldexport.TokenEnv, "tok")
	var stdout, stderr bytes.Buffer
	code := cmdExport([]string{"--mode", "world", "--target-url", rcv.srv.URL, "--worlds-root", filepath.Join(t.TempDir(), "missing")}, &stdout, &stderr)
	if code != 1 || rcv.hits.Load() != 0 {
		t.Fatalf("exit %d with %d requests, want 1 and none", code, rcv.hits.Load())
	}
}

// An export that fails part-way must never reach felis-api as a complete body:
// the chunked upload is cut off, so felis-api aborts the browser's download,
// and the failure itself is what the Job reports.
func TestStreamExportWriteErrorAbortsTheUpload(t *testing.T) {
	broken := errors.New("disk read failed")
	rcv := receiveExport(t, noContent)
	err := streamExport(context.Background(), rcv.srv.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
		if _, err := w.Write(bytes.Repeat([]byte("x"), 100_000)); err != nil {
			return err
		}
		return broken
	})
	if err != broken {
		t.Fatalf("err = %v, want the write's own error, unwrapped", err)
	}
	select {
	case <-rcv.served:
		if rcv.readErr == nil {
			t.Fatalf("felis-api read a complete %d-byte body from a failed export", len(rcv.body))
		}
	case <-time.After(5 * time.Second):
		t.Fatal("the request never reached felis-api")
	}
}

// When felis-api refuses first, its reason is reported, not the closed pipe
// that then stops the writer.
func TestStreamExportRefusalStopsTheWriter(t *testing.T) {
	refusing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusNotFound)
	}))
	defer refusing.Close()
	stopped := make(chan error, 1)
	err := streamExport(context.Background(), refusing.URL, "tok", "application/gzip", -1, func(w io.Writer) error {
		for {
			if _, err := w.Write(make([]byte, 32<<10)); err != nil {
				stopped <- err
				return err
			}
		}
	})
	if err == nil || err.Error() != "felis-api answered 404 Not Found" {
		t.Fatalf("err = %v, want felis-api's answer", err)
	}
	if werr := <-stopped; !errors.Is(werr, io.ErrClosedPipe) {
		t.Fatalf("the writer stopped on %v, want the closed pipe", werr)
	}

	// A PUT that never starts leaves no transport to close the body: the writer
	// is still stopped, and the export fails rather than hangs.
	done := make(chan error, 1)
	go func() {
		done <- streamExport(context.Background(), "http://[::1", "tok", "application/gzip", -1, func(w io.Writer) error {
			_, err := w.Write([]byte("x"))
			return err
		})
	}()
	select {
	case err := <-done:
		if err == nil || !strings.Contains(err.Error(), "missing ']'") {
			t.Fatalf("err = %v, want the bad URL", err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("an export whose PUT never started hung")
	}
}

// storedBackup writes, at path, a gzip+tar like one the backup store holds:
// the world whole, both secrets included, and a region file that does not
// compress. It returns the archive's sha256.
func storedBackup(t *testing.T, path string) string {
	t.Helper()
	region := make([]byte, 64<<10)
	if _, err := rand.Read(region); err != nil {
		t.Fatal(err)
	}
	var buf bytes.Buffer
	zw := gzip.NewWriter(&buf)
	tw := tar.NewWriter(zw)
	for _, e := range []struct{ name, body string }{
		{"server.properties", secretProps},
		{"config/paper-global.yml", forwardingKey},
		{"world/level.dat", "level"},
		{"world/region/r.0.0.mca", string(region)},
	} {
		if err := tw.WriteHeader(&tar.Header{Name: e.name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(e.body))}); err != nil {
			t.Fatal(err)
		}
		if _, err := io.WriteString(tw, e.body); err != nil {
			t.Fatal(err)
		}
	}
	if err := tw.Close(); err != nil {
		t.Fatal(err)
	}
	if err := zw.Close(); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil {
		t.Fatal(err)
	}
	sum := sha256.Sum256(buf.Bytes())
	return hex.EncodeToString(sum[:])
}

func TestCmdExportBackup(t *testing.T) {
	root := t.TempDir()
	ref := filepath.Join(root, "survival-1.tar.gz")
	sum := storedBackup(t, ref)
	stored, err := os.ReadFile(ref)
	if err != nil {
		t.Fatal(err)
	}
	region := tarEntries(t, stored)["world/region/r.0.0.mca"]
	args := func(url, ref string) []string {
		return []string{"--mode", "backup", "--server", "survival", "--target-url", url, "--ref", ref, "--backup-root", root}
	}

	for name, extra := range map[string][]string{
		"no digest recorded":      nil,
		"recorded digest matches": {"--sha256", sum},
	} {
		t.Run(name+": re-streamed through the guards", func(t *testing.T) {
			rcv := receiveExport(t, noContent)
			t.Setenv(worldexport.TokenEnv, "tok")
			var stdout, stderr bytes.Buffer
			if code := cmdExport(append(args(rcv.srv.URL, ref), extra...), &stdout, &stderr); code != 0 {
				t.Fatalf("exit %d, stderr %q", code, stderr.String())
			}
			r := rcv.req
			if r.ContentLength != -1 || r.Header.Get("Content-Type") != "application/gzip" || r.Header.Get("Authorization") != "Bearer tok" {
				t.Fatalf("length %d, headers %v", r.ContentLength, r.Header)
			}
			want := map[string]string{"server.properties": redactedProps, "world/level.dat": "level", "world/region/r.0.0.mca": region}
			if got := tarEntries(t, rcv.body); !reflect.DeepEqual(got, want) {
				t.Fatalf("archive holds %d entries, want exactly the redacted properties, level.dat and the region file", len(got))
			}
			sentWhole(t, rcv, -1)
			if want := "felis export: left out 1 files that hold platform secrets\nfelis export: server=survival mode=backup downloaded\n"; stdout.String() != want {
				t.Errorf("stdout = %q, want %q", stdout.String(), want)
			}
		})
	}

	// The stored bytes are checked as they stream, and the archive the Job sends
	// is only closed once they are all read: a mismatch cuts the upload off
	// short of its end, so felis-api never passes on a complete-looking copy.
	t.Run("a digest mismatch cuts the upload off before its end", func(t *testing.T) {
		rcv := receiveExport(t, noContent)
		t.Setenv(worldexport.TokenEnv, "tok")
		var stdout, stderr bytes.Buffer
		if code := cmdExport(append(args(rcv.srv.URL, ref), "--sha256", strings.Repeat("ab", 32)), &stdout, &stderr); code != 1 {
			t.Fatalf("exit %d, want 1", code)
		}
		if want := "felis export: the backup archive does not match the sha256 recorded when it was written\n"; stderr.String() != want {
			t.Fatalf("stderr = %q, want %q", stderr.String(), want)
		}
		select {
		case <-rcv.served:
		case <-time.After(5 * time.Second):
			t.Fatal("the upload never reached felis-api")
		}
		if rcv.readErr == nil {
			t.Fatalf("felis-api read a complete %d-byte body", len(rcv.body))
		}
		zr, err := gzip.NewReader(bytes.NewReader(rcv.body))
		if err == nil {
			_, err = io.ReadAll(zr)
		}
		if err == nil {
			t.Fatal("what felis-api got is a complete archive")
		}
	})

	t.Run("a ref outside the backup root is refused before any request", func(t *testing.T) {
		outside := filepath.Join(t.TempDir(), "secret.tar.gz")
		if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil {
			t.Fatal(err)
		}
		rcv := receiveExport(t, noContent)
		t.Setenv(worldexport.TokenEnv, "tok")
		var stdout, stderr bytes.Buffer
		if code := cmdExport(args(rcv.srv.URL, outside), &stdout, &stderr); code != 1 {
			t.Fatalf("exit %d, want 1", code)
		}
		if n := rcv.hits.Load(); n != 0 {
			t.Fatalf("felis-api got %d requests", n)
		}
	})

	t.Run("a refusal exits 1 with felis-api's reason", func(t *testing.T) {
		rcv := receiveExport(t, func(w http.ResponseWriter) {
			w.Header().Set("Content-Type", "application/json")
			w.WriteHeader(http.StatusGone)
			io.WriteString(w, `{"error":{"code":"export_expired","message":"nobody opened the download"}}`)
		})
		t.Setenv(worldexport.TokenEnv, "tok")
		var stdout, stderr bytes.Buffer
		if code := cmdExport(args(rcv.srv.URL, ref), &stdout, &stderr); code != 1 {
			t.Fatalf("exit %d, want 1", code)
		}
		if got := stderr.String(); got != "felis export: felis-api answered 410 Gone: nobody opened the download\n" {
			t.Fatalf("stderr = %q", got)
		}
	})

	// The request carries the token and the internal face never redirects, so a
	// redirect is refused rather than followed with the token attached. A 302 or
	// 303 is the one net/http would follow on its own (as a GET, and to the same
	// host with the Authorization header still on it).
	for _, status := range []int{http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect} {
		t.Run("a redirect is not followed: "+strconv.Itoa(status), func(t *testing.T) {
			elsewhere := receiveExport(t, noContent)
			redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.srv.URL, status))
			defer redirecting.Close()
			t.Setenv(worldexport.TokenEnv, "tok")
			var stdout, stderr bytes.Buffer
			if code := cmdExport(args(redirecting.URL, ref), &stdout, &stderr); code != 1 {
				t.Fatalf("exit %d, want 1", code)
			}
			if n := elsewhere.hits.Load(); n != 0 {
				t.Fatalf("the redirect target got %d requests", n)
			}
			if got, want := stderr.String(), "felis export: felis-api answered "+strconv.Itoa(status)+" "+http.StatusText(status)+"\n"; got != want {
				t.Fatalf("stderr = %q, want %q", got, want)
			}
		})
	}
}

func TestCmdExportFiles(t *testing.T) {
	root := secretWorld(t)
	writeTree(t, root, map[string]string{"plugins/Essentials/config.yml": "x: 1"})
	if err := os.Symlink("config.yml", filepath.Join(root, "plugins/Essentials/link.yml")); err != nil {
		t.Fatal(err)
	}
	export := func(t *testing.T, path string, dir bool) (*exportReceiver, int, string, string) {
		t.Helper()
		rcv := receiveExport(t, noContent)
		t.Setenv(worldexport.TokenEnv, "tok")
		args := []string{"--mode", "files", "--server", "survival", "--target-url", rcv.srv.URL, "--worlds-root", root, "--path", path}
		if dir {
			args = append(args, "--dir")
		}
		var stdout, stderr bytes.Buffer
		code := cmdExport(args, &stdout, &stderr)
		return rcv, code, stdout.String(), stderr.String()
	}

	for path, want := range map[string]string{
		"world/region/r.0.0.mca": "chunks",
		"server.properties":      redactedProps,
	} {
		t.Run("a file goes with its exact length: "+path, func(t *testing.T) {
			rcv, code, stdout, stderr := export(t, path, false)
			if code != 0 || stdout != "felis export: server=survival mode=files downloaded\n" {
				t.Fatalf("exit %d, stdout %q, stderr %q", code, stdout, stderr)
			}
			if string(rcv.body) != want || rcv.req.Header.Get("Content-Type") != "application/octet-stream" {
				t.Fatalf("body %q, type %q; want %q", rcv.body, rcv.req.Header.Get("Content-Type"), want)
			}
			sentWhole(t, rcv, int64(len(want)))
		})
	}

	t.Run("a folder goes as a zip, guarded", func(t *testing.T) {
		rcv, code, stdout, stderr := export(t, "plugins", true)
		if code != 0 {
			t.Fatalf("exit %d, stderr %q", code, stderr)
		}
		if rcv.req.ContentLength != -1 || rcv.req.Header.Get("Content-Type") != "application/zip" {
			t.Fatalf("length %d, type %q", rcv.req.ContentLength, rcv.req.Header.Get("Content-Type"))
		}
		zr, err := zip.NewReader(bytes.NewReader(rcv.body), int64(len(rcv.body)))
		if err != nil {
			t.Fatal(err)
		}
		var names []string
		for _, f := range zr.File {
			names = append(names, f.Name)
		}
		if want := []string{"plugins/", "plugins/Essentials/", "plugins/Essentials/config.yml"}; !slices.Equal(names, want) {
			t.Fatalf("zip holds %v, want %v", names, want)
		}
		want := "felis export: left out 1 entries a zip does not carry (symbolic links, devices, sockets)\n" +
			"felis export: left out 1 files that hold platform secrets\n" +
			"felis export: server=survival mode=files downloaded\n"
		if stdout != want {
			t.Errorf("stdout = %q, want %q", stdout, want)
		}
	})

	for _, c := range []struct {
		path string
		dir  bool
		want string
	}{
		{"config/paper-global.yml", false, "forwarding secret"},
		{"plugins/copy.yml", false, "forwarding secret"},
		{"plugins", false, "is a folder now"},
		{"server.properties", true, "is not a folder now"},
	} {
		t.Run("refused before any request: "+c.path, func(t *testing.T) {
			rcv, code, _, stderr := export(t, c.path, c.dir)
			if code != 1 || rcv.hits.Load() != 0 || !strings.Contains(stderr, c.want) {
				t.Fatalf("exit %d, %d requests, stderr %q; want 1, none, and %q", code, rcv.hits.Load(), stderr, c.want)
			}
		})
	}
}

func TestCmdExportUsage(t *testing.T) {
	for name, tc := range map[string]struct {
		token string
		args  []string
	}{
		"no token":           {"", []string{"--mode", "world", "--target-url", "http://api/x"}},
		"no target":          {"tok", []string{"--mode", "world"}},
		"unknown mode":       {"tok", []string{"--mode", "both", "--target-url", "http://api/x"}},
		"backup without ref": {"tok", []string{"--mode", "backup", "--target-url", "http://api/x"}},
		"files without path": {"tok", []string{"--mode", "files", "--target-url", "http://api/x"}},
	} {
		t.Run(name, func(t *testing.T) {
			t.Setenv(worldexport.TokenEnv, tc.token)
			var stdout, stderr bytes.Buffer
			if code := cmdExport(tc.args, &stdout, &stderr); code != 2 {
				t.Fatalf("exit %d, want 2; stderr %q", code, stderr.String())
			}
		})
	}
}

// TestCmdExportWiring: the Job's `felis export` reaches cmdExport, and
// felis-api's executor mounts the backup store at the path the archives were
// written under, since a backup's ref is an absolute path there.
func TestCmdExportWiring(t *testing.T) {
	t.Setenv(worldexport.TokenEnv, "")
	var stdout, stderr bytes.Buffer
	if code := run([]string{"export"}, &stdout, &stderr); code != 2 ||
		stderr.String() != "felis export: --target-url and "+worldexport.TokenEnv+" are required\n" {
		t.Fatalf("felis export = %d, stderr %q", code, stderr.String())
	}
	cfg := &config.Config{}
	cfg.K8s.Namespace, cfg.Archive.LocalPath = "games", "/srv/felis-backups"
	want := worldexport.Config{Namespace: "games", Image: "felis:1", BackupPVC: "felis-backups", BackupRoot: "/srv/felis-backups"}
	if got := exportConfig(cfg, "felis:1", "felis-backups"); got != want {
		t.Fatalf("exportConfig = %+v, want %+v", got, want)
	}
}