Unverified Commit 051cc1c9 authored by Lemon-miaow's avatar Lemon-miaow
Browse files

feat(files): 文件管理可新建、建目录、删除、重命名和上传,写入内容拆成多个环境变量不再超内核单变量上限

parent b6751eac
Loading
Loading
Loading
Loading
+25 −0
Changes for cmd/felis/api.go: 25 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -300,12 +300,22 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
	// endpoints honestly return 503. It takes the typed clientset rather than the
	// controller-runtime client because the log subresource lives only on the typed
	// CoreV1 client, and one client covers its Job create, Pod list, and log read.
	//
	// Uploads additionally stage their bytes on this pod's disk until the Job
	// fetches them from the internal face; whatever a previous process staged is
	// orphaned (the index is in memory), so the stage starts empty.
	var files api.FileEditor
	var fileStage *fileedit.Stage
	if felisImage != "" {
		files = &fileedit.Editor{
			Runner: fileedit.NewK8sRunner(clientset),
			Config: fileEditConfig(cfg, felisImage),
		}
		fileStage = &fileedit.Stage{Dir: fileStagingDir()}
		if err := fileStage.Sweep(); err != nil {
			fmt.Fprintf(stderr, "felis api: %v — file uploads return 503\n", err)
			fileStage = nil
		}
	} else {
		fmt.Fprintln(stderr, "felis api: file editor disabled (needs FELIS_IMAGE) — file endpoints return 503")
	}
@@ -353,6 +363,11 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
		// restore behind each one.
		RestoreChains: jobStatus,
		Files:         files,
		FileStage:     fileStage,
		// The file Job fetches an upload from here; it runs in the minecraft
		// namespace, where the internal face is reachable like it is for the login
		// gate.
		InternalBaseURL: internalAPIBaseURL(),
		Submissions:     submissions,
		Mailer:          mailer,
		// The external face authenticates the local session cookie the sign-in doors
@@ -947,6 +962,16 @@ func uploadPartsDir(contextBase string) string {
	return filepath.Join(os.TempDir(), "felis-upload-parts")
}

// fileStagingDir is where file uploads wait for their Job: on the uploads
// volume, whose capacity is its own, or the pod's /tmp when run by hand without
// it — /tmp is the node's disk, which a burst of uploads should not fill.
func fileStagingDir() string {
	if fi, err := os.Stat(platform.UploadsLocalPath); err == nil && fi.IsDir() {
		return filepath.Join(platform.UploadsLocalPath, ".file-staging")
	}
	return filepath.Join(os.TempDir(), "felis-file-staging")
}

// contextMaxBytes resolves [registry] context_max_bytes. 0 keeps the submit
// package's own default (1 GiB). The Cloudflare edge refuses a single request
// body over 100 MB, which the panel's chunked upload stays under, so the edge
+67 −19
Changes for cmd/felis/files.go: 67 added lines, 19 removed lines.
Original line number Diff line number Diff line
package main

import (
	"encoding/base64"
	"context"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"felis.lolicon.best/internal/fileedit"
)
@@ -20,32 +24,42 @@ import (
// config.Load: felis-api made the authorization decision (the caller owns this
// server, and the server is stopped so the RWO world volume is free); this process
// is the unprivileged hands that touch bytes. Its entire input is the flags
// below plus, for a write, one environment variable. Every isolation guarantee
// lives in the Pod spec (internal/fileedit/jobspec.go), and the path-containment
// guarantee lives in fileedit.Execute, which resolves the path through os.Root and
// therefore cannot be walked out of the world mount.
// below plus, for a write, the content variables and, for an upload, one token.
// Every isolation guarantee lives in the Pod spec (internal/fileedit/jobspec.go),
// and the path-containment guarantee lives in fileedit.Execute, which resolves
// every path through os.Root and therefore cannot be walked out of the world
// mount.
//
// Exit status carries a specific meaning that felis-api depends on: a CALLER-fault
// outcome — a path that escapes the root, a file that is missing or too large — is
// a SUCCESSFUL run that prints a Result carrying an error code, so the API can map
// it to a precise 4xx. A non-zero exit means the operation could not be attempted
// at all (the world mount is unreadable, the result unprintable), which the API
// reports as a 500.
// at all (the world mount is unreadable, an upload's bytes could not be fetched
// intact, the result unprintable), which the API reports as a 500.
func cmdFiles(args []string, stdout, stderr io.Writer) int {
	fs := flag.NewFlagSet("files", flag.ContinueOnError)
	fs.SetOutput(stderr)
	op := fs.String("op", "", "operation: list, read, or write")
	op := fs.String("op", "", "operation: list, read, write, mkdir, delete, rename or upload")
	path := fs.String("path", "", "path to operate on, relative to the world root (empty = the root itself)")
	worldsRoot := fs.String("worlds-root", "/data", "mount path of the world PVC; every path resolves under it")
	expect := fs.String("expect-sha256", "", "write only: refuse unless the file's current SHA-256 (hex) is this")
	createOnly := fs.Bool("create-only", false, "write only: refuse a path that already exists")
	to := fs.String("to", "", "rename only: the destination path")
	sourceURL := fs.String("source-url", "", "upload only: felis-api URL to fetch the bytes from")
	size := fs.Int64("size", -1, "upload only: the byte count the fetched file must have")
	sum := fs.String("sha256", "", "upload only: the SHA-256 (hex) the fetched file must have")
	overwrite := fs.Bool("overwrite", false, "upload only: replace a file already at the path")
	if err := fs.Parse(args); err != nil {
		return 2
	}

	if *op == "" {
		fmt.Fprintln(stderr, "felis files: --op is required (list, read, or write)")
		fmt.Fprintln(stderr, "felis files: --op is required")
		return 2
	}
	req := fileedit.Request{
		Op: *op, Path: *path, To: *to, Expect: *expect, CreateOnly: *createOnly, Overwrite: *overwrite,
	}

	// New content arrives base64-encoded in the environment rather than in argv:
	// a process's arguments are world-readable on the node (/proc/<pid>/cmdline),
@@ -53,22 +67,29 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
	// secrets — an RCON password in server.properties is the obvious case. The
	// encoding is what lets arbitrary bytes (CRLF endings, a BOM, a NUL) survive a
	// channel that must be a valid string.
	var content []byte
	if *op == fileedit.OpWrite {
		raw, ok := os.LookupEnv(fileedit.ContentEnv)
		if !ok {
			fmt.Fprintf(stderr, "felis files: a write needs %s in the environment\n", fileedit.ContentEnv)
	switch *op {
	case fileedit.OpWrite:
		content, err := fileedit.ContentFromEnv(os.LookupEnv)
		if err != nil {
			fmt.Fprintf(stderr, "felis files: %v\n", err)
			return 2
		}
		decoded, err := base64.StdEncoding.DecodeString(raw)
		if err != nil {
			fmt.Fprintf(stderr, "felis files: %s is not valid base64: %v\n", fileedit.ContentEnv, err)
		req.Content = content
	case fileedit.OpUpload:
		token := os.Getenv(fileedit.UploadTokenEnv)
		if *sourceURL == "" || token == "" {
			fmt.Fprintf(stderr, "felis files: an upload needs --source-url and %s\n", fileedit.UploadTokenEnv)
			return 2
		}
		content = decoded
		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
		defer stop()
		req.Upload = &fileedit.Upload{
			Size: *size, SHA256: *sum,
			Open: func() (io.ReadCloser, error) { return fetchUpload(ctx, *sourceURL, token) },
		}
	}

	res, err := fileedit.Execute(*worldsRoot, *op, *path, content, *expect)
	res, err := fileedit.Execute(*worldsRoot, req)
	if err != nil {
		// The operation could not be attempted — infrastructure, not caller fault.
		fmt.Fprintf(stderr, "felis files: %v\n", err)
@@ -83,3 +104,30 @@ func cmdFiles(args []string, stdout, stderr io.Writer) int {
	}
	return 0
}

// fetchUpload opens the staged upload on felis-api's internal face. There is no
// retry: the token opens the upload once (fileedit.Stage), so a second attempt
// could only be refused, and felis-api answers the failed Job with a 500 the
// caller can retry whole. Redirects are refused because the request carries the
// token and the internal face never redirects; the header timeout catches a
// wedged endpoint, and the Job's activeDeadlineSeconds bounds the body.
func fetchUpload(ctx context.Context, url, token string) (io.ReadCloser, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Authorization", "Bearer "+token)
	client := &http.Client{
		Transport:     &http.Transport{ResponseHeaderTimeout: 30 * time.Second},
		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		resp.Body.Close()
		return nil, fmt.Errorf("GET returned %s", resp.Status)
	}
	return resp.Body, nil
}
+203 −0
Changes for cmd/felis/files_test.go: 203 added lines, 0 removed lines.
Original line number Diff line number Diff line
package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"testing"

	"felis.lolicon.best/internal/fileedit"
)

// filesResult is the Result a `felis files` run printed on its marked line.
func filesResult(t *testing.T, stdout string) fileedit.Result {
	t.Helper()
	line, ok := strings.CutPrefix(strings.TrimSpace(stdout), fileedit.ResultPrefix)
	if !ok {
		t.Fatalf("stdout has no result line: %q", stdout)
	}
	var res fileedit.Result
	if err := json.Unmarshal([]byte(line), &res); err != nil {
		t.Fatalf("result line %q: %v", line, err)
	}
	return res
}

// stagedUpload serves body to a request carrying Bearer token, and 404 to any
// other, the way felis-api's internal face does.
func stagedUpload(t *testing.T, token string, body []byte) *httptest.Server {
	t.Helper()
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Authorization") != "Bearer "+token {
			http.Error(w, "no such upload", http.StatusNotFound)
			return
		}
		w.Write(body)
	}))
	t.Cleanup(srv.Close)
	return srv
}

func uploadArgs(root, sourceURL string, body []byte) []string {
	sum := sha256.Sum256(body)
	return []string{
		"--op", "upload", "--path", "plugins/a.jar", "--worlds-root", root,
		"--source-url", sourceURL, "--size", "4", "--sha256", hex.EncodeToString(sum[:]),
	}
}

// uploadRoot is a world with the plugins folder an upload lands in.
func uploadRoot(t *testing.T) string {
	t.Helper()
	root := t.TempDir()
	if err := os.Mkdir(filepath.Join(root, "plugins"), 0o755); err != nil {
		t.Fatal(err)
	}
	return root
}

func TestCmdFilesUpload(t *testing.T) {
	body := []byte("PK\x03\x04")

	t.Run("fetches the staged bytes with its token and lands them", func(t *testing.T) {
		root := uploadRoot(t)
		srv := stagedUpload(t, "tok", body)
		t.Setenv(fileedit.UploadTokenEnv, "tok")
		var stdout, stderr bytes.Buffer
		if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 0 {
			t.Fatalf("exit %d, stderr %q", code, stderr.String())
		}
		if res := filesResult(t, stdout.String()); res.Code != "" {
			t.Fatalf("result = %+v", res)
		}
		got, err := os.ReadFile(filepath.Join(root, "plugins", "a.jar"))
		if err != nil || !bytes.Equal(got, body) {
			t.Fatalf("landed %q, %v", got, err)
		}
	})

	// A refused fetch is the Job failing, never a Result: the API answers it with a
	// 500 the caller retries whole.
	t.Run("a refused fetch exits 1 and lands nothing", func(t *testing.T) {
		root := uploadRoot(t)
		srv := stagedUpload(t, "tok", body)
		t.Setenv(fileedit.UploadTokenEnv, "wrong")
		var stdout, stderr bytes.Buffer
		if code := cmdFiles(uploadArgs(root, srv.URL+"/u", body), &stdout, &stderr); code != 1 {
			t.Fatalf("exit %d, want 1; stdout %q", code, stdout.String())
		}
		if !strings.Contains(stderr.String(), "404") {
			t.Fatalf("stderr %q does not name the status", stderr.String())
		}
		if _, err := os.Lstat(filepath.Join(root, "plugins", "a.jar")); !os.IsNotExist(err) {
			t.Fatalf("a refused fetch left a file: %v", err)
		}
	})

	// The request carries the token, and the internal face never redirects, so a
	// redirect is refused rather than followed with the token attached.
	t.Run("a redirect is not followed", func(t *testing.T) {
		root := uploadRoot(t)
		var hits atomic.Int32
		elsewhere := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			hits.Add(1)
			w.Write(body)
		}))
		defer elsewhere.Close()
		redirecting := httptest.NewServer(http.RedirectHandler(elsewhere.URL+"/u", http.StatusFound))
		defer redirecting.Close()
		t.Setenv(fileedit.UploadTokenEnv, "tok")
		var stdout, stderr bytes.Buffer
		if code := cmdFiles(uploadArgs(root, redirecting.URL+"/u", body), &stdout, &stderr); code != 1 {
			t.Fatalf("exit %d, want 1", code)
		}
		if n := hits.Load(); n != 0 {
			t.Fatalf("the redirect target was fetched %d times", n)
		}
	})

	for name, tc := range map[string]struct {
		token string
		drop  string
	}{
		"no token":      {"", ""},
		"no source URL": {"tok", "--source-url"},
	} {
		t.Run(name+" exits 2", func(t *testing.T) {
			srv := stagedUpload(t, "tok", body)
			t.Setenv(fileedit.UploadTokenEnv, tc.token)
			args := uploadArgs(uploadRoot(t), srv.URL+"/u", body)
			if tc.drop != "" {
				for i, a := range args {
					if a == tc.drop {
						args = append(args[:i:i], args[i+2:]...)
						break
					}
				}
			}
			var stdout, stderr bytes.Buffer
			if code := cmdFiles(args, &stdout, &stderr); code != 2 {
				t.Fatalf("exit %d, want 2", code)
			}
		})
	}
}

func TestCmdFilesWrite(t *testing.T) {
	root := t.TempDir()
	args := []string{"--op", "write", "--path", "ops.json", "--worlds-root", root}

	t.Run("reassembles the content parts", func(t *testing.T) {
		content := []byte("[]\r\n")
		t.Setenv(fileedit.ContentPartsEnv, "1")
		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString(content))
		var stdout, stderr bytes.Buffer
		if code := cmdFiles(args, &stdout, &stderr); code != 0 {
			t.Fatalf("exit %d, stderr %q", code, stderr.String())
		}
		if res := filesResult(t, stdout.String()); res.Code != "" {
			t.Fatalf("result = %+v", res)
		}
		if got, err := os.ReadFile(filepath.Join(root, "ops.json")); err != nil || !bytes.Equal(got, content) {
			t.Fatalf("wrote %q, %v", got, err)
		}
	})

	// Writing what did arrive of an incomplete spec would truncate the file.
	t.Run("an incomplete content spec exits 2 and writes nothing", func(t *testing.T) {
		t.Setenv(fileedit.ContentPartsEnv, "2")
		t.Setenv(fileedit.ContentEnv+"_0", base64.StdEncoding.EncodeToString([]byte("x")))
		var stdout, stderr bytes.Buffer
		if code := cmdFiles([]string{"--op", "write", "--path", "new.txt", "--worlds-root", root}, &stdout, &stderr); code != 2 {
			t.Fatalf("exit %d, want 2", code)
		}
		if _, err := os.Lstat(filepath.Join(root, "new.txt")); !os.IsNotExist(err) {
			t.Fatalf("an incomplete spec wrote a file: %v", err)
		}
	})
}

// A caller-fault outcome is a successful run carrying a code, so felis-api can
// answer the precise 4xx instead of a 500.
func TestCmdFilesCallerFaultIsAResult(t *testing.T) {
	root := t.TempDir()
	var stdout, stderr bytes.Buffer
	if code := cmdFiles([]string{"--op", "mkdir", "--path", "../out", "--worlds-root", root}, &stdout, &stderr); code != 0 {
		t.Fatalf("exit %d, stderr %q", code, stderr.String())
	}
	if res := filesResult(t, stdout.String()); res.Code != fileedit.CodeBadPath {
		t.Fatalf("result = %+v, want code %s", res, fileedit.CodeBadPath)
	}
	stdout.Reset()
	if code := cmdFiles([]string{"--worlds-root", root}, &stdout, &stderr); code != 2 {
		t.Fatalf("no --op: exit %d, want 2", code)
	}
}
+323 −1

File changed.

Preview size limit exceeded, changes collapsed.

+77 −7
Changes for docs/troubleshooting.md: 77 added lines, 7 removed lines.
Original line number Diff line number Diff line
@@ -220,7 +220,7 @@ per-server cooldown → global running cap**. Map the API result:
| HTTP | Code | Cause | Fix |
|---|---|---|---|
| `403` | `forbidden` | `autostartPolicy=allowlist` and UUID not allowlisted, or `ownerOnly` and caller is not owner | Add the UUID / claim the server / set `autostartPolicy=public` |
| `409` | `maintenance_in_progress` | A restore, backup or file write holds the server's world volume (§3b) | Wait for the Job to finish |
| `409` | `maintenance_in_progress` | A restore, backup or file change holds the server's world volume (§3b) | Wait for the Job to finish |
| `409` | `world_reclaiming` | The idle reaper is archiving the world (§3b item 3); afterwards the server is released with an empty world | Nothing to wait for; the old world stays in the archive |
| `429` | (cooldown) | Wake retried within the 30s per-server `WakeCooldown` | Wait out the cooldown |
| `503` | `at_capacity` | Global `MaxRunningServers` cap reached | Stop another server or raise the cap |
@@ -238,11 +238,11 @@ shortly.") lives in the Java plugin and is **[CODE-ONLY]** — the codes it reac
to are produced by the Go-tested `authorizeWakeByUUID` / cooldown limiter, so
grade the two halves separately.

### 3b. Wake, restore, backup or file save refused with `maintenance_in_progress`
### 3b. Wake, restore, backup or file change refused with `maintenance_in_progress`

A server's world volume is ReadWriteOnce, and on a single node RWO lets a game
pod and a restore Job mount it side by side. So felis-api serialises them per
server: a restore, a backup, or a file write takes the world, and until its Job
server: a restore, a backup, or a file change takes the world, and until its Job
finishes every wake (panel or join) and every other world operation on that
server gets `409 maintenance_in_progress`. File reads and listings never hold
it. The operator applies the same rule when `desiredState` is flipped to
@@ -253,7 +253,8 @@ What holds the world, in order:

1. An unfinished Job labelled `felis.lolicon.best/server=<name>` with
   `app.kubernetes.io/managed-by` `felis-restore`, `felis-backup`, or
   `felis-files` plus `felis.lolicon.best/files-mode=write`:
   `felis-files` with any `felis.lolicon.best/files-mode` but `list` or `read`
   (a save, new file, new folder, rename, delete or upload; §18):

   ```sh
   kubectl -n minecraft get jobs -l felis.lolicon.best/server=<name>
@@ -281,7 +282,7 @@ What holds the world, in order:
   the lock before it deletes the world volume, keeps the world and retries the
   next day.

A restore, backup or file write refused with `409 not_stopped` although the
A restore, backup or file change refused with `409 not_stopped` although the
panel shows `Stopped` means the game pod is still terminating (its shutdown save
can take a while); retry once `kubectl -n minecraft get pods -l
felis.lolicon.best/server=<name>` shows nothing.
@@ -465,7 +466,7 @@ installer (`sudo bash deploy/bootstrap.sh`) puts the value everywhere. [GO-TESTE
  stale. A proxy on another host is left alone: set `service-token` in its
  `felis-link.properties` to the value in Secret `felis/felis-service-token`, and
  it takes it within a few seconds.
- **forwarding** — a server whose world a backup, restore or file write holds is
- **forwarding** — a server whose world a backup, restore or file change holds is
  left running and named in the plan and the output; players cannot join it until
  it restarts, so stop and start it from the panel once that finishes. The next
  installer run restarts the proxy once more (its record of what the proxy was
@@ -970,7 +971,7 @@ The reap sequence (all [GO-TESTED] hermetically) preserves the world unless a
0. The world must be at rest before it is archived. A server still meant to
   run is told to stop (`desiredState: Stopped`) and left for the next run; one
   still stopping, whose game pod still exists, or whose world a restore,
   backup or file write holds is left too. Each of these counts in
   backup or file change holds is left too. Each of these counts in
   `awaiting_stop=` and does not fail the run. Once the server is down, the
   reaper takes the world's maintenance lock (§3b) and holds it through the
   archive and the volume delete: nothing can start the server or touch its
@@ -2990,6 +2991,74 @@ for 10 seconds (the Free plan's limits).

---

## 18. Server files: a change or an upload is refused

The panel's Files page is for the server's owner or an admin, and only while
the server is fully stopped. Each call runs a one-shot `felis files` Job in the
`minecraft` namespace, labelled `app.kubernetes.io/managed-by=felis-files` and
`felis.lolicon.best/files-mode=<list|read|write|mkdir|delete|rename|upload>`.
A listing or a read holds nothing. Every change (a save, a new file or folder,
a rename, a delete, an upload) holds the world for its Job (§3b), so a wake or a
second change in the meantime gets `409 maintenance_in_progress`. The panel
sends uploads one at a time and greys its other changes until they finish.

| Status | Code | Meaning | What to do |
|---|---|---|---|
| `409` | `not_stopped` | The game pod is still there, usually finishing its shutdown save | Retry once `kubectl -n minecraft get pods -l felis.lolicon.best/server=<name>` shows nothing |
| `409` | `maintenance_in_progress` | Another change, a backup, a restore or the reaper holds the world | §3b |
| `409` | `file_exists` | A new file, new folder, rename, or upload sent without replace found something at the path | Pick another name or clear the path; the panel offers Replace for an upload |
| `409` | `file_changed` | The file changed after the editor read it | The editor offers to load the latest or overwrite it |
| `400` | `bad_path` | The path leaves the world volume (`..`, an absolute path, a link pointing out), or it would move `server.properties`, `config` or `config/paper-global.yml`, or read `config/paper-global.yml` | Those three keep their names: the read path withholds their secrets by name, and `paper-global.yml` holds the proxy forwarding secret every server shares |
| `404` | `not_found` | The path, or a new folder's parent, is gone | Refresh the listing |
| `413` | `too_large` | A read over 1 MiB, a save over 256 KiB, or an upload over 64 MiB | Upload a large file whole instead of editing it |
| `411` | `length_required` | An upload without `Content-Length` (a chunked body) | Upload from the panel, or with `curl -T`, which sends the length |
| `400` | `upload_incomplete` | The body ended before its declared length | Retry; nothing was changed |
| `507` | `upload_staging_full` | Staging this upload would leave felis-api's staging filesystem under 10% free | Free space on the uploads volume |
| `507` | `volume_full` | The world volume ran out of space; the old file is left as it was | Delete files the server no longer needs, or grow its volume |
| `504` | `files_timeout` | felis-api stopped waiting after 90 s | See below: the Job may still finish |
| `503` | `files_unavailable` | felis-api runs without the file Job runner, or (for an upload) without a staging directory or its internal address | Check felis-api's startup log |

**An upload travels in two legs.** The browser sends the body to felis-api,
which stages it under `/var/lib/felis/uploads/.file-staging` on the uploads
volume (`felis-file-staging` in the pod's temp directory when that volume is not
mounted). The Job then fetches it once from felis-api's internal face,
`http://felis-api-internal.felis.svc.cluster.local:8081/api/v1/internal/file-uploads/<id>`,
with a one-time token, checks the size and SHA-256, and lands it. The staged
copy is deleted once the Job has answered, and a felis-api restart empties the
directory. A Job that cannot fetch its upload, or fetches bytes that do not
match, fails and leaves the target as it was; the panel shows a server error
it can retry. The Job's log names the cause:

```sh
kubectl -n minecraft get jobs -l felis.lolicon.best/server=<name>,app.kubernetes.io/managed-by=felis-files
kubectl -n minecraft logs job/<job>
```

**After `files_timeout`** the Job runs on to its own two-minute deadline and may
still land the change. It keeps holding the world until it ends, so the next
change waits on it with `maintenance_in_progress`; refresh the listing once it
is gone to see whether the change landed. A Job is kept for two minutes after
it ends, with its log.

Every change is audited as `file.write`, `file.mkdir`, `file.delete`,
`file.rename` (with `to`) or `file.upload` (with `size_bytes`, `sha256` and
`overwrite`), with `server_name` set to `<server>:<path>`:

```sh
sudo k3s kubectl -n felis exec deploy/felis-postgres -c postgres -- psql -U postgres felis -c "
  SELECT created_at, actor, action, server_name, payload
  FROM audit_logs WHERE action LIKE 'file.%' ORDER BY created_at DESC LIMIT 20;"
```

[GO-TESTED: `internal/fileedit`, `handlers_files_test.go`, `cmd/felis/files_test.go`,
`internal/maintenance`.] [VM-TESTED: a pod labelled as a files Job in `minecraft`
reaches `felis-api-internal:8081`; a 256 KiB save's content, split across six
variables, lands byte for byte through the real binary, where one 140 KB variable
fails with `argument list too long`. An upload through the API, both legs end to
end, has not been run on a cluster.]

---

## Quick reference: symptom → section

| Symptom | Section |
@@ -3033,3 +3102,4 @@ for 10 seconds (the Free plan's limits).
| `FelisAuditWriteFailing` | §17 |
| `felis breakGlass` sends no code / shows `Root override`; `otp_skipped` in the audit | §17 |
| How long sessions, codes and audit rows are kept; export audit rows | §17 |
| Files page: a change or upload refused (`file_exists`, `bad_path`, `too_large`, `upload_staging_full`, `volume_full`, `files_timeout`) | §18 |
Loading