feat(offsite): 异地副本加入提交上传的整合包，按内容去重加密，索引按版本保留 14 天，新增 fetch-uploads 恢复

internal/offsite/images.go

@@ -455,19 +455,7 @@ func sameImages(a, b *ImageIndex) bool {
 // pruneImages drops the index versions replaced more than ImageHistory ago,
 // then every blob and manifest no remaining version names.
 func (s *Syncer) pruneImages(ctx context.Context, versions []string, newest *ImageIndex, remote map[string]int64, res *Result, fail func(string, ...any)) {
-	cutoff := s.now().Add(-ImageHistory)
-	var keep, drop []string
-	for i, v := range versions {
-		if i == len(versions)-1 {
-			break
-		}
-		replaced, err := time.Parse(imageStampLayout, versions[i+1])
-		if err == nil && replaced.Before(cutoff) {
-			drop = append(drop, v)
-		} else {
-			keep = append(keep, v)
-		}
-	}
+	keep, drop := retire(versions, s.now().Add(-ImageHistory))
 	live := map[string]bool{}
 	mark := func(x *ImageIndex) {
 		for d, m := range x.Manifests {
@@ -557,7 +545,7 @@ func (d *digestReader) Read(p []byte) (int, error) {
 	d.h.Write(p[:n])
 	d.n += int64(n)
 	if d.n > d.size {
-		return n, fmt.Errorf("blob %s is longer than the %d bytes its manifest records", d.digest, d.size)
+		return n, fmt.Errorf("blob %s is longer than the %d bytes recorded for it", d.digest, d.size)
 	}
 	if err == io.EOF {
 		if d.n != d.size {
@@ -570,11 +558,32 @@ func (d *digestReader) Read(p []byte) (int, error) {
 	return n, err
 }
 
-// imageIndexStamps lists the index versions among keys, oldest first.
-func imageIndexStamps(keys map[string]int64) []string {
+// retire splits the index versions before the newest (oldest first) into the
+// ones still kept and the ones replaced before cutoff. The newest is in
+// neither: it is the state the pass just recorded.
+func retire(versions []string, cutoff time.Time) (keep, drop []string) {
+	for i, v := range versions {
+		if i == len(versions)-1 {
+			break
+		}
+		replaced, err := time.Parse(imageStampLayout, versions[i+1])
+		if err == nil && replaced.Before(cutoff) {
+			drop = append(drop, v)
+		} else {
+			keep = append(keep, v)
+		}
+	}
+	return keep, drop
+}
+
+// imageIndexStamps lists the registry index versions among keys, oldest first.
+func imageIndexStamps(keys map[string]int64) []string { return indexStamps(keys, imageIndexDir) }
+
+// indexStamps lists the index versions under dir among keys, oldest first.
+func indexStamps(keys map[string]int64, dir string) []string {
 	var out []string
 	for key := range keys {
-		stamp, ok := strings.CutPrefix(key, imageIndexDir)
+		stamp, ok := strings.CutPrefix(key, dir)
 		if !ok {
 			continue
 		}

internal/offsite/sync.go

@@ -1,7 +1,8 @@
 // Package offsite keeps a second copy of what a lost node would take with it:
 // every world archive (world_backups), the newest control-plane database
-// bundles (internal/dbbackup) and the user images in the platform registry
-// (images.go), encrypted, in an S3-compatible bucket off the machine. `felis
+// bundles (internal/dbbackup), the user images in the platform registry
+// (images.go) and the submission uploads (uploads.go), encrypted, in an
+// S3-compatible bucket off the machine. `felis
 // offsite sync` runs it from felis-offsite.timer on the host, which is where
 // the archive volume and the bundle directory live and where the registry
 // answers on its loopback hostPort.
@@ -94,8 +95,11 @@ type Syncer struct {
 	// Images is the platform registry whose user images are copied (images.go);
 	// nil copies none.
 	Images ImageSource
-	Now    func() time.Time
-	Log    io.Writer
+	// UploadsDir is the host directory of the uploads volume, whose submission
+	// contexts are copied (uploads.go); empty copies none.
+	UploadsDir string
+	Now        func() time.Time
+	Log        io.Writer
 }
 
 // Result is what one Run did and found.
@@ -127,7 +131,14 @@ type Result struct {
 	// ImagesIncomplete are manifests the registry lists without holding all
 	// of them, so there was nothing whole to copy.
 	ImagesIncomplete []string `json:"images_incomplete,omitempty"`
-	Errors           []string `json:"errors,omitempty"`
+	// UploadIndex is the newest uploads index version in the bucket, which
+	// names Uploads submission contexts.
+	UploadIndex         string   `json:"upload_index,omitempty"`
+	Uploads             int      `json:"uploads"`
+	UploadsUploaded     int      `json:"uploads_uploaded"`
+	UploadObjectsPruned int      `json:"upload_objects_pruned"`
+	RemoteUploadBytes   int64    `json:"remote_upload_bytes"`
+	Errors              []string `json:"errors,omitempty"`
 }
 
 func (s *Syncer) now() time.Time {
@@ -143,8 +154,8 @@ func (s *Syncer) logf(format string, args ...any) {
 	}
 }
 
-// Run does one pass: world archives, database bundles, registry images, then
-// expiry. A failure on one item is recorded and the pass carries on; the
+// Run does one pass: world archives, database bundles, registry images,
+// submission uploads, then expiry. A failure on one item is recorded and the pass carries on; the
 // returned error is non-nil when anything failed.
 func (s *Syncer) Run(ctx context.Context) (Result, error) {
 	var res Result
@@ -161,6 +172,7 @@ func (s *Syncer) Run(ctx context.Context) (Result, error) {
 	s.syncWorlds(ctx, remoteWorlds, &res, fail)
 	s.syncDB(ctx, &res, fail)
 	s.syncImages(ctx, &res, fail)
+	s.syncUploads(ctx, &res, fail)
 	s.expireWorlds(ctx, remoteWorlds, &res, fail)
 
 	for _, size := range remoteWorlds {

internal/offsite/uploads.go

@@ -0,0 +1,433 @@
+package offsite
+
+// Submission build contexts: the modpacks users uploaded, kept on the uploads
+// volume as <id>/context.tar.gz (internal/submit.LocalContextStore). A reviewer
+// downloads one to inspect it, and building the submission again needs it; the
+// image already built from it is in the registry copy (images.go).
+//
+// The copy is content-addressed like the images: uploads/blobs/<sha256>.fenc
+// holds each context once, and uploads/index/<stamp>.json.fenc versions which
+// submission holds which. A context deleted here (a withdrawn submission, a
+// rejected one reaped) leaves the newest version but stays in the versions
+// before it for UploadHistory. A volume that comes back empty, as it does on a
+// rebuilt host before its restore, therefore takes nothing out of the bucket
+// for two weeks.
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"maps"
+	"os"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
+	"time"
+)
+
+const (
+	uploadsDir          = "uploads/"
+	uploadBlobsDir      = uploadsDir + "blobs/"
+	uploadIndexDir      = uploadsDir + "index/"
+	maxUploadIndexBytes = 16 << 20
+
+	// UploadContextFile is the one file of a submission's directory on the
+	// uploads volume (internal/submit's contextBlobName).
+	UploadContextFile = "context.tar.gz"
+)
+
+// UploadHistory is how long an uploads index version is kept after a newer one
+// replaced it, with the contexts only it names.
+const UploadHistory = ImageHistory
+
+// uploadIDRE is internal/submit's idRE: a submission id can hold no path
+// separator and no "..".
+var uploadIDRE = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,127}$`)
+
+// UploadIndex is one version of the uploads copy: which submission holds which
+// context.
+type UploadIndex struct {
+	Created  time.Time                `json:"created"`
+	Contexts map[string]UploadContext `json:"contexts"`
+}
+
+// UploadContext is one submission's context as the sync last saw it. Size and
+// ModTime let the next pass skip hashing a file that has not changed.
+type UploadContext struct {
+	Digest  string    `json:"digest"`
+	Size    int64     `json:"size"`
+	ModTime time.Time `json:"mtime"`
+}
+
+// Bytes is the total size of the contexts x names.
+func (x *UploadIndex) Bytes() int64 {
+	var n int64
+	for _, c := range x.Contexts {
+		n += c.Size
+	}
+	return n
+}
+
+func newUploadIndex(at time.Time) *UploadIndex {
+	return &UploadIndex{Created: at, Contexts: map[string]UploadContext{}}
+}
+
+func uploadBlobKey(digest string) string {
+	return uploadBlobsDir + strings.TrimPrefix(digest, "sha256:") + objExt
+}
+
+func uploadIndexKey(stamp string) string { return uploadIndexDir + stamp + imageIndexExt }
+
+func (s *Syncer) syncUploads(ctx context.Context, res *Result, fail func(string, ...any)) {
+	if s.UploadsDir == "" {
+		return
+	}
+	remote, err := s.listSizes(ctx, uploadsDir)
+	if err != nil {
+		fail("list %s in the bucket: %v", uploadsDir, err)
+		return
+	}
+	versions := indexStamps(remote, uploadIndexDir)
+	prev := newUploadIndex(time.Time{})
+	if n := len(versions); n > 0 {
+		if prev, err = LoadUploadIndex(ctx, s.Bucket, s.Key, versions[n-1]); err != nil {
+			fail("read the uploads index %s from the bucket: %v", versions[n-1], err)
+			return
+		}
+	}
+	entries, err := os.ReadDir(s.UploadsDir)
+	if err != nil {
+		fail("read the uploads volume %s: %v", s.UploadsDir, err)
+		return
+	}
+	next := newUploadIndex(s.now().UTC())
+	// failed is set by anything that leaves next short of the volume, which
+	// rules out pruning in this pass.
+	failed := false
+	for _, e := range entries {
+		id := e.Name()
+		if !e.IsDir() || !uploadIDRE.MatchString(id) {
+			continue
+		}
+		old, had := prev.Contexts[id]
+		if ctx.Err() != nil {
+			fail("stopped before the upload of %s: %v", id, ctx.Err())
+			failed = true
+			if had {
+				next.Contexts[id] = old
+			}
+			continue
+		}
+		c, err := s.copyUpload(ctx, id, old, had, remote, res)
+		switch {
+		case errors.Is(err, fs.ErrNotExist):
+			// A directory without its context: an upload still being written,
+			// or one reaped between the listing and here.
+		case err != nil:
+			fail("copy the upload of %s: %v", id, err)
+			failed = true
+			if had {
+				next.Contexts[id] = old
+			}
+		default:
+			next.Contexts[id] = c
+		}
+	}
+
+	stamp := ""
+	if len(versions) > 0 {
+		stamp = versions[len(versions)-1]
+	}
+	if len(versions) == 0 || !maps.EqualFunc(prev.Contexts, next.Contexts, sameContext) {
+		stamp = next.Created.Format(imageStampLayout)
+		raw, err := json.Marshal(next)
+		if err == nil {
+			err = s.putBytes(ctx, uploadIndexKey(stamp), raw)
+		}
+		if err != nil {
+			fail("write the uploads index: %v", err)
+			return
+		}
+		if len(versions) == 0 || versions[len(versions)-1] != stamp {
+			versions = append(versions, stamp)
+		}
+		s.logf("recorded uploads index %s: %d contexts (%s)", stamp, len(next.Contexts), HumanBytes(next.Bytes()))
+	}
+	res.UploadIndex = stamp
+	res.Uploads = len(next.Contexts)
+	if !failed {
+		s.pruneUploads(ctx, versions, next, remote, res, fail)
+	}
+	for key, size := range remote {
+		if strings.HasPrefix(key, uploadBlobsDir) {
+			res.RemoteUploadBytes += size
+		}
+	}
+}
+
+// copyUpload makes sure the bucket holds the context of submission id and
+// returns what the index records for it. A file whose size and modification
+// time match the previous index is not read again.
+func (s *Syncer) copyUpload(ctx context.Context, id string, old UploadContext, had bool, remote map[string]int64, res *Result) (UploadContext, error) {
+	p := filepath.Join(s.UploadsDir, id, UploadContextFile)
+	fi, err := os.Lstat(p)
+	if err != nil {
+		return UploadContext{}, err
+	}
+	if !fi.Mode().IsRegular() {
+		return UploadContext{}, fmt.Errorf("%s is not a regular file", p)
+	}
+	c := UploadContext{Size: fi.Size(), ModTime: fi.ModTime().UTC()}
+	if had && old.Size == c.Size && old.ModTime.Equal(c.ModTime) && remote[uploadBlobKey(old.Digest)] == SealedSize(old.Size) {
+		return old, nil
+	}
+	if c.Digest, err = hashFile(p); err != nil {
+		return UploadContext{}, err
+	}
+	key := uploadBlobKey(c.Digest)
+	if remote[key] == SealedSize(c.Size) {
+		return c, nil
+	}
+	f, err := os.Open(p)
+	if err != nil {
+		return UploadContext{}, err
+	}
+	defer f.Close()
+	// Checked again on the way up: a context replaced between the hash and
+	// the upload fails instead of being stored under the old digest.
+	if err := s.putStream(ctx, key, newDigestReader(f, c.Digest, c.Size), c.Size); err != nil {
+		return UploadContext{}, err
+	}
+	remote[key] = SealedSize(c.Size)
+	res.UploadsUploaded++
+	res.BytesUploaded += c.Size
+	s.logf("copied the upload of %s (%s)", id, HumanBytes(c.Size))
+	return c, nil
+}
+
+func sameContext(a, b UploadContext) bool {
+	return a.Digest == b.Digest && a.Size == b.Size && a.ModTime.Equal(b.ModTime)
+}
+
+func hashFile(p string) (string, error) {
+	f, err := os.Open(p)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+	return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
+}
+
+// pruneUploads drops the index versions replaced more than UploadHistory ago,
+// then every context no remaining version names.
+func (s *Syncer) pruneUploads(ctx context.Context, versions []string, newest *UploadIndex, remote map[string]int64, res *Result, fail func(string, ...any)) {
+	keep, drop := retire(versions, s.now().Add(-UploadHistory))
+	live := map[string]bool{}
+	mark := func(x *UploadIndex) {
+		for _, c := range x.Contexts {
+			live[uploadBlobKey(c.Digest)] = true
+		}
+	}
+	mark(newest)
+	for _, v := range keep {
+		x, err := LoadUploadIndex(ctx, s.Bucket, s.Key, v)
+		if err != nil {
+			fail("read the uploads index %s from the bucket: %v; nothing pruned", v, err)
+			return
+		}
+		mark(x)
+	}
+	for _, v := range drop {
+		if err := s.Bucket.Remove(ctx, uploadIndexKey(v)); err != nil {
+			fail("prune uploads index %s: %v", v, err)
+			return
+		}
+		delete(remote, uploadIndexKey(v))
+		res.UploadObjectsPruned++
+	}
+	for _, key := range slices.Sorted(maps.Keys(remote)) {
+		if !strings.HasPrefix(key, uploadBlobsDir) || live[key] {
+			continue
+		}
+		if err := s.Bucket.Remove(ctx, key); err != nil {
+			fail("prune %s: %v", key, err)
+			continue
+		}
+		delete(remote, key)
+		res.UploadObjectsPruned++
+	}
+}
+
+// UploadIndexes lists the uploads index versions in the bucket, oldest first.
+func UploadIndexes(ctx context.Context, b Bucket) ([]string, error) {
+	objs, err := b.List(ctx, uploadIndexDir)
+	if err != nil {
+		return nil, err
+	}
+	keys := make(map[string]int64, len(objs))
+	for _, o := range objs {
+		keys[o.Key] = o.Size
+	}
+	return indexStamps(keys, uploadIndexDir), nil
+}
+
+// LoadUploadIndex reads one uploads index version.
+func LoadUploadIndex(ctx context.Context, b Bucket, key []byte, stamp string) (*UploadIndex, error) {
+	raw, err := getSealed(ctx, b, key, uploadIndexKey(stamp), maxUploadIndexBytes)
+	if err != nil {
+		return nil, err
+	}
+	x := newUploadIndex(time.Time{})
+	if err := json.Unmarshal(raw, x); err != nil {
+		return nil, fmt.Errorf("uploads index %s: %w", stamp, err)
+	}
+	if x.Contexts == nil {
+		x.Contexts = map[string]UploadContext{}
+	}
+	return x, nil
+}
+
+// ChooseUploadIndex picks the version a restore uses: at when given, otherwise
+// the newest. An empty newest version while an older one names contexts is
+// what a rebuilt host's first sync records, so it is refused with the versions
+// worth choosing instead.
+func ChooseUploadIndex(ctx context.Context, b Bucket, key []byte, at string) (string, *UploadIndex, error) {
+	versions, err := UploadIndexes(ctx, b)
+	if err != nil {
+		return "", nil, err
+	}
+	if len(versions) == 0 {
+		return "", nil, errors.New("the bucket holds no uploads index: no sync has copied the uploads volume yet")
+	}
+	if at != "" {
+		if !slices.Contains(versions, at) {
+			return "", nil, fmt.Errorf("the bucket holds no uploads index %s; it holds %s", at, strings.Join(versions, ", "))
+		}
+		x, err := LoadUploadIndex(ctx, b, key, at)
+		return at, x, err
+	}
+	newest := versions[len(versions)-1]
+	x, err := LoadUploadIndex(ctx, b, key, newest)
+	if err != nil || len(x.Contexts) > 0 {
+		return newest, x, err
+	}
+	var older []string
+	for i := len(versions) - 2; i >= 0; i-- {
+		o, err := LoadUploadIndex(ctx, b, key, versions[i])
+		if err != nil {
+			return "", nil, err
+		}
+		if len(o.Contexts) > 0 {
+			older = append(older, fmt.Sprintf("%s (%d contexts)", versions[i], len(o.Contexts)))
+		}
+	}
+	if len(older) == 0 {
+		return newest, x, nil
+	}
+	return "", nil, fmt.Errorf("the newest uploads index %s lists no contexts, which is what a rebuilt host records before its restore; pick the state to restore with -at: %s",
+		newest, strings.Join(older, ", "))
+}
+
+// FetchUploadsResult counts what FetchUploads wrote.
+type FetchUploadsResult struct {
+	Written  int
+	Present  int
+	Bytes    int64
+	Failures []string
+}
+
+// FetchUploads writes every context x names into dir as <id>/context.tar.gz,
+// skipping one already there with the same content. Each is written to a
+// temporary file beside its place, checked against its digest, and renamed
+// into place, so a failure leaves no partial context. uid and gid own what it
+// creates (-1 leaves the caller's).
+func FetchUploads(ctx context.Context, b Bucket, key []byte, x *UploadIndex, dir string, uid, gid int, log io.Writer) (FetchUploadsResult, error) {
+	var res FetchUploadsResult
+	for _, id := range slices.Sorted(maps.Keys(x.Contexts)) {
+		if err := ctx.Err(); err != nil {
+			return res, err
+		}
+		c := x.Contexts[id]
+		if !uploadIDRE.MatchString(id) {
+			res.Failures = append(res.Failures, fmt.Sprintf("%s: not a submission id", id))
+			continue
+		}
+		wrote, err := fetchUpload(ctx, b, key, id, c, dir, uid, gid)
+		switch {
+		case err != nil:
+			res.Failures = append(res.Failures, fmt.Sprintf("%s: %v", id, err))
+		case wrote:
+			res.Written++
+			res.Bytes += c.Size
+			if log != nil {
+				fmt.Fprintf(log, "felis offsite: restored the upload of %s (%s)\n", id, HumanBytes(c.Size))
+			}
+		default:
+			res.Present++
+		}
+	}
+	if len(res.Failures) > 0 {
+		return res, fmt.Errorf("%d contexts failed; first: %s", len(res.Failures), res.Failures[0])
+	}
+	return res, nil
+}
+
+func fetchUpload(ctx context.Context, b Bucket, key []byte, id string, c UploadContext, dir string, uid, gid int) (bool, error) {
+	sub := filepath.Join(dir, id)
+	final := filepath.Join(sub, UploadContextFile)
+	if fi, err := os.Lstat(final); err == nil && fi.Mode().IsRegular() && fi.Size() == c.Size {
+		if d, err := hashFile(final); err == nil && d == c.Digest {
+			return false, nil
+		}
+	}
+	if err := os.MkdirAll(sub, 0o770); err != nil {
+		return false, err
+	}
+	if err := chown(sub, uid, gid); err != nil {
+		return false, err
+	}
+	rc, err := openSealed(ctx, b, key, uploadBlobKey(c.Digest))
+	if err != nil {
+		return false, err
+	}
+	defer rc.Close()
+	tmp, err := os.CreateTemp(sub, UploadContextFile+".*.restore")
+	if err != nil {
+		return false, err
+	}
+	defer os.Remove(tmp.Name())
+	_, err = io.Copy(tmp, newDigestReader(rc, c.Digest, c.Size))
+	if err == nil {
+		err = tmp.Sync()
+	}
+	if cerr := tmp.Close(); err == nil {
+		err = cerr
+	}
+	if err == nil {
+		err = os.Chmod(tmp.Name(), 0o660)
+	}
+	if err == nil {
+		err = chown(tmp.Name(), uid, gid)
+	}
+	if err == nil {
+		err = os.Rename(tmp.Name(), final)
+	}
+	return err == nil, err
+}
+
+func chown(p string, uid, gid int) error {
+	if uid < 0 && gid < 0 {
+		return nil
+	}
+	return os.Lchown(p, uid, gid)
+}

internal/offsite/uploads_test.go

@@ -0,0 +1,198 @@
+package offsite
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func newUploadsSyncer(t *testing.T) (*Syncer, *memBucket, *time.Time, string) {
+	t.Helper()
+	dir := t.TempDir()
+	b := newMemBucket()
+	clock := now
+	return &Syncer{
+		Bucket: b, Catalog: &fakeCatalog{}, Key: testKey(t), UploadsDir: dir,
+		Now: func() time.Time { return clock },
+	}, b, &clock, dir
+}
+
+func putContext(t *testing.T, dir, id string, data []byte, mtime time.Time) {
+	t.Helper()
+	sub := filepath.Join(dir, id)
+	if err := os.MkdirAll(sub, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	p := filepath.Join(sub, UploadContextFile)
+	if err := os.WriteFile(p, data, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(p, mtime, mtime); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func runSync(t *testing.T, s *Syncer) Result {
+	t.Helper()
+	res, err := s.Run(context.Background())
+	if err != nil {
+		t.Fatalf("Run: %v (%v)", err, res.Errors)
+	}
+	return res
+}
+
+// TestSyncUploadsCopiesContexts: every submission's context is copied, two
+// identical ones once; a directory without its context and a name that is not
+// a submission id are left out. A second run with nothing new sends nothing.
+func TestSyncUploadsCopiesContexts(t *testing.T) {
+	s, b, _, dir := newUploadsSyncer(t)
+	pack := bytes.Repeat([]byte("modpack"), 5000)
+	putContext(t, dir, "sub-aaa", pack, now.Add(-time.Hour))
+	putContext(t, dir, "sub-bbb", pack, now.Add(-time.Hour))
+	putContext(t, dir, "sub-ccc", []byte("another pack"), now.Add(-time.Hour))
+	if err := os.MkdirAll(filepath.Join(dir, "sub-writing"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.MkdirAll(filepath.Join(dir, "lost+found"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	res := runSync(t, s)
+	if res.Uploads != 3 || res.UploadsUploaded != 2 {
+		t.Fatalf("uploads=%d uploaded=%d, want 3 contexts in 2 objects", res.Uploads, res.UploadsUploaded)
+	}
+	if got := keysUnder(b, uploadBlobsDir); len(got) != 2 {
+		t.Fatalf("blobs %v, want 2", got)
+	}
+	if got := keysUnder(b, uploadIndexDir); len(got) != 1 {
+		t.Fatalf("index versions %v, want 1", got)
+	}
+	x, err := LoadUploadIndex(context.Background(), b, s.Key, res.UploadIndex)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(x.Contexts) != 3 || x.Contexts["sub-aaa"].Digest != x.Contexts["sub-bbb"].Digest {
+		t.Fatalf("index %+v", x.Contexts)
+	}
+
+	puts := b.puts
+	res = runSync(t, s)
+	if b.puts != puts || res.UploadsUploaded != 0 {
+		t.Fatalf("second run put %d objects, uploaded %d; want nothing", b.puts-puts, res.UploadsUploaded)
+	}
+}
+
+// TestSyncUploadsEmptyVolumeKeepsCopies: a volume that comes back empty (a
+// rebuilt host before its restore) records an empty version, keeps every
+// context for UploadHistory, and a restore refuses the empty version until
+// told which one to use. Past the history both go.
+func TestSyncUploadsEmptyVolumeKeepsCopies(t *testing.T) {
+	s, b, clock, dir := newUploadsSyncer(t)
+	putContext(t, dir, "sub-aaa", []byte("pack a"), now.Add(-time.Hour))
+	putContext(t, dir, "sub-bbb", []byte("pack b"), now.Add(-time.Hour))
+	first := runSync(t, s).UploadIndex
+
+	for _, id := range []string{"sub-aaa", "sub-bbb"} {
+		if err := os.RemoveAll(filepath.Join(dir, id)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	*clock = clock.Add(time.Hour)
+	res := runSync(t, s)
+	if res.Uploads != 0 || res.UploadObjectsPruned != 0 {
+		t.Fatalf("empty volume: uploads=%d pruned=%d", res.Uploads, res.UploadObjectsPruned)
+	}
+	if got := keysUnder(b, uploadBlobsDir); len(got) != 2 {
+		t.Fatalf("blobs %v, want both kept", got)
+	}
+	ctx := context.Background()
+	if _, _, err := ChooseUploadIndex(ctx, b, s.Key, ""); err == nil || !strings.Contains(err.Error(), first) {
+		t.Fatalf("choosing the empty newest version: %v, want a refusal naming %s", err, first)
+	}
+	stamp, x, err := ChooseUploadIndex(ctx, b, s.Key, first)
+	if err != nil || stamp != first || len(x.Contexts) != 2 {
+		t.Fatalf("choose %s: %s %v %v", first, stamp, x, err)
+	}
+
+	*clock = clock.Add(UploadHistory + time.Hour)
+	res = runSync(t, s)
+	if got := keysUnder(b, uploadBlobsDir); len(got) != 0 {
+		t.Fatalf("blobs %v after the history, want none", got)
+	}
+	if got := keysUnder(b, uploadIndexDir); len(got) != 1 {
+		t.Fatalf("index versions %v after the history, want the newest only", got)
+	}
+	if res.UploadObjectsPruned != 3 {
+		t.Fatalf("pruned %d, want the old version and its 2 contexts", res.UploadObjectsPruned)
+	}
+}
+
+// TestSyncUploadsReplacedContext: a context rewritten in place is copied again;
+// the old bytes stay while the version that names them is kept.
+func TestSyncUploadsReplacedContext(t *testing.T) {
+	s, b, clock, dir := newUploadsSyncer(t)
+	putContext(t, dir, "sub-aaa", []byte("version one"), now.Add(-time.Hour))
+	runSync(t, s)
+
+	putContext(t, dir, "sub-aaa", []byte("version two"), now)
+	*clock = clock.Add(time.Hour)
+	res := runSync(t, s)
+	if res.UploadsUploaded != 1 {
+		t.Fatalf("uploaded %d, want the new version", res.UploadsUploaded)
+	}
+	if got := keysUnder(b, uploadBlobsDir); len(got) != 2 {
+		t.Fatalf("blobs %v, want old and new", got)
+	}
+
+	*clock = clock.Add(UploadHistory + time.Hour)
+	runSync(t, s)
+	if got := keysUnder(b, uploadBlobsDir); len(got) != 1 {
+		t.Fatalf("blobs %v after the history, want the new one only", got)
+	}
+}
+
+// TestFetchUploadsRestores: a restore writes every context back in place,
+// leaves one that is already right alone, and rewrites one whose bytes differ.
+func TestFetchUploadsRestores(t *testing.T) {
+	s, b, _, dir := newUploadsSyncer(t)
+	a, c := bytes.Repeat([]byte("a"), 70000), []byte("pack c")
+	putContext(t, dir, "sub-aaa", a, now.Add(-time.Hour))
+	putContext(t, dir, "sub-ccc", c, now.Add(-time.Hour))
+	stamp := runSync(t, s).UploadIndex
+
+	ctx := context.Background()
+	x, err := LoadUploadIndex(ctx, b, s.Key, stamp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	target := t.TempDir()
+	res, err := FetchUploads(ctx, b, s.Key, x, target, -1, -1, nil)
+	if err != nil || res.Written != 2 || res.Bytes != int64(len(a)+len(c)) {
+		t.Fatalf("fetch: %+v %v", res, err)
+	}
+	for id, want := range map[string][]byte{"sub-aaa": a, "sub-ccc": c} {
+		got, err := os.ReadFile(filepath.Join(target, id, UploadContextFile))
+		if err != nil || !bytes.Equal(got, want) {
+			t.Fatalf("%s restored as %d bytes (%v)", id, len(got), err)
+		}
+	}
+
+	if err := os.WriteFile(filepath.Join(target, "sub-ccc", UploadContextFile), []byte("pack X"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	res, err = FetchUploads(ctx, b, s.Key, x, target, -1, -1, nil)
+	if err != nil || res.Written != 1 || res.Present != 1 {
+		t.Fatalf("second fetch: %+v %v, want the damaged one rewritten", res, err)
+	}
+	if got, _ := os.ReadFile(filepath.Join(target, "sub-ccc", UploadContextFile)); !bytes.Equal(got, c) {
+		t.Fatalf("sub-ccc is %q after the second fetch", got)
+	}
+	leftovers, _ := filepath.Glob(filepath.Join(target, "*", "*.restore"))
+	if len(leftovers) != 0 {
+		t.Fatalf("temporary files left behind: %v", leftovers)
+	}
+}

internal/platform/workloads.go

@@ -189,6 +189,15 @@ const (
 	nonRootUID int64 = 1000
 )
 
+// UploadsPVCName is felis-api's submission uploads PVC in the control-plane
+// namespace; the host-side off-site copy reads and restores it.
+const UploadsPVCName = uploadsPVCName
+
+// ControlPlaneUID is the uid and gid the control-plane pods run as, which own
+// what they write to their volumes; a host-side restore into one of them
+// writes as it.
+const ControlPlaneUID = int(nonRootUID)
+
 // APIInternalServiceName is the ClusterIP Service that fronts the felis-api
 // internal face (8081). It is SEPARATE from the external NodePort Service (SAAPI)
 // on purpose — see apiInternalService. The login pod resolves it by cross-namespace

internal/watchdog/probes.go

@@ -361,8 +361,8 @@ func OffsiteFinding(statusFile string, now time.Time) *Finding {
 	}
 	f := &Finding{
 		Key: "offsite", Severity: Warning, For: backupFor,
-		Summary:   "异地备份从未成功同步过，世界归档、数据库备份与用户镜像只在本机",
-		SummaryEN: "the off-site copy has never completed; world archives, database bundles and user images exist on this machine only",
+		Summary:   "异地备份从未成功同步过，世界归档、数据库备份、用户镜像与上传的整合包只在本机",
+		SummaryEN: "the off-site copy has never completed; world archives, database bundles, user images and uploaded modpacks exist on this machine only",
 		Hint:      hint,
 	}
 	if st != nil && !st.LastSuccess.IsZero() {