feat(offsite): 世界归档与数据库备份加密同步到异地 S3，reaper 确认异地副本后才删除世界

internal/config/config.go

@@ -23,6 +23,7 @@ type Config struct {
 	K8s      K8sConfig      `toml:"k8s"`
 	Registry RegistryConfig `toml:"registry"`
 	Archive  ArchiveConfig  `toml:"archive"`
+	Offsite  OffsiteConfig  `toml:"offsite"`
 	SMTP     SMTPConfig     `toml:"smtp"`
 	// AuthSources is the [[auth_source]] array-of-tables: the third-party Yggdrasil
 	// roots the Felis-nano hasJoined multiplexer federates over, in priority order
@@ -236,6 +237,48 @@ type ArchiveS3Config struct {
 	SecretKeyRef string `toml:"secret_key_ref"`
 }
 
+// OffsiteConfig is the [offsite] table: the S3-compatible bucket, away from
+// this machine, that `felis offsite sync` (felis-offsite.timer on the host)
+// copies every world archive and the newest database bundles into, encrypted
+// (internal/offsite). An empty bucket means no off-site copy: the archives and
+// the database then share the node's disk with the worlds.
+//
+// When it is set the reaper deletes an idle world only once the archive it made
+// has its off-site copy, so the reaper pod reads this table too. The secrets
+// follow the credential rule of [archive.s3]: the *_ref fields NAME the
+// environment variables holding them (bootstrap writes /etc/felis/offsite.env),
+// and they are never written into felis.toml.
+type OffsiteConfig struct {
+	// Endpoint is https://host[:port]; http:// only for a store on a trusted
+	// network. A bare host means TLS.
+	Endpoint string `toml:"endpoint"`
+	Region   string `toml:"region"`
+	Bucket   string `toml:"bucket"`
+	// Prefix places every object under this key prefix, so one bucket can hold
+	// several installs.
+	Prefix       string `toml:"prefix"`
+	AccessKeyRef string `toml:"access_key_ref"`
+	SecretKeyRef string `toml:"secret_key_ref"`
+	// KeyRef names the variable holding the encryption key (`felis offsite
+	// keygen`). The copies are unreadable without it, so it must also be kept
+	// somewhere other than this machine.
+	KeyRef string `toml:"key_ref"`
+	// DBKeep is how many of the newest database bundles the bucket keeps.
+	DBKeep int `toml:"db_keep"`
+}
+
+// Enabled reports whether an off-site bucket is configured.
+func (o OffsiteConfig) Enabled() bool { return o.Bucket != "" }
+
+// Default environment variable names for the [offsite] secrets, and the bundle
+// count kept off-site.
+const (
+	DefaultOffsiteAccessKeyEnv = "FELIS_OFFSITE_ACCESS_KEY"
+	DefaultOffsiteSecretKeyEnv = "FELIS_OFFSITE_SECRET_KEY"
+	DefaultOffsiteKeyEnv       = "FELIS_OFFSITE_KEY"
+	DefaultOffsiteDBKeep       = 30
+)
+
 // archive store backends recognized by §19.
 var archiveStores = map[string]struct{}{
 	"tarLocal":       {},
@@ -347,6 +390,20 @@ func (c *Config) applyDefaults() {
 	if c.SMTP.Host != "" && c.SMTP.Port == 0 {
 		c.SMTP.Port = defaultSMTPPort
 	}
+	if c.Offsite.Enabled() {
+		if c.Offsite.AccessKeyRef == "" {
+			c.Offsite.AccessKeyRef = DefaultOffsiteAccessKeyEnv
+		}
+		if c.Offsite.SecretKeyRef == "" {
+			c.Offsite.SecretKeyRef = DefaultOffsiteSecretKeyEnv
+		}
+		if c.Offsite.KeyRef == "" {
+			c.Offsite.KeyRef = DefaultOffsiteKeyEnv
+		}
+		if c.Offsite.DBKeep == 0 {
+			c.Offsite.DBKeep = DefaultOffsiteDBKeep
+		}
+	}
 }
 
 // Validate enforces the mandatory fields (spec §24: database.url is 强制) and
@@ -398,12 +455,43 @@ func (c *Config) Validate() error {
 	if c.SMTP.MaxPerHour < 0 {
 		return fmt.Errorf("config: [smtp] max_per_hour %d must be positive (0 means the default %d)", c.SMTP.MaxPerHour, DefaultMailPerHour)
 	}
+	if err := c.Offsite.validate(); err != nil {
+		return err
+	}
 	if h := c.Auth.ClientIPHeader; strings.ContainsAny(h, " :\t\r\n") {
 		return fmt.Errorf("config: [auth] client_ip_header %q must be a bare header name such as CF-Connecting-IP or X-Forwarded-For", h)
 	}
 	return c.validateAuthSources()
 }
 
+// validate checks a configured [offsite] table. Nothing is required of an
+// unconfigured one; a half-filled one (an endpoint and no bucket) is refused,
+// since it reads as configured while nothing is copied anywhere.
+func (o OffsiteConfig) validate() error {
+	if !o.Enabled() {
+		if o.Endpoint != "" || o.Prefix != "" {
+			return fmt.Errorf("config: [offsite] names an endpoint or prefix but no bucket; set bucket, or remove the table")
+		}
+		return nil
+	}
+	if strings.TrimSpace(o.Endpoint) == "" {
+		return fmt.Errorf("config: [offsite] endpoint is required with bucket %q (e.g. https://s3.eu-central-1.amazonaws.com)", o.Bucket)
+	}
+	if u, err := url.Parse(o.Endpoint); strings.Contains(o.Endpoint, "://") && (err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || strings.Trim(u.Path, "/") != "") {
+		return fmt.Errorf("config: [offsite] endpoint %q must be http(s)://host[:port] with no path; the bucket goes in bucket", o.Endpoint)
+	}
+	if strings.ContainsAny(o.Bucket, "/ ") {
+		return fmt.Errorf("config: [offsite] bucket %q must be a bare bucket name; put a key prefix in prefix", o.Bucket)
+	}
+	if strings.Contains(o.Prefix, "..") {
+		return fmt.Errorf("config: [offsite] prefix %q must not contain ..", o.Prefix)
+	}
+	if o.DBKeep < 1 {
+		return fmt.Errorf("config: [offsite] db_keep %d must be at least 1", o.DBKeep)
+	}
+	return nil
+}
+
 // authSourcePrefixRe is the shape of a prefix. It is prepended to a real Minecraft
 // username (LS_steve), so it is confined to the username charset and kept short enough to
 // leave a legible name behind after truncation.

internal/config/config_test.go

@@ -609,3 +609,50 @@ url = "postgres://felis@db/felis"
 		t.Fatal("negative max_per_hour accepted")
 	}
 }
+
+// TestLoadOffsite: a bucket turns the table on and fills in the secret refs and
+// bundle count; a half-filled or malformed table fails at load.
+func TestLoadOffsite(t *testing.T) {
+	const head = `
+[server]
+root_domain = "mc.example.net"
+[database]
+url = "postgres://felis@db/felis"
+`
+	cfg, err := config.Load(writeTOML(t, head))
+	if err != nil {
+		t.Fatalf("Load without [offsite]: %v", err)
+	}
+	if cfg.Offsite.Enabled() || cfg.Offsite.KeyRef != "" {
+		t.Fatalf("absent [offsite] must stay off and zero, got %+v", cfg.Offsite)
+	}
+
+	cfg, err = config.Load(writeTOML(t, head+`
+[offsite]
+endpoint = "https://s3.example.net"
+bucket = "felis-copies"
+prefix = "site-a"
+`))
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	o := cfg.Offsite
+	if !o.Enabled() || o.AccessKeyRef != "FELIS_OFFSITE_ACCESS_KEY" || o.SecretKeyRef != "FELIS_OFFSITE_SECRET_KEY" ||
+		o.KeyRef != "FELIS_OFFSITE_KEY" || o.DBKeep != 30 {
+		t.Fatalf("defaults not applied: %+v", o)
+	}
+
+	for name, body := range map[string]string{
+		"no bucket":        "endpoint = \"https://s3.example.net\"",
+		"no endpoint":      "bucket = \"b\"",
+		"path in endpoint": "endpoint = \"https://s3.example.net/b\"\nbucket = \"b\"",
+		"ftp endpoint":     "endpoint = \"ftp://s3.example.net\"\nbucket = \"b\"",
+		"bucket with key":  "endpoint = \"s3.example.net\"\nbucket = \"b/x\"",
+		"negative keep":    "endpoint = \"s3.example.net\"\nbucket = \"b\"\ndb_keep = -1",
+		"secret in toml":   "endpoint = \"s3.example.net\"\nbucket = \"b\"\nsecret_key = \"x\"",
+	} {
+		if _, err := config.Load(writeTOML(t, head+"[offsite]\n"+body+"\n")); err == nil {
+			t.Errorf("%s: loaded", name)
+		}
+	}
+}

internal/dbbackup/dbbackup.go

@@ -266,8 +266,8 @@ func BundleName(t time.Time, label string) string {
 	return bundlePrefix + t.UTC().Format(stampLayout) + "-" + label + bundleExt
 }
 
-// parseBundleName reverses BundleName.
-func parseBundleName(name string) (time.Time, string, bool) {
+// ParseBundleName reverses BundleName: the stamp and label of a bundle file name.
+func ParseBundleName(name string) (time.Time, string, bool) {
 	if !strings.HasPrefix(name, bundlePrefix) || !strings.HasSuffix(name, bundleExt) {
 		return time.Time{}, "", false
 	}
@@ -297,7 +297,7 @@ func List(dir string) ([]Bundle, error) {
 		if !e.Type().IsRegular() {
 			continue
 		}
-		t, label, ok := parseBundleName(e.Name())
+		t, label, ok := ParseBundleName(e.Name())
 		if !ok {
 			continue
 		}

internal/dbbackup/dbbackup_test.go

@@ -474,8 +474,8 @@ func TestParseBundleName(t *testing.T) {
 		"felis-db-20260924T033000Z-daily.tar.sha256": false,
 		"other.tar": false,
 	} {
-		if _, _, got := parseBundleName(name); got != ok {
-			t.Errorf("parseBundleName(%q) ok = %v, want %v", name, got, ok)
+		if _, _, got := ParseBundleName(name); got != ok {
+			t.Errorf("ParseBundleName(%q) ok = %v, want %v", name, got, ok)
 		}
 	}
 }

internal/offsite/bucket.go

@@ -0,0 +1,160 @@
+package offsite
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+// Object is one object in the bucket, its key relative to the configured
+// prefix.
+type Object struct {
+	Key      string
+	Size     int64
+	Modified time.Time
+}
+
+// Bucket is the object store the sync writes to. S3 is the production one; the
+// tests use an in-memory map.
+type Bucket interface {
+	// Put stores exactly size bytes read from r under key.
+	Put(ctx context.Context, key string, r io.Reader, size int64) error
+	// Get opens key. A missing key is ErrNotFound.
+	Get(ctx context.Context, key string) (io.ReadCloser, error)
+	// List returns every object whose key starts with prefix.
+	List(ctx context.Context, prefix string) ([]Object, error)
+	Remove(ctx context.Context, key string) error
+}
+
+// ErrNotFound is a key the bucket does not hold.
+var ErrNotFound = errors.New("offsite: no such object")
+
+// S3Config locates an S3-compatible bucket. Endpoint takes an http:// or
+// https:// scheme; a bare host means TLS.
+type S3Config struct {
+	Endpoint  string
+	Region    string
+	Bucket    string
+	Prefix    string
+	AccessKey string
+	SecretKey string
+}
+
+// S3 is a Bucket on an S3-compatible store (AWS S3, Backblaze B2, Cloudflare
+// R2, Wasabi, MinIO...). Every key is placed under Prefix.
+type S3 struct {
+	client *minio.Client
+	bucket string
+	prefix string
+}
+
+// partSize bounds what one multipart upload holds in memory.
+const partSize = 16 << 20
+
+// NewS3 builds the client. It does not touch the network; Check does.
+func NewS3(c S3Config) (*S3, error) {
+	host, secure, err := splitEndpoint(c.Endpoint)
+	if err != nil {
+		return nil, err
+	}
+	if c.Bucket == "" {
+		return nil, errors.New("offsite: no bucket configured")
+	}
+	if c.AccessKey == "" || c.SecretKey == "" {
+		return nil, errors.New("offsite: the access key or the secret key is empty")
+	}
+	cl, err := minio.New(host, &minio.Options{
+		Creds:  credentials.NewStaticV4(c.AccessKey, c.SecretKey, ""),
+		Secure: secure,
+		Region: c.Region,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("offsite: %w", err)
+	}
+	return &S3{client: cl, bucket: c.Bucket, prefix: cleanPrefix(c.Prefix)}, nil
+}
+
+func splitEndpoint(ep string) (host string, secure bool, err error) {
+	ep = strings.TrimSpace(ep)
+	switch {
+	case ep == "":
+		return "", false, errors.New("offsite: no endpoint configured")
+	case strings.HasPrefix(ep, "https://"):
+		return strings.Trim(strings.TrimPrefix(ep, "https://"), "/"), true, nil
+	case strings.HasPrefix(ep, "http://"):
+		return strings.Trim(strings.TrimPrefix(ep, "http://"), "/"), false, nil
+	default:
+		return strings.Trim(ep, "/"), true, nil
+	}
+}
+
+func cleanPrefix(p string) string {
+	p = strings.Trim(p, "/")
+	if p == "" {
+		return ""
+	}
+	return p + "/"
+}
+
+// Check proves the bucket is reachable and the credentials may use it.
+func (s *S3) Check(ctx context.Context) error {
+	ok, err := s.client.BucketExists(ctx, s.bucket)
+	if err != nil {
+		return fmt.Errorf("offsite: cannot reach bucket %q (endpoint unreachable or credentials rejected): %w", s.bucket, err)
+	}
+	if !ok {
+		return fmt.Errorf("offsite: bucket %q does not exist; create it first", s.bucket)
+	}
+	return nil
+}
+
+func (s *S3) Put(ctx context.Context, key string, r io.Reader, size int64) error {
+	_, err := s.client.PutObject(ctx, s.bucket, s.prefix+key, r, size, minio.PutObjectOptions{
+		ContentType: "application/octet-stream",
+		PartSize:    partSize,
+	})
+	return err
+}
+
+func (s *S3) Get(ctx context.Context, key string) (io.ReadCloser, error) {
+	obj, err := s.client.GetObject(ctx, s.bucket, s.prefix+key, minio.GetObjectOptions{})
+	if err != nil {
+		return nil, err
+	}
+	// GetObject is lazy; Stat surfaces a missing key before the first read.
+	if _, err := obj.Stat(); err != nil {
+		obj.Close()
+		if isNotFound(err) {
+			return nil, fmt.Errorf("%w: %s", ErrNotFound, key)
+		}
+		return nil, err
+	}
+	return obj, nil
+}
+
+func (s *S3) List(ctx context.Context, prefix string) ([]Object, error) {
+	var out []Object
+	for o := range s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: s.prefix + prefix, Recursive: true}) {
+		if o.Err != nil {
+			return nil, o.Err
+		}
+		out = append(out, Object{Key: strings.TrimPrefix(o.Key, s.prefix), Size: o.Size, Modified: o.LastModified})
+	}
+	return out, nil
+}
+
+func (s *S3) Remove(ctx context.Context, key string) error {
+	return s.client.RemoveObject(ctx, s.bucket, s.prefix+key, minio.RemoveObjectOptions{})
+}
+
+func isNotFound(err error) bool {
+	resp := minio.ToErrorResponse(err)
+	return resp.Code == "NoSuchKey" || resp.StatusCode == http.StatusNotFound
+}

internal/offsite/crypt.go

@@ -0,0 +1,195 @@
+package offsite
+
+import (
+	"bufio"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// An object in the bucket is its file encrypted with AES-256-GCM in fixed
+// segments, so a multi-gigabyte world streams through in constant memory:
+//
+//	magic "FELISOS1" | 7-byte random nonce prefix | segment 0 | segment 1 | ...
+//
+// Segment i seals up to segmentSize bytes under the nonce
+// prefix || uint32(i) || last, where last is 1 on the final segment only. The
+// counter stops segments being reordered or dropped, and the last flag stops
+// the object being cut short at a segment boundary: either change fails to
+// authenticate.
+const (
+	magic       = "FELISOS1"
+	prefixSize  = 7
+	segmentSize = 64 << 10
+	headerSize  = len(magic) + prefixSize
+	// KeySize is the key length: AES-256.
+	KeySize = 32
+)
+
+// ErrAuth is a segment that does not authenticate: the wrong key, or an object
+// damaged in the bucket or on the way.
+var ErrAuth = errors.New("offsite: object does not decrypt with this key (wrong key, or the object is damaged)")
+
+// NewKey returns a fresh random key in the text form ParseKey reads.
+func NewKey() (string, error) {
+	k := make([]byte, KeySize)
+	if _, err := rand.Read(k); err != nil {
+		return "", err
+	}
+	return base64.StdEncoding.EncodeToString(k), nil
+}
+
+// ParseKey decodes a key written by NewKey (standard base64 of 32 bytes).
+func ParseKey(s string) ([]byte, error) {
+	k, err := base64.StdEncoding.DecodeString(strings.TrimSpace(s))
+	if err != nil || len(k) != KeySize {
+		return nil, fmt.Errorf("offsite: the key must be %d random bytes in base64 (generate one with `felis offsite keygen`)", KeySize)
+	}
+	return k, nil
+}
+
+// KeyID names a key without revealing it, so status output and logs can say
+// which key a bucket's objects were written with.
+func KeyID(key []byte) string {
+	h := sha256.Sum256(append([]byte("felis-offsite-key-id\x00"), key...))
+	return hex.EncodeToString(h[:8])
+}
+
+// SealedSize is the size of the object Encrypt makes from n plaintext bytes.
+// Uploads need it up front: an S3 upload of unknown length buffers far more.
+func SealedSize(n int64) int64 {
+	segs := (n + segmentSize - 1) / segmentSize
+	if segs == 0 {
+		segs = 1
+	}
+	return int64(headerSize) + n + segs*16
+}
+
+func newAEAD(key []byte) (cipher.AEAD, error) {
+	if len(key) != KeySize {
+		return nil, fmt.Errorf("offsite: key is %d bytes, want %d", len(key), KeySize)
+	}
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	return cipher.NewGCM(block)
+}
+
+func segmentNonce(prefix []byte, i uint32, last bool) []byte {
+	n := make([]byte, 12)
+	copy(n, prefix)
+	binary.BigEndian.PutUint32(n[prefixSize:], i)
+	if last {
+		n[11] = 1
+	}
+	return n
+}
+
+// Encrypt writes src to dst in the segmented format.
+func Encrypt(dst io.Writer, src io.Reader, key []byte) error {
+	aead, err := newAEAD(key)
+	if err != nil {
+		return err
+	}
+	prefix := make([]byte, prefixSize)
+	if _, err := rand.Read(prefix); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(dst, magic); err != nil {
+		return err
+	}
+	if _, err := dst.Write(prefix); err != nil {
+		return err
+	}
+	in := bufio.NewReaderSize(src, segmentSize+1)
+	buf := make([]byte, segmentSize)
+	out := make([]byte, 0, segmentSize+aead.Overhead())
+	for i := uint32(0); ; i++ {
+		n, err := io.ReadFull(in, buf)
+		switch {
+		case err == io.EOF || err == io.ErrUnexpectedEOF:
+			err = nil
+		case err != nil:
+			return err
+		}
+		last := n < segmentSize
+		if !last {
+			if _, perr := in.Peek(1); perr == io.EOF {
+				last = true
+			} else if perr != nil {
+				return perr
+			}
+		}
+		out = aead.Seal(out[:0], segmentNonce(prefix, i, last), buf[:n], []byte(magic))
+		if _, err := dst.Write(out); err != nil {
+			return err
+		}
+		if last {
+			return nil
+		}
+		if i == ^uint32(0) {
+			return errors.New("offsite: file too large to encrypt")
+		}
+	}
+}
+
+// Decrypt reverses Encrypt. Nothing unauthenticated is written: each segment
+// is checked before its plaintext reaches dst, and a missing tail is an error.
+// A caller writing to a file still has to discard it on error, since earlier
+// segments were already written.
+func Decrypt(dst io.Writer, src io.Reader, key []byte) error {
+	aead, err := newAEAD(key)
+	if err != nil {
+		return err
+	}
+	hdr := make([]byte, headerSize)
+	if _, err := io.ReadFull(src, hdr); err != nil {
+		return fmt.Errorf("offsite: object too short for its header: %w", err)
+	}
+	if string(hdr[:len(magic)]) != magic {
+		return errors.New("offsite: not a Felis off-site object (bad magic)")
+	}
+	prefix := hdr[len(magic):]
+	sealed := segmentSize + aead.Overhead()
+	in := bufio.NewReaderSize(src, sealed+1)
+	buf := make([]byte, sealed)
+	var plain []byte
+	for i := uint32(0); ; i++ {
+		n, err := io.ReadFull(in, buf)
+		switch {
+		case err == io.EOF:
+			return fmt.Errorf("offsite: object is cut short after %d segments", i)
+		case err == io.ErrUnexpectedEOF:
+			err = nil
+		case err != nil:
+			return err
+		}
+		last := n < sealed
+		if !last {
+			if _, perr := in.Peek(1); perr == io.EOF {
+				last = true
+			} else if perr != nil {
+				return perr
+			}
+		}
+		plain, err = aead.Open(plain[:0], segmentNonce(prefix, i, last), buf[:n], []byte(magic))
+		if err != nil {
+			return ErrAuth
+		}
+		if _, err := dst.Write(plain); err != nil {
+			return err
+		}
+		if last {
+			return nil
+		}
+	}
+}

internal/offsite/offsite_test.go

@@ -0,0 +1,446 @@
+package offsite
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+func testKey(t *testing.T) []byte {
+	t.Helper()
+	s, err := NewKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	k, err := ParseKey(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return k
+}
+
+func seal(t *testing.T, plain, key []byte) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	if err := Encrypt(&buf, bytes.NewReader(plain), key); err != nil {
+		t.Fatal(err)
+	}
+	return buf.Bytes()
+}
+
+func TestEncryptRoundTrip(t *testing.T) {
+	key := testKey(t)
+	for _, n := range []int{0, 1, segmentSize - 1, segmentSize, segmentSize + 1, 3*segmentSize + 5} {
+		plain := make([]byte, n)
+		rand.Read(plain)
+		sealed := seal(t, plain, key)
+		if int64(len(sealed)) != SealedSize(int64(n)) {
+			t.Errorf("n=%d: sealed %d bytes, SealedSize says %d", n, len(sealed), SealedSize(int64(n)))
+		}
+		var out bytes.Buffer
+		if err := Decrypt(&out, bytes.NewReader(sealed), key); err != nil {
+			t.Fatalf("n=%d: decrypt: %v", n, err)
+		}
+		if !bytes.Equal(out.Bytes(), plain) {
+			t.Fatalf("n=%d: round trip changed the data", n)
+		}
+	}
+}
+
+// TestDecryptRejectsTampering: a wrong key, a flipped bit, a missing tail at a
+// segment boundary and swapped segments all fail instead of yielding data.
+func TestDecryptRejectsTampering(t *testing.T) {
+	key := testKey(t)
+	plain := make([]byte, 3*segmentSize)
+	rand.Read(plain)
+	sealed := seal(t, plain, key)
+	seg := segmentSize + 16
+
+	flipped := bytes.Clone(sealed)
+	flipped[headerSize+10] ^= 1
+	cut := sealed[:headerSize+2*seg]
+	swapped := bytes.Clone(sealed)
+	copy(swapped[headerSize:], sealed[headerSize+seg:headerSize+2*seg])
+	copy(swapped[headerSize+seg:], sealed[headerSize:headerSize+seg])
+
+	cases := map[string]struct {
+		data []byte
+		key  []byte
+	}{
+		"wrong key":   {sealed, testKey(t)},
+		"flipped bit": {flipped, key},
+		"cut short":   {cut, key},
+		"swapped":     {swapped, key},
+		"header only": {sealed[:headerSize], key},
+		"not ours":    {[]byte("PK\x03\x04 a zip file, not a sealed object"), key},
+	}
+	for name, c := range cases {
+		if err := Decrypt(io.Discard, bytes.NewReader(c.data), c.key); err == nil {
+			t.Errorf("%s: decrypted without error", name)
+		}
+	}
+	if err := Decrypt(io.Discard, bytes.NewReader(sealed), testKey(t)); !errors.Is(err, ErrAuth) {
+		t.Errorf("wrong key error = %v, want ErrAuth", err)
+	}
+}
+
+func TestParseKey(t *testing.T) {
+	s, _ := NewKey()
+	k, err := ParseKey(" " + s + "\n")
+	if err != nil || len(k) != KeySize {
+		t.Fatalf("ParseKey(NewKey()) = %d bytes, %v", len(k), err)
+	}
+	for _, bad := range []string{"", "short", strings.Repeat("A", 40)} {
+		if _, err := ParseKey(bad); err == nil {
+			t.Errorf("ParseKey(%q) accepted", bad)
+		}
+	}
+	if KeyID(k) == KeyID(testKey(t)) || len(KeyID(k)) != 16 {
+		t.Errorf("KeyID = %q, want 16 hex chars that differ per key", KeyID(k))
+	}
+}
+
+// ---- fakes ----------------------------------------------------------------
+
+type memBucket struct {
+	mu      sync.Mutex
+	objs    map[string][]byte
+	putErr  map[string]error
+	puts    int
+	removed []string
+}
+
+func newMemBucket() *memBucket {
+	return &memBucket{objs: map[string][]byte{}, putErr: map[string]error{}}
+}
+
+func (b *memBucket) Put(_ context.Context, key string, r io.Reader, size int64) error {
+	if err := b.putErr[key]; err != nil {
+		io.Copy(io.Discard, r)
+		return err
+	}
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return err
+	}
+	if int64(len(data)) != size {
+		return fmt.Errorf("put %s: got %d bytes, declared %d", key, len(data), size)
+	}
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.objs[key] = data
+	b.puts++
+	return nil
+}
+
+func (b *memBucket) Get(_ context.Context, key string) (io.ReadCloser, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	data, ok := b.objs[key]
+	if !ok {
+		return nil, fmt.Errorf("%w: %s", ErrNotFound, key)
+	}
+	return io.NopCloser(bytes.NewReader(data)), nil
+}
+
+func (b *memBucket) List(_ context.Context, prefix string) ([]Object, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	var out []Object
+	for k, v := range b.objs {
+		if strings.HasPrefix(k, prefix) {
+			out = append(out, Object{Key: k, Size: int64(len(v))})
+		}
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key })
+	return out, nil
+}
+
+func (b *memBucket) Remove(_ context.Context, key string) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	delete(b.objs, key)
+	b.removed = append(b.removed, key)
+	return nil
+}
+
+type row struct {
+	WorldBackup
+	status  string
+	expires time.Time
+	offsite time.Time
+}
+
+type fakeCatalog struct{ rows []*row }
+
+func (c *fakeCatalog) PendingWorlds(context.Context) ([]WorldBackup, error) {
+	var out []WorldBackup
+	for _, r := range c.rows {
+		if r.status == "present" && r.offsite.IsZero() {
+			out = append(out, r.WorldBackup)
+		}
+	}
+	return out, nil
+}
+
+func (c *fakeCatalog) PresentWorlds(context.Context) ([]WorldBackup, error) {
+	var out []WorldBackup
+	for _, r := range c.rows {
+		if r.status == "present" {
+			out = append(out, r.WorldBackup)
+		}
+	}
+	return out, nil
+}
+
+func (c *fakeCatalog) MarkOffsite(_ context.Context, id string, at time.Time) error {
+	for _, r := range c.rows {
+		if r.ID == id {
+			r.offsite = at
+		}
+	}
+	return nil
+}
+
+func (c *fakeCatalog) ExpiredRefs(_ context.Context, now time.Time) ([]string, error) {
+	var out []string
+	for _, r := range c.rows {
+		if r.status == "deleted" && r.expires.Before(now) && !r.offsite.IsZero() {
+			out = append(out, r.Ref)
+		}
+	}
+	return out, nil
+}
+
+func writeFile(t *testing.T, dir, name string, size int) []byte {
+	t.Helper()
+	data := make([]byte, size)
+	rand.Read(data)
+	if err := os.WriteFile(filepath.Join(dir, name), data, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	return data
+}
+
+var now = time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
+
+func newSyncer(t *testing.T, cat *fakeCatalog) (*Syncer, *memBucket) {
+	t.Helper()
+	b := newMemBucket()
+	return &Syncer{
+		Bucket: b, Catalog: cat, Key: testKey(t),
+		ArchiveDir: t.TempDir(), DBDir: t.TempDir(), DBKeep: 2,
+		Now: func() time.Time { return now },
+	}, b
+}
+
+// TestSyncCopiesWorldsAndRecordsThem: each pending archive is encrypted into
+// the bucket and marked; a second run sends nothing again.
+func TestSyncCopiesWorldsAndRecordsThem(t *testing.T) {
+	cat := &fakeCatalog{rows: []*row{
+		{WorldBackup: WorldBackup{ID: "b1", Server: "alpha", Ref: "/var/lib/felis/archives/alpha-1.tar.gz"}, status: "present"},
+		{WorldBackup: WorldBackup{ID: "b2", Server: "beta", Ref: "/var/lib/felis/archives/beta-2.tar.gz"}, status: "present"},
+	}}
+	s, b := newSyncer(t, cat)
+	alpha := writeFile(t, s.ArchiveDir, "alpha-1.tar.gz", 3*segmentSize+7)
+	writeFile(t, s.ArchiveDir, "beta-2.tar.gz", 10)
+
+	res, err := s.Run(context.Background())
+	if err != nil {
+		t.Fatalf("run: %v (%+v)", err, res)
+	}
+	if res.WorldsUploaded != 2 || res.WorldsPending != 0 || res.RemoteWorlds != 2 {
+		t.Fatalf("result = %+v, want 2 uploaded, 0 pending, 2 remote", res)
+	}
+	for _, r := range cat.rows {
+		if !r.offsite.Equal(now) {
+			t.Errorf("row %s offsite_at = %v, want %v", r.ID, r.offsite, now)
+		}
+	}
+	var out bytes.Buffer
+	if err := Decrypt(&out, bytes.NewReader(b.objs["worlds/alpha-1.tar.gz.fenc"]), s.Key); err != nil || !bytes.Equal(out.Bytes(), alpha) {
+		t.Fatalf("stored alpha does not decrypt to the archive (err %v)", err)
+	}
+
+	puts := b.puts
+	if res, err := s.Run(context.Background()); err != nil || res.WorldsUploaded != 0 || b.puts != puts {
+		t.Fatalf("second run uploaded again: %+v, puts %d→%d, err %v", res, puts, b.puts, err)
+	}
+}
+
+// TestSyncResumesWithoutResending: an object stored by a run that died before
+// recording it is recorded, not uploaded twice; a partial one is replaced.
+func TestSyncResumesWithoutResending(t *testing.T) {
+	cat := &fakeCatalog{rows: []*row{
+		{WorldBackup: WorldBackup{ID: "b1", Server: "alpha", Ref: "/a/alpha-1.tar.gz"}, status: "present"},
+		{WorldBackup: WorldBackup{ID: "b2", Server: "beta", Ref: "/a/beta-2.tar.gz"}, status: "present"},
+	}}
+	s, b := newSyncer(t, cat)
+	alpha := writeFile(t, s.ArchiveDir, "alpha-1.tar.gz", 1000)
+	writeFile(t, s.ArchiveDir, "beta-2.tar.gz", 1000)
+	b.objs["worlds/alpha-1.tar.gz.fenc"] = seal(t, alpha, s.Key)
+	b.objs["worlds/beta-2.tar.gz.fenc"] = []byte("partial")
+
+	res, err := s.Run(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.WorldsUploaded != 1 || b.puts != 1 {
+		t.Fatalf("uploaded %d (puts %d), want only the partial beta resent", res.WorldsUploaded, b.puts)
+	}
+	if cat.rows[0].offsite.IsZero() || cat.rows[1].offsite.IsZero() {
+		t.Fatal("both rows should be recorded as copied")
+	}
+}
+
+// TestSyncFailureLeavesRowPending: a failed upload is reported, the row stays
+// unmarked (so the reaper keeps the world), and the rest still go.
+func TestSyncFailureLeavesRowPending(t *testing.T) {
+	cat := &fakeCatalog{rows: []*row{
+		{WorldBackup: WorldBackup{ID: "b1", Server: "alpha", Ref: "/a/alpha-1.tar.gz"}, status: "present"},
+		{WorldBackup: WorldBackup{ID: "b2", Server: "beta", Ref: "/a/beta-2.tar.gz"}, status: "present"},
+		{WorldBackup: WorldBackup{ID: "b3", Server: "gamma", Ref: "/a/gamma-3.tar.gz"}, status: "present"},
+	}}
+	s, b := newSyncer(t, cat)
+	writeFile(t, s.ArchiveDir, "alpha-1.tar.gz", 100)
+	writeFile(t, s.ArchiveDir, "beta-2.tar.gz", 100)
+	b.putErr["worlds/alpha-1.tar.gz.fenc"] = errors.New("503 slow down")
+
+	res, err := s.Run(context.Background())
+	if err == nil {
+		t.Fatal("run reported success despite a failed upload")
+	}
+	if !cat.rows[0].offsite.IsZero() {
+		t.Error("the failed archive was recorded as copied")
+	}
+	if cat.rows[1].offsite.IsZero() {
+		t.Error("the archive after the failure was not copied")
+	}
+	if res.WorldsPending != 1 || len(res.WorldsMissing) != 1 || !strings.Contains(res.WorldsMissing[0], "gamma") {
+		t.Fatalf("result = %+v, want alpha pending, gamma missing", res)
+	}
+}
+
+// TestSyncExpiresOnlyPastRetention: a remote archive goes once its row has
+// expired; one evicted early from the local disk stays until then, and an
+// object with no row at all is left alone.
+func TestSyncExpiresOnlyPastRetention(t *testing.T) {
+	cat := &fakeCatalog{rows: []*row{
+		{WorldBackup: WorldBackup{ID: "old", Ref: "/a/old.tar.gz"}, status: "deleted", expires: now.Add(-time.Hour), offsite: now.Add(-100 * 24 * time.Hour)},
+		{WorldBackup: WorldBackup{ID: "evicted", Ref: "/a/evicted.tar.gz"}, status: "deleted", expires: now.Add(30 * 24 * time.Hour), offsite: now.Add(-24 * time.Hour)},
+	}}
+	s, b := newSyncer(t, cat)
+	for _, k := range []string{"worlds/old.tar.gz.fenc", "worlds/evicted.tar.gz.fenc", "worlds/unknown.tar.gz.fenc"} {
+		b.objs[k] = []byte("x")
+	}
+	res, err := s.Run(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.WorldsExpired != 1 || len(b.removed) != 1 || b.removed[0] != "worlds/old.tar.gz.fenc" {
+		t.Fatalf("removed %v (expired %d), want only the expired archive", b.removed, res.WorldsExpired)
+	}
+	if res.RemoteWorlds != 2 {
+		t.Fatalf("remote worlds = %d, want 2 left", res.RemoteWorlds)
+	}
+}
+
+// TestSyncDBKeepsNewest: only the newest DBKeep bundles are sent, and older
+// remote bundles are pruned down to DBKeep.
+func TestSyncDBKeepsNewest(t *testing.T) {
+	s, b := newSyncer(t, &fakeCatalog{})
+	for _, name := range []string{
+		"felis-db-20260920T030000Z-daily.tar",
+		"felis-db-20260922T030000Z-daily.tar",
+		"felis-db-20260923T030000Z-manual.tar",
+		"felis-db-20260924T030000Z-daily.tar",
+	} {
+		writeFile(t, s.DBDir, name, 50)
+	}
+	b.objs["db/felis-db-20260901T030000Z-daily.tar.fenc"] = []byte("old")
+	b.objs["db/notes.txt"] = []byte("not ours")
+
+	res, err := s.Run(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.DBUploaded != 2 || res.RemoteDB != 2 || res.NewestDB != "felis-db-20260924T030000Z-daily.tar" {
+		t.Fatalf("result = %+v, want the newest 2 uploaded", res)
+	}
+	var keys []string
+	for k := range b.objs {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	want := "db/felis-db-20260923T030000Z-manual.tar.fenc db/felis-db-20260924T030000Z-daily.tar.fenc db/notes.txt"
+	if strings.Join(keys, " ") != want {
+		t.Fatalf("bucket = %v, want %s", keys, want)
+	}
+	listed, err := ListDB(context.Background(), b)
+	if err != nil || len(listed) != 2 || listed[0].Key != "felis-db-20260924T030000Z-daily.tar" {
+		t.Fatalf("ListDB = %+v, %v", listed, err)
+	}
+}
+
+// TestFetchWorldsRestoresVolume: after a rebuild, every present archive the
+// volume lacks comes back byte for byte; ones already there are left alone.
+func TestFetchWorldsRestoresVolume(t *testing.T) {
+	cat := &fakeCatalog{rows: []*row{
+		{WorldBackup: WorldBackup{ID: "b1", Server: "alpha", Ref: "/a/alpha-1.tar.gz"}, status: "present"},
+		{WorldBackup: WorldBackup{ID: "b2", Server: "beta", Ref: "/a/beta-2.tar.gz"}, status: "present"},
+		{WorldBackup: WorldBackup{ID: "b3", Server: "gamma", Ref: "/a/gamma-3.tar.gz"}, status: "present"},
+	}}
+	s, b := newSyncer(t, cat)
+	alpha := writeFile(t, s.ArchiveDir, "alpha-1.tar.gz", segmentSize+3)
+	writeFile(t, s.ArchiveDir, "beta-2.tar.gz", 20)
+	if res, err := s.Run(context.Background()); err != nil || len(res.WorldsMissing) != 1 {
+		t.Fatalf("run = %+v, %v; want gamma reported missing", res, err)
+	}
+
+	fresh := t.TempDir()
+	os.WriteFile(filepath.Join(fresh, "beta-2.tar.gz"), []byte("kept"), 0o600)
+	res, err := FetchWorlds(context.Background(), b, cat, s.Key, fresh, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(res.Fetched, ",") != "alpha-1.tar.gz" || len(res.Missing) != 1 {
+		t.Fatalf("fetch = %+v, want alpha fetched, gamma missing", res)
+	}
+	got, _ := os.ReadFile(filepath.Join(fresh, "alpha-1.tar.gz"))
+	if !bytes.Equal(got, alpha) {
+		t.Fatal("fetched alpha differs from the original")
+	}
+	if kept, _ := os.ReadFile(filepath.Join(fresh, "beta-2.tar.gz")); string(kept) != "kept" {
+		t.Fatal("fetch overwrote an archive already on the volume")
+	}
+
+	// A wrong key leaves no file behind, partial or whole.
+	other := t.TempDir()
+	if err := FetchObject(context.Background(), b, testKey(t), "worlds/alpha-1.tar.gz.fenc", filepath.Join(other, "alpha-1.tar.gz"), 0o600); !errors.Is(err, ErrAuth) {
+		t.Fatalf("wrong key fetch = %v, want ErrAuth", err)
+	}
+	if entries, _ := os.ReadDir(other); len(entries) != 0 {
+		t.Fatalf("wrong key left files: %v", entries)
+	}
+}
+
+func TestWorldKeyRejectsOddRefs(t *testing.T) {
+	if k, ok := WorldKey("/var/lib/felis/archives/alpha-1.tar.gz"); !ok || k != "worlds/alpha-1.tar.gz.fenc" {
+		t.Fatalf("WorldKey = %q, %v", k, ok)
+	}
+	for _, ref := range []string{"", "/", "..", "/a/.."} {
+		if _, ok := WorldKey(ref); ok {
+			t.Errorf("WorldKey(%q) accepted", ref)
+		}
+	}
+}

internal/offsite/pgcatalog.go

@@ -0,0 +1,73 @@
+package offsite
+
+import (
+	"context"
+	"database/sql"
+	"time"
+)
+
+// PGCatalog is the Catalog on the felis database.
+type PGCatalog struct{ DB *sql.DB }
+
+func (c PGCatalog) PendingWorlds(ctx context.Context) ([]WorldBackup, error) {
+	return c.query(ctx, `SELECT id, server_name, backup_ref, created_at FROM world_backups
+		WHERE status = 'present' AND offsite_at IS NULL ORDER BY created_at`)
+}
+
+func (c PGCatalog) PresentWorlds(ctx context.Context) ([]WorldBackup, error) {
+	return c.query(ctx, `SELECT id, server_name, backup_ref, created_at FROM world_backups
+		WHERE status = 'present' ORDER BY created_at`)
+}
+
+func (c PGCatalog) query(ctx context.Context, q string, args ...any) ([]WorldBackup, error) {
+	rows, err := c.DB.QueryContext(ctx, q, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []WorldBackup
+	for rows.Next() {
+		var w WorldBackup
+		if err := rows.Scan(&w.ID, &w.Server, &w.Ref, &w.Created); err != nil {
+			return nil, err
+		}
+		out = append(out, w)
+	}
+	return out, rows.Err()
+}
+
+func (c PGCatalog) MarkOffsite(ctx context.Context, id string, at time.Time) error {
+	_, err := c.DB.ExecContext(ctx,
+		`UPDATE world_backups SET offsite_at = $2 WHERE id = $1 AND offsite_at IS NULL`, id, at)
+	return err
+}
+
+func (c PGCatalog) ExpiredRefs(ctx context.Context, now time.Time) ([]string, error) {
+	rows, err := c.DB.QueryContext(ctx, `SELECT backup_ref FROM world_backups
+		WHERE status = 'deleted' AND expires_at < $1 AND offsite_at IS NOT NULL`, now)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []string
+	for rows.Next() {
+		var ref string
+		if err := rows.Scan(&ref); err != nil {
+			return nil, err
+		}
+		out = append(out, ref)
+	}
+	return out, rows.Err()
+}
+
+// PendingCount is how many present archives wait for their copy, and since
+// when the oldest has waited.
+func (c PGCatalog) PendingCount(ctx context.Context) (int, time.Time, error) {
+	var (
+		n      int
+		oldest sql.NullTime
+	)
+	err := c.DB.QueryRowContext(ctx, `SELECT count(*), min(created_at) FROM world_backups
+		WHERE status = 'present' AND offsite_at IS NULL`).Scan(&n, &oldest)
+	return n, oldest.Time, err
+}

internal/offsite/status.go

@@ -0,0 +1,64 @@
+package offsite
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// DefaultStatusFile is where `felis offsite sync` records its last run; the
+// watchdog and `felis offsite status` read it.
+const DefaultStatusFile = "/var/lib/felis/offsite/status.json"
+
+// StaleAfter is how long the sync may go without a clean run before the
+// watchdog mails the owners. The timer runs hourly, so this rides out a
+// provider's bad morning, and still leaves a day's database bundle uncopied
+// for at most half a day.
+const StaleAfter = 12 * time.Hour
+
+// Status is the record of the last run.
+type Status struct {
+	LastAttempt time.Time `json:"last_attempt"`
+	// LastSuccess is the last run in which every step succeeded.
+	LastSuccess time.Time `json:"last_success,omitempty"`
+	LastError   string    `json:"last_error,omitempty"`
+	Endpoint    string    `json:"endpoint"`
+	Bucket      string    `json:"bucket"`
+	Prefix      string    `json:"prefix,omitempty"`
+	KeyID       string    `json:"key_id"`
+	Result      Result    `json:"result"`
+}
+
+// ReadStatus reads the status file. A missing file is (nil, nil): no sync has
+// run yet.
+func ReadStatus(p string) (*Status, error) {
+	raw, err := os.ReadFile(p)
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	var st Status
+	if err := json.Unmarshal(raw, &st); err != nil {
+		return nil, err
+	}
+	return &st, nil
+}
+
+// WriteStatus replaces the status file atomically.
+func WriteStatus(p string, st Status) error {
+	if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil {
+		return err
+	}
+	raw, err := json.MarshalIndent(st, "", "  ")
+	if err != nil {
+		return err
+	}
+	tmp := p + ".tmp"
+	if err := os.WriteFile(tmp, append(raw, '\n'), 0o600); err != nil {
+		return err
+	}
+	return os.Rename(tmp, p)
+}

internal/offsite/sync.go

@@ -0,0 +1,464 @@
+// Package offsite keeps a second copy of what a lost node would take with it:
+// every world archive (world_backups) and the newest control-plane database
+// bundles (internal/dbbackup), encrypted, in an S3-compatible bucket off the
+// machine. `felis offsite sync` runs it from felis-offsite.timer on the host,
+// which is where both the archive volume and the bundle directory live.
+//
+// The database records the copy: world_backups.offsite_at is set once an
+// archive's object is in the bucket, and with [offsite] configured the reaper
+// deletes an idle world only after that (internal/reaper). Remote world
+// objects go when their row has expired, so the bucket keeps each archive for
+// the same retention the panel promises, including archives evicted early from
+// the local disk to make room. Remote bundles are pruned to the newest DBKeep.
+package offsite
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+	"syscall"
+	"time"
+
+	"felis.lolicon.best/internal/dbbackup"
+)
+
+// Object key layout under the configured prefix.
+const (
+	worldsDir = "worlds/"
+	dbDir     = "db/"
+	objExt    = ".fenc"
+)
+
+// WorldKey is the object key of the world archive stored at ref (the
+// world_backups.backup_ref, an in-pod path whose last element is the file name
+// on the archive volume).
+func WorldKey(ref string) (string, bool) {
+	name := path.Base(ref)
+	if !safeName(name) {
+		return "", false
+	}
+	return worldsDir + name + objExt, true
+}
+
+// DBKey is the object key of a database bundle.
+func DBKey(bundle string) string { return dbDir + bundle + objExt }
+
+func safeName(name string) bool {
+	return name != "" && name != "." && name != ".." && name != "/" && !strings.ContainsAny(name, `/\`)
+}
+
+// WorldBackup is one world_backups row the sync works on.
+type WorldBackup struct {
+	ID      string
+	Server  string
+	Ref     string
+	Created time.Time
+}
+
+// Catalog is the world_backups view the sync needs; PGCatalog in production.
+type Catalog interface {
+	// PendingWorlds lists present archives without an off-site copy yet,
+	// oldest first.
+	PendingWorlds(ctx context.Context) ([]WorldBackup, error)
+	// MarkOffsite records that the archive of row id is in the bucket.
+	MarkOffsite(ctx context.Context, id string, at time.Time) error
+	// ExpiredRefs lists the backup_ref of every row past its retention
+	// (deleted and expires_at < now) whose archive was copied off-site.
+	ExpiredRefs(ctx context.Context, now time.Time) ([]string, error)
+	// PresentWorlds lists every present archive, for a restore of the volume.
+	PresentWorlds(ctx context.Context) ([]WorldBackup, error)
+}
+
+// Syncer copies what is missing from the bucket and prunes what has expired.
+type Syncer struct {
+	Bucket  Bucket
+	Catalog Catalog
+	Key     []byte
+	// ArchiveDir is the host directory of the world archive volume; "" when
+	// there is none yet (nothing has been archived on this install).
+	ArchiveDir string
+	// DBDir holds the database bundles; DBKeep is how many of the newest the
+	// bucket keeps.
+	DBDir  string
+	DBKeep int
+	Now    func() time.Time
+	Log    io.Writer
+}
+
+// Result is what one Run did and found.
+type Result struct {
+	WorldsUploaded int   `json:"worlds_uploaded"`
+	BytesUploaded  int64 `json:"bytes_uploaded"`
+	// WorldsPending are present archives still without an off-site copy
+	// after this run; the missing ones below are counted there alone.
+	WorldsPending int `json:"worlds_pending"`
+	// WorldsMissing are present rows whose archive file is not on the volume:
+	// nothing to copy, and nothing a restore could use.
+	WorldsMissing []string `json:"worlds_missing,omitempty"`
+	WorldsExpired int      `json:"worlds_expired"`
+	RemoteWorlds  int      `json:"remote_worlds"`
+	RemoteBytes   int64    `json:"remote_bytes"`
+	DBUploaded    int      `json:"db_uploaded"`
+	DBPruned      int      `json:"db_pruned"`
+	RemoteDB      int      `json:"remote_db"`
+	NewestDB      string   `json:"newest_db,omitempty"`
+	Errors        []string `json:"errors,omitempty"`
+}
+
+func (s *Syncer) now() time.Time {
+	if s.Now != nil {
+		return s.Now()
+	}
+	return time.Now()
+}
+
+func (s *Syncer) logf(format string, args ...any) {
+	if s.Log != nil {
+		fmt.Fprintf(s.Log, "felis offsite: "+format+"\n", args...)
+	}
+}
+
+// Run does one pass: world archives, then database bundles, then expiry. A
+// failure on one item is recorded and the pass carries on; the returned error
+// is non-nil when anything failed.
+func (s *Syncer) Run(ctx context.Context) (Result, error) {
+	var res Result
+	fail := func(format string, args ...any) {
+		msg := fmt.Sprintf(format, args...)
+		res.Errors = append(res.Errors, msg)
+		s.logf("%s", msg)
+	}
+
+	remoteWorlds, err := s.listSizes(ctx, worldsDir)
+	if err != nil {
+		return res, fmt.Errorf("list %s in the bucket: %w", worldsDir, err)
+	}
+	s.syncWorlds(ctx, remoteWorlds, &res, fail)
+	s.syncDB(ctx, &res, fail)
+	s.expireWorlds(ctx, remoteWorlds, &res, fail)
+
+	for _, size := range remoteWorlds {
+		res.RemoteWorlds++
+		res.RemoteBytes += size
+	}
+	if len(res.Errors) > 0 {
+		return res, fmt.Errorf("%d of this run's steps failed; first: %s", len(res.Errors), res.Errors[0])
+	}
+	return res, nil
+}
+
+func (s *Syncer) listSizes(ctx context.Context, prefix string) (map[string]int64, error) {
+	objs, err := s.Bucket.List(ctx, prefix)
+	if err != nil {
+		return nil, err
+	}
+	out := make(map[string]int64, len(objs))
+	for _, o := range objs {
+		out[o.Key] = o.Size
+	}
+	return out, nil
+}
+
+func (s *Syncer) syncWorlds(ctx context.Context, remote map[string]int64, res *Result, fail func(string, ...any)) {
+	pending, err := s.Catalog.PendingWorlds(ctx)
+	if err != nil {
+		fail("list world archives waiting for a copy: %v", err)
+		return
+	}
+	if s.ArchiveDir == "" {
+		res.WorldsPending = len(pending)
+		if len(pending) > 0 {
+			fail("%d world archives wait for a copy, but there is no archive volume to read them from", len(pending))
+		}
+		return
+	}
+	for _, w := range pending {
+		if ctx.Err() != nil {
+			fail("stopped: %v", ctx.Err())
+			res.WorldsPending++
+			continue
+		}
+		key, ok := WorldKey(w.Ref)
+		if !ok {
+			fail("world archive %s of %s has an unusable path %q", w.ID, w.Server, w.Ref)
+			res.WorldsPending++
+			continue
+		}
+		local := filepath.Join(s.ArchiveDir, path.Base(w.Ref))
+		fi, err := os.Stat(local)
+		if errors.Is(err, os.ErrNotExist) {
+			res.WorldsMissing = append(res.WorldsMissing, fmt.Sprintf("%s (%s, backup %s)", path.Base(w.Ref), w.Server, w.ID))
+			continue
+		}
+		if err != nil {
+			fail("world archive %s: %v", local, err)
+			res.WorldsPending++
+			continue
+		}
+		want := SealedSize(fi.Size())
+		// An earlier run may have stored the object and died before recording
+		// it; a complete object is only recorded, not sent again.
+		if size, ok := remote[key]; !ok || size != want {
+			if err := s.putFile(ctx, key, local, fi.Size()); err != nil {
+				fail("upload %s (%s): %v", path.Base(w.Ref), w.Server, err)
+				res.WorldsPending++
+				continue
+			}
+			remote[key] = want
+			res.WorldsUploaded++
+			res.BytesUploaded += fi.Size()
+			s.logf("copied world archive %s (%s, %s)", path.Base(w.Ref), w.Server, HumanBytes(fi.Size()))
+		}
+		if err := s.Catalog.MarkOffsite(ctx, w.ID, s.now()); err != nil {
+			fail("record the copy of %s: %v", path.Base(w.Ref), err)
+			res.WorldsPending++
+		}
+	}
+}
+
+func (s *Syncer) syncDB(ctx context.Context, res *Result, fail func(string, ...any)) {
+	if s.DBDir == "" {
+		return
+	}
+	keep := s.DBKeep
+	if keep < 1 {
+		keep = 1
+	}
+	local, err := dbbackup.List(s.DBDir)
+	if err != nil {
+		fail("list database bundles in %s: %v", s.DBDir, err)
+		return
+	}
+	remote, err := s.listSizes(ctx, dbDir)
+	if err != nil {
+		fail("list %s in the bucket: %v", dbDir, err)
+		return
+	}
+	// Only the newest keep bundles are worth sending: older ones would be
+	// pruned again at the end of this very pass.
+	if len(local) > keep {
+		local = local[:keep]
+	}
+	for _, b := range local {
+		key := DBKey(b.Name)
+		want := SealedSize(b.Size)
+		if size, ok := remote[key]; ok && size == want {
+			continue
+		}
+		if err := s.putFile(ctx, key, b.Path, b.Size); err != nil {
+			fail("upload database bundle %s: %v", b.Name, err)
+			continue
+		}
+		remote[key] = want
+		res.DBUploaded++
+		res.BytesUploaded += b.Size
+		s.logf("copied database bundle %s (%s)", b.Name, HumanBytes(b.Size))
+	}
+
+	var names []string
+	for key := range remote {
+		name := strings.TrimSuffix(strings.TrimPrefix(key, dbDir), objExt)
+		if _, _, ok := dbbackup.ParseBundleName(name); ok && strings.HasSuffix(key, objExt) {
+			names = append(names, name)
+		}
+	}
+	// Bundle names start with their UTC stamp, so newest sorts last.
+	sort.Sort(sort.Reverse(sort.StringSlice(names)))
+	for i, name := range names {
+		if i < keep {
+			continue
+		}
+		if err := s.Bucket.Remove(ctx, DBKey(name)); err != nil {
+			fail("prune database bundle %s: %v", name, err)
+			continue
+		}
+		res.DBPruned++
+	}
+	res.RemoteDB = min(len(names), keep)
+	if len(names) > 0 {
+		res.NewestDB = names[0]
+	}
+}
+
+func (s *Syncer) expireWorlds(ctx context.Context, remote map[string]int64, res *Result, fail func(string, ...any)) {
+	refs, err := s.Catalog.ExpiredRefs(ctx, s.now())
+	if err != nil {
+		fail("list expired world archives: %v", err)
+		return
+	}
+	for _, ref := range refs {
+		key, ok := WorldKey(ref)
+		if !ok {
+			continue
+		}
+		if _, ok := remote[key]; !ok {
+			continue
+		}
+		if err := s.Bucket.Remove(ctx, key); err != nil {
+			fail("remove expired %s: %v", key, err)
+			continue
+		}
+		delete(remote, key)
+		res.WorldsExpired++
+	}
+}
+
+// putFile encrypts the file at p into key. The sealed size is known in
+// advance, so the upload streams: nothing larger than one part is buffered.
+func (s *Syncer) putFile(ctx context.Context, key, p string, size int64) error {
+	f, err := os.Open(p)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	pr, pw := io.Pipe()
+	go func() {
+		// A file that changed size under us would not match the declared
+		// length; LimitReader keeps the stream to the size we announced and
+		// the length check below catches a short one.
+		err := Encrypt(pw, io.LimitReader(f, size), s.Key)
+		pw.CloseWithError(err)
+	}()
+	err = s.Bucket.Put(ctx, key, pr, SealedSize(size))
+	pr.CloseWithError(errors.New("upload finished"))
+	return err
+}
+
+// FetchResult is what Fetch did.
+type FetchResult struct {
+	Fetched  []string
+	Present  int
+	Missing  []string // present rows with no object in the bucket
+	Failures []string
+}
+
+// FetchWorlds downloads every present archive the volume lacks: the volume
+// half of a rebuild, after the database came back from a bundle.
+func FetchWorlds(ctx context.Context, b Bucket, cat Catalog, key []byte, archiveDir string, log io.Writer) (FetchResult, error) {
+	var res FetchResult
+	worlds, err := cat.PresentWorlds(ctx)
+	if err != nil {
+		return res, err
+	}
+	res.Present = len(worlds)
+	for _, w := range worlds {
+		objKey, ok := WorldKey(w.Ref)
+		if !ok {
+			res.Failures = append(res.Failures, fmt.Sprintf("%s: unusable path %q", w.ID, w.Ref))
+			continue
+		}
+		dst := filepath.Join(archiveDir, path.Base(w.Ref))
+		if _, err := os.Stat(dst); err == nil {
+			continue
+		}
+		// World-readable like the archives the backup Jobs write: the restore
+		// Job reads them as its own non-root user.
+		switch err := FetchObject(ctx, b, key, objKey, dst, 0o644); {
+		case errors.Is(err, ErrNotFound):
+			res.Missing = append(res.Missing, fmt.Sprintf("%s (%s)", path.Base(w.Ref), w.Server))
+		case err != nil:
+			res.Failures = append(res.Failures, fmt.Sprintf("%s: %v", path.Base(w.Ref), err))
+		default:
+			alignOwner(dst, archiveDir)
+			res.Fetched = append(res.Fetched, path.Base(w.Ref))
+			if log != nil {
+				fmt.Fprintf(log, "felis offsite: restored %s (%s)\n", path.Base(w.Ref), w.Server)
+			}
+		}
+	}
+	if len(res.Failures) > 0 {
+		return res, fmt.Errorf("%d archives failed; first: %s", len(res.Failures), res.Failures[0])
+	}
+	return res, nil
+}
+
+// alignOwner gives a restored archive the archive volume's owner, which is the
+// user the backup Jobs write as, so the volume looks as they left it. Only root
+// can; for anyone else the file stays theirs, readable all the same.
+func alignOwner(file, dir string) {
+	if os.Geteuid() != 0 {
+		return
+	}
+	fi, err := os.Stat(dir)
+	if err != nil {
+		return
+	}
+	if st, ok := fi.Sys().(*syscall.Stat_t); ok {
+		_ = os.Lchown(file, int(st.Uid), int(st.Gid))
+	}
+}
+
+// FetchObject downloads and decrypts key into dst, created with mode. The file
+// appears under its name only once it has decrypted completely; until then it
+// is a hidden .partial next to it.
+func FetchObject(ctx context.Context, b Bucket, key []byte, objKey, dst string, mode os.FileMode) error {
+	rc, err := b.Get(ctx, objKey)
+	if err != nil {
+		return err
+	}
+	defer rc.Close()
+	tmp := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".partial")
+	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
+	if err != nil {
+		return err
+	}
+	if err := f.Chmod(mode); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := Decrypt(f, rc, key); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	return os.Rename(tmp, dst)
+}
+
+// ListDB returns the database bundles in the bucket, newest first.
+func ListDB(ctx context.Context, b Bucket) ([]Object, error) {
+	objs, err := b.List(ctx, dbDir)
+	if err != nil {
+		return nil, err
+	}
+	var out []Object
+	for _, o := range objs {
+		name := strings.TrimSuffix(strings.TrimPrefix(o.Key, dbDir), objExt)
+		if _, _, ok := dbbackup.ParseBundleName(name); !ok || !strings.HasSuffix(o.Key, objExt) {
+			continue
+		}
+		o.Key = name
+		out = append(out, o)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Key > out[j].Key })
+	return out, nil
+}
+
+// HumanBytes formats n in binary units.
+func HumanBytes(n int64) string {
+	const unit = 1024
+	if n < unit {
+		return fmt.Sprintf("%d B", n)
+	}
+	div, exp := int64(unit), 0
+	for m := n / unit; m >= unit; m /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
+}

internal/platform/workloads.go

@@ -1029,6 +1029,43 @@ func backupPVC(p Params) *corev1.PersistentVolumeClaim {
 	}
 }
 
+// VolumeBinderPod mounts pvc and exits at once. A WaitForFirstConsumer volume
+// (k3s local-path) has no directory until a pod uses it, and on a rebuilt node
+// nothing has yet, so `felis offsite fetch-worlds` runs this to have the
+// archive volume provisioned before it writes the archives back into it. It
+// runs as the control-plane identity, which the archive volume's files already
+// belong to (fsGroup).
+func VolumeBinderPod(ns, pvc, image string) *corev1.Pod {
+	return &corev1.Pod{
+		TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "felis-bind-" + pvc + "-",
+			Namespace:    ns,
+			Labels:       map[string]string{LabelName: appName, LabelComponent: "volume-binder"},
+		},
+		Spec: corev1.PodSpec{
+			RestartPolicy:                corev1.RestartPolicyNever,
+			AutomountServiceAccountToken: boolPtr(false),
+			SecurityContext:              hardenedPodSecurityContext(),
+			Containers: []corev1.Container{{
+				Name:            "bind",
+				Image:           image,
+				Command:         []string{felisBinaryPath, "version"},
+				SecurityContext: hardenedContainerSecurityContext(),
+				VolumeMounts:    []corev1.VolumeMount{{Name: "archives", MountPath: "/archives"}},
+				Resources: corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("10m"), corev1.ResourceMemory: resource.MustParse("16Mi")},
+					Limits:   corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("64Mi")},
+				},
+			}},
+			Volumes: []corev1.Volume{{
+				Name:         "archives",
+				VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc}},
+			}},
+		},
+	}
+}
+
 // registryLabels are the registry's recommended labels. Note the absence of
 // part-of=felis-control-plane: that is what keeps the registry out of the RCON
 // NetworkPolicy peer's reach (asserted in workloads_test.go).

internal/platform/workloads_test.go

@@ -164,6 +164,35 @@ func TestControlPlanePods_Hardened(t *testing.T) {
 	}
 }
 
+// TestVolumeBinderPod pins the pod `felis offsite fetch-worlds` runs to get the
+// archive volume provisioned: hardened like the control plane (it runs in the
+// Minecraft namespace under PSA), mounting the named claim, gone once it exits.
+func TestVolumeBinderPod(t *testing.T) {
+	pod := VolumeBinderPod("minecraft", "felis-backups", "registry.example/felis:1")
+	ps := pod.Spec
+	if pod.Namespace != "minecraft" || pod.GenerateName == "" || ps.RestartPolicy != corev1.RestartPolicyNever {
+		t.Fatalf("pod meta = %+v, restart %s", pod.ObjectMeta, ps.RestartPolicy)
+	}
+	if ps.SecurityContext == nil || ps.SecurityContext.RunAsNonRoot == nil || !*ps.SecurityContext.RunAsNonRoot ||
+		ps.SecurityContext.SeccompProfile == nil || ps.SecurityContext.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault {
+		t.Errorf("pod security context = %+v", ps.SecurityContext)
+	}
+	if ps.AutomountServiceAccountToken == nil || *ps.AutomountServiceAccountToken {
+		t.Error("the binder needs no API token")
+	}
+	c := ps.Containers[0]
+	if got := append(append([]string{}, c.Command...), c.Args...); !containsSeq(got, []string{felisBinaryPath, "version"}) {
+		t.Errorf("command = %v", got)
+	}
+	if sc := c.SecurityContext; sc == nil || sc.AllowPrivilegeEscalation == nil || *sc.AllowPrivilegeEscalation ||
+		sc.Capabilities == nil || len(sc.Capabilities.Drop) == 0 || sc.Capabilities.Drop[0] != "ALL" {
+		t.Errorf("container security context = %+v", c.SecurityContext)
+	}
+	if len(ps.Volumes) != 1 || ps.Volumes[0].PersistentVolumeClaim == nil || ps.Volumes[0].PersistentVolumeClaim.ClaimName != "felis-backups" {
+		t.Errorf("volumes = %+v", ps.Volumes)
+	}
+}
+
 // TestAPIDeployment_Wiring pins the api entrypoint, the credential plumbing, and
 // the FELIS_IMAGE passthrough.
 func TestAPIDeployment_Wiring(t *testing.T) {

internal/reaper/pgstore.go

@@ -49,18 +49,18 @@ func (s *PGStore) ListActiveServers(ctx context.Context) ([]Candidate, error) {
 	return out, rows.Err()
 }
 
-func (s *PGStore) FreshBackup(ctx context.Context, server string, since time.Time) (string, bool, error) {
-	const q = `SELECT backup_ref FROM world_backups
+func (s *PGStore) FreshBackup(ctx context.Context, server string, since time.Time) (Fresh, bool, error) {
+	const q = `SELECT backup_ref, offsite_at IS NOT NULL FROM world_backups
 		WHERE server_name = $1 AND status = 'present' AND created_at >= $2
-		ORDER BY created_at DESC LIMIT 1`
-	var ref string
-	switch err := s.db.QueryRowContext(ctx, q, server, since).Scan(&ref); {
+		ORDER BY offsite_at IS NOT NULL DESC, created_at DESC LIMIT 1`
+	var f Fresh
+	switch err := s.db.QueryRowContext(ctx, q, server, since).Scan(&f.Ref, &f.Offsite); {
 	case err == sql.ErrNoRows:
-		return "", false, nil
+		return Fresh{}, false, nil
 	case err != nil:
-		return "", false, err
+		return Fresh{}, false, err
 	}
-	return ref, true, nil
+	return f, true, nil
 }
 
 func (s *PGStore) InsertBackup(ctx context.Context, rec BackupRecord) error {

internal/reaper/reaper.go

@@ -81,6 +81,12 @@ type Config struct {
 	WarnBefore     []time.Duration // §24: warn these long before the deadline (default 3d, 1d)
 	Retention      time.Duration   // §18: keep a backup this long after deletion (default 3mo≈90d)
 	MaxLocalBytes  int64           // §26: backup store soft cap; 0 = unlimited
+	// RequireOffsite holds each deletion until the world's archive has its
+	// off-site copy ([offsite] configured; internal/offsite records the copy).
+	// The archive is written on the run that finds the world idle, and the
+	// world is deleted on the first run after the copy lands, normally the
+	// next day.
+	RequireOffsite bool
 }
 
 // DefaultConfig is the spec's §24 default window set.
@@ -131,6 +137,13 @@ type BackupRecord struct {
 	ExpiresAt   time.Time
 }
 
+// Fresh is the backup FreshBackup found.
+type Fresh struct {
+	Ref string
+	// Offsite reports that the archive has its off-site copy (offsite_at).
+	Offsite bool
+}
+
 // StoredBackup is an existing world_backups row, used by both the expiry pass
 // and the capacity-eviction path.
 type StoredBackup struct {
@@ -156,10 +169,11 @@ type Store interface {
 	ListActiveServers(ctx context.Context) ([]Candidate, error)
 
 	// FreshBackup reports an existing present backup for server whose world is
-	// still current — created at or after since (the world's last_active_at).
-	// It makes a reap idempotent across a DeletePVC failure: the retry reuses
-	// the archive instead of writing a duplicate.
-	FreshBackup(ctx context.Context, server string, since time.Time) (ref string, ok bool, err error)
+	// still current — created at or after since (the world's last_active_at),
+	// preferring one already copied off-site. It makes a reap idempotent across
+	// a DeletePVC failure, and across the wait for the off-site copy: the retry
+	// reuses the archive instead of writing a duplicate.
+	FreshBackup(ctx context.Context, server string, since time.Time) (b Fresh, ok bool, err error)
 
 	// InsertBackup records a world_backups row (status=present).
 	InsertBackup(ctx context.Context, rec BackupRecord) error
@@ -232,6 +246,9 @@ type Summary struct {
 	Skipped        int // exempt, CRD gone, or could not back up
 	EvictedEarly   int
 	BackupsExpired int
+	// AwaitingOffsite are idle worlds that are archived and kept until the
+	// archive's off-site copy lands.
+	AwaitingOffsite int
 }
 
 func (r *Reaper) now() time.Time {
@@ -336,10 +353,11 @@ func (r *Reaper) reap(ctx context.Context, now time.Time, c Candidate, crd Serve
 	// world but failed before deleting the PVC, reuse that backup rather than
 	// writing a duplicate. The world has not changed since last_active_at, so
 	// any present backup created after it still describes the current world.
-	ref, ok, err := r.Store.FreshBackup(ctx, c.Name, c.LastActiveAt)
+	fresh, ok, err := r.Store.FreshBackup(ctx, c.Name, c.LastActiveAt)
 	if err != nil {
 		return fmt.Errorf("lookup fresh backup: %w", err)
 	}
+	ref, offsite := fresh.Ref, fresh.Offsite
 	if !ok {
 		aref, size, err := r.Archiver.Archive(ctx, c.Name, crd.PVC)
 		if err != nil {
@@ -364,7 +382,15 @@ func (r *Reaper) reap(ctx context.Context, now time.Time, c Candidate, crd Serve
 			}
 			return fmt.Errorf("insert backup: %w", err)
 		}
-		ref = string(aref)
+		ref, offsite = string(aref), false
+	}
+
+	// With an off-site bucket configured, the archive on this node's disk is
+	// not enough on its own: a lost disk would take it along with the world.
+	if r.Cfg.RequireOffsite && !offsite {
+		sum.AwaitingOffsite++
+		r.log().Info("reaper: world archived, kept until the archive's off-site copy is confirmed", "server", c.Name, "backup_ref", ref)
+		return nil
 	}
 
 	// World is safely archived and recorded — now (and only now) delete it.

internal/reaper/reaper_test.go

@@ -107,6 +107,7 @@ type fakeBackup struct {
 	size               int64
 	status             string // present | deleted
 	createdAt, expires time.Time
+	offsite            bool
 }
 
 type fakeStore struct {
@@ -133,13 +134,19 @@ func (s *fakeStore) ListActiveServers(context.Context) ([]Candidate, error) {
 	return out, nil
 }
 
-func (s *fakeStore) FreshBackup(_ context.Context, server string, since time.Time) (string, bool, error) {
+func (s *fakeStore) FreshBackup(_ context.Context, server string, since time.Time) (Fresh, bool, error) {
+	var found *fakeBackup
 	for _, b := range s.backups {
 		if b.server == server && b.status == "present" && !b.createdAt.Before(since) {
-			return b.ref, true, nil
+			if found == nil || (b.offsite && !found.offsite) {
+				found = b
+			}
 		}
 	}
-	return "", false, nil
+	if found == nil {
+		return Fresh{}, false, nil
+	}
+	return Fresh{Ref: found.ref, Offsite: found.offsite}, true, nil
 }
 
 func (s *fakeStore) InsertBackup(_ context.Context, rec BackupRecord) error {
@@ -426,6 +433,38 @@ func TestReapDeletePVCFailureIsIdempotent(t *testing.T) {
 	}
 }
 
+// With an off-site bucket configured the reaper archives an idle world but
+// keeps it until the archive's copy is confirmed; the next run reuses that
+// archive and deletes the world, without archiving it again.
+func TestReapWaitsForOffsiteCopy(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.RequireOffsite = true
+	r, st, cl, ar := newReaper(cfg,
+		Candidate{Name: "echo", OwnerID: "user-5", LastActiveAt: idleBy(20 * Day)})
+
+	sum := mustRun(t, r)
+	if sum.WorldsReaped != 0 || sum.AwaitingOffsite != 1 || sum.Skipped != 0 {
+		t.Fatalf("run1 summary = %+v, want archived and awaiting the copy", sum)
+	}
+	if ar.archives != 1 || len(st.backups) != 1 || cl.deletePVCCalls != 0 {
+		t.Fatalf("run1: archives=%d backups=%d deletes=%d, want 1/1/0", ar.archives, len(st.backups), cl.deletePVCCalls)
+	}
+
+	// The copy has not landed yet: still kept, still one archive.
+	if sum := mustRun(t, r); sum.AwaitingOffsite != 1 || ar.archives != 1 || cl.deletePVCCalls != 0 {
+		t.Fatalf("run2 = %+v archives=%d deletes=%d, want the world still kept", sum, ar.archives, cl.deletePVCCalls)
+	}
+
+	st.backups[0].offsite = true
+	sum = mustRun(t, r)
+	if sum.WorldsReaped != 1 || sum.AwaitingOffsite != 0 {
+		t.Fatalf("run3 summary = %+v, want reaped", sum)
+	}
+	if ar.archives != 1 || len(cl.deletedPVCs) != 1 {
+		t.Fatalf("run3: archives=%d deleted=%v, want the copied archive reused and the PVC gone", ar.archives, cl.deletedPVCs)
+	}
+}
+
 // Red line ⑤ (reap arm): an unowned server is still reaped on time; the audit
 // records an empty former_owner.
 func TestReapUnownedServerStillReaped(t *testing.T) {

internal/store/migrations/0024_world_backups_offsite.sql

@@ -0,0 +1,11 @@
+-- Off-site copies of world archives. offsite_at is when `felis offsite sync`
+-- (felis-offsite.timer on the host) confirmed the archive's encrypted copy in
+-- the [offsite] bucket; NULL means the archive exists on this node only. With
+-- [offsite] configured the reaper deletes an idle world only once the archive
+-- it made has an off-site copy, so losing the node's disk cannot take a reaped
+-- world with it.
+ALTER TABLE world_backups ADD COLUMN offsite_at timestamptz;
+
+-- The sync's work list: present archives still waiting for their copy.
+CREATE INDEX world_backups_offsite_pending_idx ON world_backups (created_at)
+  WHERE status = 'present' AND offsite_at IS NULL;

internal/watchdog/probes.go

@@ -13,6 +13,7 @@ import (
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/dbbackup"
 	"felis.lolicon.best/internal/naming"
+	"felis.lolicon.best/internal/offsite"
 	"felis.lolicon.best/internal/platform"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -333,6 +334,43 @@ func BackupFinding(dir string, now time.Time) *Finding {
 	return f
 }
 
+// OffsiteFinding reports an off-site copy that has not completed a clean run
+// within offsite.StaleAfter, going by the record `felis offsite sync` leaves
+// in statusFile. It is a warning: the local copies are intact, but a lost
+// node would now lose what the bucket lacks, and the reaper keeps idle worlds
+// on disk until their archives reach the bucket.
+func OffsiteFinding(statusFile string, now time.Time) *Finding {
+	const hint = "journalctl -u felis-offsite -n 50; `sudo felis offsite status` (docs/troubleshooting.md §16)"
+	st, err := offsite.ReadStatus(statusFile)
+	if err != nil {
+		return &Finding{
+			Key: "offsite", Severity: Warning, For: backupFor,
+			Summary:   fmt.Sprintf("无法读取异地备份状态 %s", statusFile),
+			SummaryEN: fmt.Sprintf("cannot read the off-site copy status %s: %v", statusFile, err),
+			Hint:      hint,
+		}
+	}
+	if st != nil && !st.LastSuccess.IsZero() && now.Sub(st.LastSuccess) <= offsite.StaleAfter {
+		return nil
+	}
+	f := &Finding{
+		Key: "offsite", Severity: Warning, For: backupFor,
+		Summary:   "异地备份从未成功同步过，世界归档与数据库备份只在本机",
+		SummaryEN: "the off-site copy has never completed; world archives and database bundles exist on this machine only",
+		Hint:      hint,
+	}
+	if st != nil && !st.LastSuccess.IsZero() {
+		age := roundHours(now.Sub(st.LastSuccess))
+		f.Summary = fmt.Sprintf("异地备份已有 %s 没有成功同步", age)
+		f.SummaryEN = fmt.Sprintf("the off-site copy last completed %s ago", age)
+	}
+	if st != nil && st.LastError != "" {
+		f.Summary += "（最近一次错误：" + st.LastError + "）"
+		f.SummaryEN += " (last error: " + st.LastError + ")"
+	}
+	return f
+}
+
 // DiskFindings reports each filesystem under paths that is running out of
 // space. Paths on one filesystem are reported once, under the first of them; a
 // path that does not exist is skipped (a feature that is not in use).
@@ -407,8 +445,9 @@ func MemoryFinding(meminfo string) *Finding {
 	}
 }
 
+// roundHours prints a duration to the hour, "35h" rather than "35h0m0s".
 func roundHours(d time.Duration) string {
-	return d.Round(time.Hour).String()
+	return strings.TrimSuffix(d.Round(time.Hour).String(), "0m0s")
 }
 
 func humanBytes(b uint64) string {

internal/watchdog/probes_test.go

@@ -12,6 +12,7 @@ import (
 
 	"felis.lolicon.best/internal/apis/felis/v1alpha1"
 	"felis.lolicon.best/internal/dbbackup"
+	"felis.lolicon.best/internal/offsite"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -146,7 +147,7 @@ func TestBackupFinding(t *testing.T) {
 		}
 	}
 	touch(t0.Add(-30 * time.Hour))
-	if f := BackupFinding(dir, t0); f == nil || !strings.Contains(f.SummaryEN, "30h0m0s old") {
+	if f := BackupFinding(dir, t0); f == nil || !strings.Contains(f.SummaryEN, "30h old") {
 		t.Fatalf("stale: %+v", f)
 	}
 	touch(t0.Add(-2 * time.Hour))
@@ -155,6 +156,30 @@ func TestBackupFinding(t *testing.T) {
 	}
 }
 
+func TestOffsiteFinding(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "offsite", "status.json")
+	if f := OffsiteFinding(path, t0); f == nil || !strings.Contains(f.SummaryEN, "never completed") {
+		t.Fatalf("no status: %+v", f)
+	}
+	write := func(st offsite.Status) {
+		if err := offsite.WriteStatus(path, st); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write(offsite.Status{LastAttempt: t0.Add(-time.Hour), LastError: "bucket unreachable"})
+	if f := OffsiteFinding(path, t0); f == nil || !strings.Contains(f.SummaryEN, "machine only (last error: bucket unreachable)") {
+		t.Fatalf("failing from the start: %+v", f)
+	}
+	write(offsite.Status{LastAttempt: t0.Add(-time.Hour), LastSuccess: t0.Add(-13 * time.Hour), LastError: "access denied"})
+	if f := OffsiteFinding(path, t0); f == nil || f.Severity != Warning || !strings.Contains(f.SummaryEN, "13h ago (last error: access denied)") {
+		t.Fatalf("stale: %+v", f)
+	}
+	write(offsite.Status{LastAttempt: t0.Add(-time.Hour), LastSuccess: t0.Add(-2 * time.Hour), LastError: "one bundle failed"})
+	if f := OffsiteFinding(path, t0); f != nil {
+		t.Fatalf("a success within %s reported: %+v", offsite.StaleAfter, f)
+	}
+}
+
 func TestMemoryFinding(t *testing.T) {
 	path := filepath.Join(t.TempDir(), "meminfo")
 	write := func(avail int) {