feat(backup): 归档先写 .partial 再 fsync 改名并记录 sha256,删除原件前回读校验,reaper 抽样巡检与清扫半截归档,保留权限与 mtime,面板标出损坏与已校验

This commit is contained in:
Lemon-miaow committed 2026-09-25 03:16:06 +08:00
1 parent 489eff4494
commit 89a0134707
25 files changed
+1489 -125

No files matched your search

+10 -1
View File
@@ -7,6 +7,7 @@ import (
"flag" "flag"
"fmt" "fmt"
"io" "io"
"strings"
"time" "time"
"felis.lolicon.best/internal/backup" "felis.lolicon.best/internal/backup"
@@ -91,11 +92,16 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
return 1 return 1
} }
ref, size, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server)) a, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
if err != nil { if err != nil {
fmt.Fprintf(stderr, "felis backup: archive: %v\n", err) fmt.Fprintf(stderr, "felis backup: archive: %v\n", err)
return 1 return 1
} }
ref, size := a.Ref, a.Size
if len(a.Skipped) > 0 {
fmt.Fprintf(stderr, "felis backup: %d entries are not plain files or directories and are not in the archive: %s\n",
len(a.Skipped), strings.Join(a.Skipped[:min(len(a.Skipped), 10)], ", "))
}
drv, err := store.Open(ctx, cfg.Database.URL) drv, err := store.Open(ctx, cfg.Database.URL)
if err != nil { if err != nil {
@@ -112,6 +118,9 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
SizeBytes: size, SizeBytes: size,
Reason: *reason, Reason: *reason,
ExpiresAt: time.Now().Add(rcfg.ManualRetention), ExpiresAt: time.Now().Add(rcfg.ManualRetention),
SHA256: a.SHA256,
SkippedEntries: len(a.Skipped),
} }
st := reaper.NewPGStore(drv.DB()) st := reaper.NewPGStore(drv.DB())
if err := st.InsertBackup(ctx, rec); err != nil { if err := st.InsertBackup(ctx, rec); err != nil {
+14 -4
View File
@@ -138,14 +138,24 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived // FelisWorldJobFailed rule) reach the operator: a world that cannot be archived
// is kept, and without this nobody would learn that it is never reaped. // is kept, and without this nobody would learn that it is never reaped.
func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int { func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {
fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d\n", fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",
sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.Warned, sum.Skipped, sum.StoreFull, sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.Warned, sum.Skipped, sum.StoreFull,
sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed) sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed,
sum.Verified, sum.Corrupt, sum.VerifyFailed, sum.Swept, sum.OrphanArchives)
if !sum.Failed() { if !sum.Failed() {
return 0 return 0
} }
fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n", if sum.Skipped > 0 || sum.ExpireFailed > 0 {
sum.Skipped, sum.StoreFull, sum.ExpireFailed) fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
sum.Skipped, sum.StoreFull, sum.ExpireFailed)
}
if sum.Corrupt > 0 {
fmt.Fprintf(stderr, "felis reaper: %d archives did not read back and are marked corrupt; they are no longer offered for restore (the errors are above)\n", sum.Corrupt)
}
if sum.VerifyFailed > 0 || sum.SweepFailed {
fmt.Fprintf(stderr, "felis reaper: %d archives could not be read back and the store sweep completed=%t; both are retried next run\n",
sum.VerifyFailed, !sum.SweepFailed)
}
return 1 return 1
} }
+4
View File
@@ -30,6 +30,10 @@ func TestReportReaperRunFailsTheJob(t *testing.T) {
{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1}, {"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},
{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1}, {"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},
{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1}, {"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},
{"corrupt archive", reaper.Summary{Evaluated: 3, Verified: 4, Corrupt: 1}, 1},
{"read-back failed", reaper.Summary{Evaluated: 3, VerifyFailed: 1}, 1},
{"sweep failed", reaper.Summary{Evaluated: 3, SweepFailed: true}, 1},
{"orphans kept", reaper.Summary{Evaluated: 3, Swept: 2, OrphanArchives: 1}, 0},
} { } {
var out, errb bytes.Buffer var out, errb bytes.Buffer
if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want { if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want {
+2 -2
View File
@@ -32,11 +32,11 @@ func archiveTempWorld(t *testing.T, files map[string]string) (ref string, backup
BackupRoot: backupRoot, BackupRoot: backupRoot,
Resolve: func(string) (string, error) { return srcDir, nil }, Resolve: func(string) (string, error) { return srcDir, nil },
} }
got, _, err := ar.Archive(context.Background(), "survival", "world-survival-0") got, err := ar.Archive(context.Background(), "survival", "world-survival-0")
if err != nil { if err != nil {
t.Fatalf("Archive: %v", err) t.Fatalf("Archive: %v", err)
} }
return string(got), backupRoot return string(got.Ref), backupRoot
} }
// The restore subcommand must extract the archived world into the target world // The restore subcommand must extract the archived world into the target world
+11 -1
View File
@@ -368,6 +368,16 @@ components:
status: { type: string } status: { type: string }
created_at: { type: string, format: date-time } created_at: { type: string, format: date-time }
expires_at: { type: string, format: date-time } expires_at: { type: string, format: date-time }
corrupt:
type: boolean
description: The archive failed a read-back (a checksum, gzip or tar error) and cannot be restored. Omitted when false.
verified_at:
type: string
format: date-time
description: The archive's last read-back that matched. Omitted until the first.
skipped_entries:
type: integer
description: World entries the archive could not hold (symbolic links, devices, sockets). Omitted when zero.
Build: Build:
type: object type: object
@@ -3016,7 +3026,7 @@ paths:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'409': '409':
description: Server is not stopped (not_stopped), a restore, backup or file write already holds its world volume (maintenance_in_progress), or a restore of another backup is still running (restore_in_progress). description: Server is not stopped (not_stopped), a restore, backup or file write already holds its world volume (maintenance_in_progress), a restore of another backup is still running (restore_in_progress), or the chosen backup failed a read-back (backup_corrupt).
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
+2 -2
View File
@@ -806,7 +806,7 @@ func (f *fakeRepo) LatestBackup(_ context.Context, serverName string) (*BackupRe
var latest *fakeBackup var latest *fakeBackup
for i := range f.backups { for i := range f.backups {
b := &f.backups[i] b := &f.backups[i]
if b.view.Status != "present" || b.view.ServerName != serverName { if b.view.Status != "present" || b.view.ServerName != serverName || b.view.Corrupt {
continue continue
} }
if latest == nil || b.view.CreatedAt.After(latest.view.CreatedAt) { if latest == nil || b.view.CreatedAt.After(latest.view.CreatedAt) {
@@ -830,7 +830,7 @@ func (f *fakeRepo) BackupByID(_ context.Context, id string) (*BackupRecord, erro
return &BackupRecord{ return &BackupRecord{
ID: b.view.ID, ServerName: b.view.ServerName, ID: b.view.ID, ServerName: b.view.ServerName,
FormerOwner: b.view.FormerOwner, BackupRef: b.ref, FormerOwner: b.view.FormerOwner, BackupRef: b.ref,
SizeBytes: b.view.SizeBytes, SizeBytes: b.view.SizeBytes, Corrupt: b.view.Corrupt,
}, nil }, nil
} }
} }
+7
View File
@@ -141,6 +141,13 @@ func (a *API) handleRestoreBackup(w http.ResponseWriter, r *http.Request) {
writeError(w, r, errForbidden) writeError(w, r, errForbidden)
return return
} }
// The reaper found this archive damaged when it read it back; the restore
// Job would refuse it too, but only after the world was locked for it.
if backup.Corrupt {
writeError(w, r, newError(http.StatusConflict, "backup_corrupt",
"this backup did not read back intact and cannot be restored; pick another"))
return
}
} else { } else {
backup, err = a.Repo.LatestBackup(r.Context(), name) backup, err = a.Repo.LatestBackup(r.Context(), name)
if err != nil { if err != nil {
+23
View File
@@ -414,6 +414,29 @@ func TestRestoreBackup(t *testing.T) {
} }
}) })
t.Run("restore by backup_id that failed a read-back -> 409 backup_corrupt", func(t *testing.T) {
api, repo, _, restorer := mkTwo()
repo.backups[1].view.Corrupt = true
api.External = staticExternal{p: owner}
w := do(api.ExternalHandler(), "POST", path, `{"backup_id":"bk2"}`, jsonHeaders)
if w.Code != http.StatusConflict || decodeErr(t, w) != "backup_corrupt" {
t.Fatalf("code = %d body %s", w.Code, w.Body.String())
}
if restorer.calls != 0 {
t.Fatal("a corrupt backup reached the restorer")
}
})
t.Run("no body skips a latest backup that failed a read-back", func(t *testing.T) {
api, repo, _, restorer := mkTwo()
repo.backups[0].view.Corrupt = true
api.External = staticExternal{p: owner}
w := do(api.ExternalHandler(), "POST", path, "", nil)
if w.Code != http.StatusAccepted || restorer.gotRef != "ref-bk2" {
t.Fatalf("code = %d ref %q, want 202 restoring the newest intact backup", w.Code, restorer.gotRef)
}
})
t.Run("no body -> falls back to LatestBackup (backward compat)", func(t *testing.T) { t.Run("no body -> falls back to LatestBackup (backward compat)", func(t *testing.T) {
api, _, _, restorer := mkTwo() api, _, _, restorer := mkTwo()
api.External = staticExternal{p: owner} api.External = staticExternal{p: owner}
+15 -9
View File
@@ -675,7 +675,7 @@ func (p *PGRepo) SeedServer(ctx context.Context, name, subdomain string, cpuMill
// listed — an expired or deleted backup is gone (spec §466). // listed — an expired or deleted backup is gone (spec §466).
func (p *PGRepo) AllBackups(ctx context.Context) ([]BackupView, error) { func (p *PGRepo) AllBackups(ctx context.Context) ([]BackupView, error) {
const q = `SELECT id, server_name, COALESCE(former_owner, ''), COALESCE(size_bytes, 0), const q = `SELECT id, server_name, COALESCE(former_owner, ''), COALESCE(size_bytes, 0),
reason, status, created_at, expires_at reason, status, created_at, expires_at, corrupt_at IS NOT NULL, verified_at, skipped_entries
FROM world_backups WHERE status = 'present' ORDER BY created_at DESC` FROM world_backups WHERE status = 'present' ORDER BY created_at DESC`
rows, err := p.db.QueryContext(ctx, q) rows, err := p.db.QueryContext(ctx, q)
if err != nil { if err != nil {
@@ -689,7 +689,7 @@ func (p *PGRepo) AllBackups(ctx context.Context) ([]BackupView, error) {
// former_owner never matches a user id, so orphaned backups stay admin-only. // former_owner never matches a user id, so orphaned backups stay admin-only.
func (p *PGRepo) BackupsForUser(ctx context.Context, userID string) ([]BackupView, error) { func (p *PGRepo) BackupsForUser(ctx context.Context, userID string) ([]BackupView, error) {
const q = `SELECT id, server_name, COALESCE(former_owner, ''), COALESCE(size_bytes, 0), const q = `SELECT id, server_name, COALESCE(former_owner, ''), COALESCE(size_bytes, 0),
reason, status, created_at, expires_at reason, status, created_at, expires_at, corrupt_at IS NOT NULL, verified_at, skipped_entries
FROM world_backups WHERE status = 'present' AND former_owner = $1 ORDER BY created_at DESC` FROM world_backups WHERE status = 'present' AND former_owner = $1 ORDER BY created_at DESC`
rows, err := p.db.QueryContext(ctx, q, userID) rows, err := p.db.QueryContext(ctx, q, userID)
if err != nil { if err != nil {
@@ -705,21 +705,26 @@ func scanBackupViews(rows *sql.Rows) ([]BackupView, error) {
var out []BackupView var out []BackupView
for rows.Next() { for rows.Next() {
var v BackupView var v BackupView
var verified sql.NullTime
if err := rows.Scan(&v.ID, &v.ServerName, &v.FormerOwner, &v.SizeBytes, if err := rows.Scan(&v.ID, &v.ServerName, &v.FormerOwner, &v.SizeBytes,
&v.Reason, &v.Status, &v.CreatedAt, &v.ExpiresAt); err != nil { &v.Reason, &v.Status, &v.CreatedAt, &v.ExpiresAt, &v.Corrupt, &verified, &v.SkippedEntries); err != nil {
return nil, err return nil, err
} }
if verified.Valid {
v.VerifiedAt = &verified.Time
}
out = append(out, v) out = append(out, v)
} }
return out, rows.Err() return out, rows.Err()
} }
// LatestBackup returns the most recent present backup for a server (spec §466 // LatestBackup returns the most recent present backup for a server that has not
// restore), or ErrNotFound. Unlike the list queries this selects backup_ref — the // failed a read-back (spec §466 restore), or ErrNotFound. Unlike the list queries
// caller (the restore handler) hands it to the Restorer and never serializes it. // this selects backup_ref — the caller (the restore handler) hands it to the
// Restorer and never serializes it.
func (p *PGRepo) LatestBackup(ctx context.Context, serverName string) (*BackupRecord, error) { func (p *PGRepo) LatestBackup(ctx context.Context, serverName string) (*BackupRecord, error) {
const q = `SELECT id, server_name, COALESCE(former_owner, ''), backup_ref, COALESCE(size_bytes, 0) const q = `SELECT id, server_name, COALESCE(former_owner, ''), backup_ref, COALESCE(size_bytes, 0)
FROM world_backups WHERE server_name = $1 AND status = 'present' FROM world_backups WHERE server_name = $1 AND status = 'present' AND corrupt_at IS NULL
ORDER BY created_at DESC LIMIT 1` ORDER BY created_at DESC LIMIT 1`
var b BackupRecord var b BackupRecord
switch err := p.db.QueryRowContext(ctx, q, serverName).Scan( switch err := p.db.QueryRowContext(ctx, q, serverName).Scan(
@@ -734,11 +739,12 @@ func (p *PGRepo) LatestBackup(ctx context.Context, serverName string) (*BackupRe
// BackupByID returns a single present backup by its id, or ErrNotFound. // BackupByID returns a single present backup by its id, or ErrNotFound.
func (p *PGRepo) BackupByID(ctx context.Context, id string) (*BackupRecord, error) { func (p *PGRepo) BackupByID(ctx context.Context, id string) (*BackupRecord, error) {
const q = `SELECT id, server_name, COALESCE(former_owner, ''), backup_ref, COALESCE(size_bytes, 0) const q = `SELECT id, server_name, COALESCE(former_owner, ''), backup_ref, COALESCE(size_bytes, 0),
corrupt_at IS NOT NULL
FROM world_backups WHERE id = $1 AND status = 'present'` FROM world_backups WHERE id = $1 AND status = 'present'`
var b BackupRecord var b BackupRecord
switch err := p.db.QueryRowContext(ctx, q, id).Scan( switch err := p.db.QueryRowContext(ctx, q, id).Scan(
&b.ID, &b.ServerName, &b.FormerOwner, &b.BackupRef, &b.SizeBytes); { &b.ID, &b.ServerName, &b.FormerOwner, &b.BackupRef, &b.SizeBytes, &b.Corrupt); {
case errors.Is(err, sql.ErrNoRows): case errors.Is(err, sql.ErrNoRows):
return nil, ErrNotFound return nil, ErrNotFound
case err != nil: case err != nil:
+9
View File
@@ -69,6 +69,13 @@ type BackupView struct {
Status string `json:"status"` Status string `json:"status"`
CreatedAt time.Time `json:"created_at"` CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at"` ExpiresAt time.Time `json:"expires_at"`
// Corrupt reports that the archive failed a read-back: it cannot be
// restored. VerifiedAt is its last read-back that matched, and
// SkippedEntries the world entries it could not hold (symbolic links,
// devices, sockets).
Corrupt bool `json:"corrupt,omitempty"`
VerifiedAt *time.Time `json:"verified_at,omitempty"`
SkippedEntries int `json:"skipped_entries,omitempty"`
} }
// BackupRecord is the server-side view of a backup used to drive a restore (spec // BackupRecord is the server-side view of a backup used to drive a restore (spec
@@ -81,6 +88,8 @@ type BackupRecord struct {
FormerOwner string FormerOwner string
BackupRef string BackupRef string
SizeBytes int64 SizeBytes int64
// Corrupt reports that the archive failed a read-back (see BackupView).
Corrupt bool
} }
// StaffUser is the login-side projection of a users row (spec §B passwordless // StaffUser is the login-side projection of a users row (spec §B passwordless
+57 -4
View File
@@ -4,7 +4,11 @@
// ArchiveRef is deliberately opaque. // ArchiveRef is deliberately opaque.
package backup package backup
import "context" import (
"context"
"errors"
"time"
)
// ArchiveRef is an opaque handle to a stored world archive. Depending on the // ArchiveRef is an opaque handle to a stored world archive. Depending on the
// backend it may be a tar path, a VolumeSnapshot name, or a Longhorn backup URL. // backend it may be a tar path, a VolumeSnapshot name, or a Longhorn backup URL.
@@ -15,15 +19,64 @@ type ArchiveRef string
// backends (VolumeSnapshot/Longhorn) cannot produce an io.Reader — they create // backends (VolumeSnapshot/Longhorn) cannot produce an io.Reader — they create
// K8s objects referencing the source PVC (spec §19). // K8s objects referencing the source PVC (spec §19).
type WorldArchiver interface { type WorldArchiver interface {
// Archive captures the world living on pvc for server and returns an opaque // Archive captures the world living on pvc for server. It returns only once
// ref plus the stored size in bytes. // the archive is complete and durable: a failed or interrupted Archive leaves
Archive(ctx context.Context, server, pvc string) (ref ArchiveRef, size int64, err error) // nothing that could be mistaken for a finished archive.
Archive(ctx context.Context, server, pvc string) (Archived, error)
// Restore writes a previously archived world into targetPVC. // Restore writes a previously archived world into targetPVC.
Restore(ctx context.Context, ref ArchiveRef, targetPVC string) error Restore(ctx context.Context, ref ArchiveRef, targetPVC string) error
// Delete removes the archive identified by ref. // Delete removes the archive identified by ref.
Delete(ctx context.Context, ref ArchiveRef) error Delete(ctx context.Context, ref ArchiveRef) error
} }
// Archived is what one Archive call stored.
type Archived struct {
Ref ArchiveRef
Size int64 // stored bytes
// SHA256 is the hex digest of the stored archive, "" when the backend keeps
// none. world_backups.sha256 records it, so a later Verify tells an archive
// that rotted on disk from a good one.
SHA256 string
// Skipped lists the world entries the archive leaves out (symbolic links,
// devices, sockets, named pipes), relative to the world root.
Skipped []string
}
// ErrCorrupt marks an archive that cannot be read back in full, or whose bytes
// no longer match the checksum recorded for it. It is the only Verify error that
// condemns the archive: any other (a mount that is not there, a cancelled
// context) says nothing about it.
var ErrCorrupt = errors.New("backup: archive corrupt")
// Verifier is implemented by backends that can read a stored archive back end to
// end. Verify returns the archive's SHA256 as read; want, when not "", is the
// digest it must match (an archive recorded before checksums were kept has
// none, and its read-back establishes one).
type Verifier interface {
Verify(ctx context.Context, ref ArchiveRef, want string) (sha256 string, err error)
}
// Sweeper is implemented by backends that can find what interrupted archives
// leave behind.
type Sweeper interface {
// Sweep removes every unfinished archive last written before partialBefore.
// A finished archive that live does not claim (its record was never inserted)
// is removed once it was last written before orphanBefore, and reported and
// kept until then.
Sweep(ctx context.Context, live func(ArchiveRef) bool, partialBefore, orphanBefore time.Time) (Swept, error)
}
// Swept is what one Sweep found.
type Swept struct {
Removed []string // paths removed
// Orphans are finished archives no record claims that are still young enough
// to keep. A backup whose record insert failed looks like this, and so do the
// archives of a store brought back before the database that records them:
// removing them early could throw away the only copy of a world.
Orphans []string
OrphanBytes int64
}
// PVCResolver maps a PVC name to the local filesystem path where it is mounted. // PVCResolver maps a PVC name to the local filesystem path where it is mounted.
// In production the reaper Job mounts the source/backup PVCs and supplies a // In production the reaper Job mounts the source/backup PVCs and supplies a
// resolver over those mount points; tests supply temp dirs. // resolver over those mount points; tests supply temp dirs.
+314 -45
View File
@@ -4,13 +4,19 @@ import (
"archive/tar" "archive/tar"
"compress/gzip" "compress/gzip"
"context" "context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt" "fmt"
"io" "io"
"io/fs" "io/fs"
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
"regexp"
"sort"
"strings" "strings"
"syscall"
"time" "time"
) )
@@ -34,38 +40,218 @@ func (t *TarLocal) now() time.Time {
return time.Now() return time.Now()
} }
// partialSuffix marks an archive still being written. Its file is the final
// name hidden behind a leading dot, so a listing of the store shows finished
// archives only and a leftover still says whose it was.
const partialSuffix = ".partial"
// archiveName matches the finished archives Archive names (<server>-<unix
// nanoseconds>.tar.gz); partialName matches their unfinished form. Sweep touches
// nothing else in the store.
var (
archiveName = regexp.MustCompile(`^[^.].*-[0-9]+\.tar\.gz$`)
partialName = regexp.MustCompile(`^\..*-[0-9]+\.tar\.gz\.partial$`)
)
// Archive tars+gzips the world on pvc into BackupRoot and returns the archive // Archive tars+gzips the world on pvc into BackupRoot and returns the archive
// path as the opaque ref plus its on-disk size. // path as the opaque ref, with its size and SHA256.
func (t *TarLocal) Archive(ctx context.Context, server, pvc string) (ArchiveRef, int64, error) { //
// The archive is written under its hidden .partial name, flushed to disk, read
// back in full, and only then renamed to its final name and the rename flushed
// too. A Job killed at its deadline or a node that loses power mid-write leaves
// a .partial for Sweep, never a truncated file under a name that looks finished,
// and the world is deleted only after an archive that has already been read
// back whole.
func (t *TarLocal) Archive(ctx context.Context, server, pvc string) (Archived, error) {
srcDir, err := t.Resolve(pvc) srcDir, err := t.Resolve(pvc)
if err != nil { if err != nil {
return "", 0, err return Archived{}, err
} }
if err := os.MkdirAll(t.BackupRoot, 0o750); err != nil { if err := os.MkdirAll(t.BackupRoot, 0o750); err != nil {
return "", 0, fmt.Errorf("backup: mkdir backup root: %w", err) return Archived{}, fmt.Errorf("backup: mkdir backup root: %w", err)
} }
name := fmt.Sprintf("%s-%d.tar.gz", server, t.now().UTC().UnixNano()) name := fmt.Sprintf("%s-%d.tar.gz", server, t.now().UTC().UnixNano())
dest := filepath.Join(t.BackupRoot, name) dest := filepath.Join(t.BackupRoot, name)
tmp := filepath.Join(t.BackupRoot, "."+name+partialSuffix)
f, err := os.Create(dest) f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
if err != nil { if err != nil {
return "", 0, fmt.Errorf("backup: create archive: %w", err) return Archived{}, fmt.Errorf("backup: create archive: %w", err)
} }
if err := writeTarGz(ctx, f, srcDir); err != nil { h := sha256.New()
f.Close() st, err := writeTarGz(ctx, io.MultiWriter(f, h), srcDir)
os.Remove(dest) if err == nil {
return "", 0, err if err = f.Sync(); err != nil {
err = fmt.Errorf("backup: sync archive: %w", err)
}
} }
if err := f.Close(); err != nil { if cerr := f.Close(); err == nil && cerr != nil {
os.Remove(dest) err = fmt.Errorf("backup: close archive: %w", cerr)
return "", 0, fmt.Errorf("backup: close archive: %w", err)
} }
if err != nil {
os.Remove(tmp)
return Archived{}, err
}
sum := hex.EncodeToString(h.Sum(nil))
// Read back what landed: the gzip CRC, every tar header and entry, and the
// entry count must all come out as written.
entries, _, err := verifyArchive(ctx, tmp, sum)
if err == nil && entries != st.entries {
err = fmt.Errorf("%w: %s: %d entries read back, %d written", ErrCorrupt, tmp, entries, st.entries)
}
if err != nil {
os.Remove(tmp)
return Archived{}, err
}
if err := os.Rename(tmp, dest); err != nil {
os.Remove(tmp)
return Archived{}, fmt.Errorf("backup: name archive: %w", err)
}
if err := syncDir(t.BackupRoot); err != nil {
os.Remove(dest)
return Archived{}, fmt.Errorf("backup: sync backup root: %w", err)
}
info, err := os.Stat(dest) info, err := os.Stat(dest)
if err != nil { if err != nil {
return "", 0, fmt.Errorf("backup: stat archive: %w", err) return Archived{}, fmt.Errorf("backup: stat archive: %w", err)
} }
return ArchiveRef(dest), info.Size(), nil return Archived{Ref: ArchiveRef(dest), Size: info.Size(), SHA256: sum, Skipped: st.skipped}, nil
}
// Verify reads the archive at ref back end to end (see verifyArchive).
func (t *TarLocal) Verify(ctx context.Context, ref ArchiveRef, want string) (string, error) {
_, sum, err := verifyArchive(ctx, string(ref), want)
return sum, err
}
// verifyArchive reads the archive at p through gzip and tar to the last byte and
// returns its entry count and SHA256. Anything that does not read back — a
// missing file, a torn gzip stream, a CRC or length mismatch, a bad tar header,
// or a digest other than want (when want is not "") — is ErrCorrupt. Only an
// error opening a file that is there, or the context, is not.
func verifyArchive(ctx context.Context, p, want string) (int, string, error) {
f, err := os.Open(p)
if errors.Is(err, fs.ErrNotExist) {
return 0, "", fmt.Errorf("%w: %s is missing", ErrCorrupt, p)
}
if err != nil {
return 0, "", fmt.Errorf("backup: open archive: %w", err)
}
defer f.Close()
h := sha256.New()
raw := io.TeeReader(f, h)
corrupt := func(err error) (int, string, error) {
if ctx.Err() != nil {
return 0, "", ctx.Err()
}
return 0, "", fmt.Errorf("%w: %s: %v", ErrCorrupt, p, err)
}
gz, err := gzip.NewReader(raw)
if err != nil {
return corrupt(err)
}
tr := tar.NewReader(gz)
entries := 0
for {
if ctx.Err() != nil {
return 0, "", ctx.Err()
}
_, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return corrupt(err)
}
if _, err := io.Copy(io.Discard, tr); err != nil {
return corrupt(err)
}
entries++
}
// The tar end marker is not the end of the gzip member: reading on to EOF is
// what checks the CRC and length in the gzip trailer.
if _, err := io.Copy(io.Discard, gz); err != nil {
return corrupt(err)
}
if err := gz.Close(); err != nil {
return corrupt(err)
}
if _, err := io.Copy(io.Discard, raw); err != nil {
return corrupt(err)
}
sum := hex.EncodeToString(h.Sum(nil))
if want != "" && sum != want {
return 0, sum, fmt.Errorf("%w: %s: sha256 %s, recorded %s", ErrCorrupt, p, sum, want)
}
return entries, sum, nil
}
// Sweep removes the leftovers of archives that never finished (see Sweeper). It
// only looks at the top level of BackupRoot and only at names Archive writes;
// partialBefore must leave room for the longest Archive, and orphanBefore for
// the insert of the record that follows it at the very least.
func (t *TarLocal) Sweep(ctx context.Context, live func(ArchiveRef) bool, partialBefore, orphanBefore time.Time) (Swept, error) {
var out Swept
ents, err := os.ReadDir(t.BackupRoot)
if errors.Is(err, fs.ErrNotExist) {
return out, nil
}
if err != nil {
return out, fmt.Errorf("backup: list backup root: %w", err)
}
var errs []error
for _, e := range ents {
if ctx.Err() != nil {
return out, ctx.Err()
}
name := e.Name()
if !e.Type().IsRegular() {
continue
}
p := filepath.Join(t.BackupRoot, name)
cutoff := partialBefore
switch {
case partialName.MatchString(name):
case archiveName.MatchString(name):
if live(ArchiveRef(p)) {
continue
}
cutoff = orphanBefore
default:
continue
}
info, err := e.Info()
if err != nil || !info.ModTime().Before(partialBefore) {
continue
}
if !info.ModTime().Before(cutoff) {
out.Orphans = append(out.Orphans, p)
out.OrphanBytes += info.Size()
continue
}
if err := os.Remove(p); err != nil && !errors.Is(err, fs.ErrNotExist) {
errs = append(errs, err)
continue
}
out.Removed = append(out.Removed, p)
}
return out, errors.Join(errs...)
}
// syncDir flushes a change to a directory's entries (a rename) to disk. A
// filesystem that cannot sync a directory has no stronger promise to give.
func syncDir(dir string) error {
d, err := os.Open(dir)
if err != nil {
return err
}
defer d.Close()
if err := d.Sync(); err != nil && !errors.Is(err, syscall.EINVAL) && !errors.Is(err, syscall.ENOTSUP) {
return err
}
return nil
} }
// Restore extracts the archive at ref into the world mount for targetPVC, // Restore extracts the archive at ref into the world mount for targetPVC,
@@ -74,10 +260,16 @@ func (t *TarLocal) Archive(ctx context.Context, server, pvc string) (ArchiveRef,
// griefer's chunks). It extracts over the target, then removes any pre-existing // griefer's chunks). It extracts over the target, then removes any pre-existing
// entry the archive did not contain. // entry the archive did not contain.
// //
// The prune runs only after a fully successful extract: a corrupt or truncated // The archive is read back in full before anything is extracted, so a corrupt
// archive fails before the prune, leaving the target as a (recoverable) partial // or truncated archive fails with the world untouched. The prune runs only after
// overlay rather than a destroyed world. The archive is retained on restore, so // a fully successful extract: an extract that still fails (the volume fills up)
// such a failure is recoverable by re-running the Job. // leaves the target as a recoverable partial overlay rather than a destroyed
// world. The archive is retained on restore, so such a failure is recoverable by
// re-running the Job.
//
// Files and directories get back the permission bits and modification times the
// archive recorded. Ownership is left to the server: every start re-owns the
// world volume to the game uid (felis init-volume).
// //
// A top-level lost+found is never a prune target. It is a filesystem artifact // A top-level lost+found is never a prune target. It is a filesystem artifact
// (root-owned, mode 0700) that the non-root restore Pod cannot delete anyway, // (root-owned, mode 0700) that the non-root restore Pod cannot delete anyway,
@@ -90,17 +282,23 @@ func (t *TarLocal) Restore(ctx context.Context, ref ArchiveRef, targetPVC string
if err := os.MkdirAll(dstDir, 0o750); err != nil { if err := os.MkdirAll(dstDir, 0o750); err != nil {
return fmt.Errorf("backup: mkdir restore target: %w", err) return fmt.Errorf("backup: mkdir restore target: %w", err)
} }
if _, _, err := verifyArchive(ctx, string(ref), ""); err != nil {
return err
}
f, err := os.Open(string(ref)) f, err := os.Open(string(ref))
if err != nil { if err != nil {
return fmt.Errorf("backup: open archive: %w", err) return fmt.Errorf("backup: open archive: %w", err)
} }
defer f.Close() defer f.Close()
keep, err := readTarGz(ctx, f, dstDir) keep, dirs, err := readTarGz(ctx, f, dstDir)
if err != nil { if err != nil {
return err return err
} }
return pruneToManifest(dstDir, keep) if err := pruneToManifest(dstDir, keep); err != nil {
return err
}
return restoreDirMeta(dirs)
} }
// pruneToManifest removes every entry under dstDir whose archive-relative path // pruneToManifest removes every entry under dstDir whose archive-relative path
@@ -151,7 +349,18 @@ func (t *TarLocal) Delete(_ context.Context, ref ArchiveRef) error {
return nil return nil
} }
func writeTarGz(ctx context.Context, w io.Writer, srcDir string) error { // tarStats is what writeTarGz put in the archive and what it left out.
type tarStats struct {
entries int
skipped []string
}
// writeTarGz archives srcDir (not the root entry itself) as gzip+tar. Each entry
// keeps its permission bits, without setuid, setgid and sticky, and its
// modification time. Entries other than regular files and directories are left
// out and listed in the stats.
func writeTarGz(ctx context.Context, w io.Writer, srcDir string) (tarStats, error) {
var st tarStats
gz := gzip.NewWriter(w) gz := gzip.NewWriter(w)
tw := tar.NewWriter(gz) tw := tar.NewWriter(gz)
@@ -173,12 +382,17 @@ func writeTarGz(ctx context.Context, w io.Writer, srcDir string) error {
// Normalize to forward slashes so archives are portable. // Normalize to forward slashes so archives are portable.
name := filepath.ToSlash(rel) name := filepath.ToSlash(rel)
mode := int64(info.Mode().Perm())
switch { switch {
case info.IsDir(): case info.IsDir():
hdr := &tar.Header{Name: name + "/", Mode: 0o750, Typeflag: tar.TypeDir} hdr := &tar.Header{Name: name + "/", Mode: mode, ModTime: info.ModTime(), Typeflag: tar.TypeDir}
return tw.WriteHeader(hdr) if err := tw.WriteHeader(hdr); err != nil {
return err
}
st.entries++
return nil
case info.Mode().IsRegular(): case info.Mode().IsRegular():
hdr := &tar.Header{Name: name, Mode: 0o640, Size: info.Size(), Typeflag: tar.TypeReg} hdr := &tar.Header{Name: name, Mode: mode, ModTime: info.ModTime(), Size: info.Size(), Typeflag: tar.TypeReg}
if err := tw.WriteHeader(hdr); err != nil { if err := tw.WriteHeader(hdr); err != nil {
return err return err
} }
@@ -187,24 +401,37 @@ func writeTarGz(ctx context.Context, w io.Writer, srcDir string) error {
return err return err
} }
defer src.Close() defer src.Close()
_, err = io.Copy(tw, src) if _, err := io.Copy(tw, src); err != nil {
return err return err
}
st.entries++
return nil
default: default:
// Skip symlinks/devices/sockets: a world directory should be plain // A symlink could point anywhere on the node, and a device or socket
// files, and refusing the rest avoids surprising archive contents. // has no bytes to keep: a world is plain files. What is left out is
// reported, so a backup never drops something silently.
st.skipped = append(st.skipped, name)
return nil return nil
} }
}) })
if err != nil { if err != nil {
return fmt.Errorf("backup: tar walk: %w", err) return st, fmt.Errorf("backup: tar walk: %w", err)
} }
if err := tw.Close(); err != nil { if err := tw.Close(); err != nil {
return fmt.Errorf("backup: close tar: %w", err) return st, fmt.Errorf("backup: close tar: %w", err)
} }
if err := gz.Close(); err != nil { if err := gz.Close(); err != nil {
return fmt.Errorf("backup: close gzip: %w", err) return st, fmt.Errorf("backup: close gzip: %w", err)
} }
return nil return st, nil
}
// dirMeta is a directory's recorded permission bits and modification time,
// applied once nothing more is written into it.
type dirMeta struct {
path string
mode fs.FileMode
mtime time.Time
} }
// readTarGz extracts the gzip+tar stream into dstDir and returns the keep-set: // readTarGz extracts the gzip+tar stream into dstDir and returns the keep-set:
@@ -213,33 +440,39 @@ func writeTarGz(ctx context.Context, w io.Writer, srcDir string) error {
// target files for replace semantics. On any error the keep-set is incomplete // target files for replace semantics. On any error the keep-set is incomplete
// and must not be used to prune (a partial manifest would delete live files the // and must not be used to prune (a partial manifest would delete live files the
// stream had not yet reached). // stream had not yet reached).
func readTarGz(ctx context.Context, r io.Reader, dstDir string) (map[string]struct{}, error) { //
// Files get their recorded mode and modification time as they are written. A
// directory's are returned instead (restoreDirMeta): extracting and pruning
// inside it would move its mtime again, and a read-only mode would stop the
// extract from writing into it.
func readTarGz(ctx context.Context, r io.Reader, dstDir string) (map[string]struct{}, []dirMeta, error) {
gz, err := gzip.NewReader(r) gz, err := gzip.NewReader(r)
if err != nil { if err != nil {
return nil, fmt.Errorf("backup: open gzip: %w", err) return nil, nil, fmt.Errorf("backup: open gzip: %w", err)
} }
defer gz.Close() defer gz.Close()
tr := tar.NewReader(gz) tr := tar.NewReader(gz)
keep := make(map[string]struct{}) keep := make(map[string]struct{})
var dirs []dirMeta
cleanDst := filepath.Clean(dstDir) cleanDst := filepath.Clean(dstDir)
for { for {
if ctx.Err() != nil { if ctx.Err() != nil {
return nil, ctx.Err() return nil, nil, ctx.Err()
} }
hdr, err := tr.Next() hdr, err := tr.Next()
if err == io.EOF { if err == io.EOF {
return keep, nil return keep, dirs, nil
} }
if err != nil { if err != nil {
return nil, fmt.Errorf("backup: read tar: %w", err) return nil, nil, fmt.Errorf("backup: read tar: %w", err)
} }
// Guard against path traversal (zip-slip): the resolved target must stay // Guard against path traversal (zip-slip): the resolved target must stay
// within dstDir. // within dstDir.
target := filepath.Join(cleanDst, filepath.FromSlash(hdr.Name)) target := filepath.Join(cleanDst, filepath.FromSlash(hdr.Name))
if target != cleanDst && !strings.HasPrefix(target, cleanDst+string(os.PathSeparator)) { if target != cleanDst && !strings.HasPrefix(target, cleanDst+string(os.PathSeparator)) {
return nil, fmt.Errorf("backup: archive entry escapes target: %q", hdr.Name) return nil, nil, fmt.Errorf("backup: archive entry escapes target: %q", hdr.Name)
} }
// Record this entry and its ancestors in the keep-set. Names are stored // Record this entry and its ancestors in the keep-set. Names are stored
@@ -247,25 +480,39 @@ func readTarGz(ctx context.Context, r io.Reader, dstDir string) (map[string]stru
// relative paths pruneToManifest derives from the on-disk walk. // relative paths pruneToManifest derives from the on-disk walk.
rememberKept(keep, hdr.Name) rememberKept(keep, hdr.Name)
mode := fs.FileMode(hdr.Mode).Perm()
switch hdr.Typeflag { switch hdr.Typeflag {
case tar.TypeDir: case tar.TypeDir:
if err := os.MkdirAll(target, 0o750); err != nil { if err := os.MkdirAll(target, 0o750); err != nil {
return nil, err return nil, nil, err
}
if target != cleanDst {
dirs = append(dirs, dirMeta{path: target, mode: mode, mtime: hdr.ModTime})
} }
case tar.TypeReg: case tar.TypeReg:
if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
return nil, err return nil, nil, err
} }
out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o640) out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
if err != nil { if err != nil {
return nil, err return nil, nil, err
} }
if _, err := io.Copy(out, tr); err != nil { if _, err := io.Copy(out, tr); err != nil {
out.Close() out.Close()
return nil, err return nil, nil, err
} }
if err := out.Close(); err != nil { if err := out.Close(); err != nil {
return nil, err return nil, nil, err
}
// Chmod rather than the create mode: the umask would narrow it, and
// a file that already existed keeps its old mode through O_TRUNC.
if err := os.Chmod(target, mode); err != nil {
return nil, nil, err
}
if recordedTime(hdr.ModTime) {
if err := os.Chtimes(target, hdr.ModTime, hdr.ModTime); err != nil {
return nil, nil, err
}
} }
default: default:
// Ignore entry types tarLocal never writes. // Ignore entry types tarLocal never writes.
@@ -273,6 +520,28 @@ func readTarGz(ctx context.Context, r io.Reader, dstDir string) (map[string]stru
} }
} }
// restoreDirMeta applies the directories' recorded modes and times, deepest
// first, so setting a parent's time comes after every change inside it.
func restoreDirMeta(dirs []dirMeta) error {
sort.SliceStable(dirs, func(i, j int) bool { return len(dirs[i].path) > len(dirs[j].path) })
for _, d := range dirs {
if err := os.Chmod(d.path, d.mode); err != nil {
return fmt.Errorf("backup: restore mode of %s: %w", d.path, err)
}
if recordedTime(d.mtime) {
if err := os.Chtimes(d.path, d.mtime, d.mtime); err != nil {
return fmt.Errorf("backup: restore time of %s: %w", d.path, err)
}
}
}
return nil
}
// recordedTime reports whether an archive recorded a modification time. Archives
// written before times were kept carry the Unix epoch, which is left alone
// rather than stamped onto every file.
func recordedTime(t time.Time) bool { return t.Unix() > 0 }
// rememberKept adds an archive entry name and every ancestor directory to keep, // rememberKept adds an archive entry name and every ancestor directory to keep,
// normalized to a cleaned forward-slash path with no trailing slash. Adding // normalized to a cleaned forward-slash path with no trailing slash. Adding
// ancestors guards against archives that list a file without an explicit entry // ancestors guards against archives that list a file without an explicit entry
+276 -3
View File
@@ -4,9 +4,15 @@ import (
"archive/tar" "archive/tar"
"compress/gzip" "compress/gzip"
"context" "context"
"crypto/sha256"
"encoding/hex"
"errors"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"sort"
"testing" "testing"
"time"
"felis.lolicon.best/internal/backup" "felis.lolicon.best/internal/backup"
) )
@@ -47,10 +53,11 @@ func TestTarLocalRoundTrip(t *testing.T) {
} }
ctx := context.Background() ctx := context.Background()
ref, size, err := archiver.Archive(ctx, "survival", "src-pvc") a, err := archiver.Archive(ctx, "survival", "src-pvc")
if err != nil { if err != nil {
t.Fatalf("Archive: %v", err) t.Fatalf("Archive: %v", err)
} }
ref, size := a.Ref, a.Size
if size <= 0 { if size <= 0 {
t.Errorf("archive size = %d, want > 0", size) t.Errorf("archive size = %d, want > 0", size)
} }
@@ -139,10 +146,11 @@ func TestTarLocalRestoreReplacesTarget(t *testing.T) {
} }
ctx := context.Background() ctx := context.Background()
ref, _, err := archiver.Archive(ctx, "survival", "src-pvc") a, err := archiver.Archive(ctx, "survival", "src-pvc")
if err != nil { if err != nil {
t.Fatalf("Archive: %v", err) t.Fatalf("Archive: %v", err)
} }
ref := a.Ref
if err := archiver.Restore(ctx, ref, "dst-pvc"); err != nil { if err := archiver.Restore(ctx, ref, "dst-pvc"); err != nil {
t.Fatalf("Restore: %v", err) t.Fatalf("Restore: %v", err)
} }
@@ -181,7 +189,7 @@ func TestTarLocalUnknownPVC(t *testing.T) {
BackupRoot: t.TempDir(), BackupRoot: t.TempDir(),
Resolve: backup.StaticResolver(map[string]string{}), Resolve: backup.StaticResolver(map[string]string{}),
} }
if _, _, err := archiver.Archive(context.Background(), "x", "missing"); err == nil { if _, err := archiver.Archive(context.Background(), "x", "missing"); err == nil {
t.Fatal("expected error for unknown pvc") t.Fatal("expected error for unknown pvc")
} }
} }
@@ -222,3 +230,268 @@ func TestTarLocalRejectsZipSlip(t *testing.T) {
t.Errorf("zip-slip wrote outside target: %v", err) t.Errorf("zip-slip wrote outside target: %v", err)
} }
} }
// TestTarLocalArchiveIsDurableAndChecksummed pins the write path: the archive
// lands under its final name only, no .partial is left beside it, and the
// SHA256 it reports is the one of the bytes on disk, which Verify reads back.
func TestTarLocalArchiveIsDurableAndChecksummed(t *testing.T) {
src, backupRoot := t.TempDir(), t.TempDir()
writeTree(t, src, map[string]string{"level.dat": "seed", "region/r.0.0.mca": "chunk"})
archiver := &backup.TarLocal{BackupRoot: backupRoot, Resolve: backup.StaticResolver(map[string]string{"src": src})}
ctx := context.Background()
a, err := archiver.Archive(ctx, "survival", "src")
if err != nil {
t.Fatalf("Archive: %v", err)
}
ents, err := os.ReadDir(backupRoot)
if err != nil {
t.Fatal(err)
}
if len(ents) != 1 || filepath.Join(backupRoot, ents[0].Name()) != string(a.Ref) {
t.Fatalf("backup root holds %v, want only %s", ents, a.Ref)
}
body, err := os.ReadFile(string(a.Ref))
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(body)
if a.SHA256 != hex.EncodeToString(sum[:]) || a.Size != int64(len(body)) {
t.Errorf("Archived = %+v, want sha256 %x size %d", a, sum, len(body))
}
got, err := archiver.Verify(ctx, a.Ref, a.SHA256)
if err != nil || got != a.SHA256 {
t.Errorf("Verify = %q, %v; want %q, nil", got, err, a.SHA256)
}
// With no digest recorded, Verify still reads it through and reports one.
if got, err := archiver.Verify(ctx, a.Ref, ""); err != nil || got != a.SHA256 {
t.Errorf("Verify without a digest = %q, %v", got, err)
}
}
// TestTarLocalFailedArchiveLeavesNothing: an Archive that fails part way (here
// a cancelled context) removes its .partial, so no leftover can pass for an
// archive.
func TestTarLocalFailedArchiveLeavesNothing(t *testing.T) {
src, backupRoot := t.TempDir(), t.TempDir()
writeTree(t, src, map[string]string{"level.dat": "seed"})
archiver := &backup.TarLocal{BackupRoot: backupRoot, Resolve: backup.StaticResolver(map[string]string{"src": src})}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := archiver.Archive(ctx, "survival", "src"); err == nil {
t.Fatal("Archive with a cancelled context succeeded")
}
if ents, _ := os.ReadDir(backupRoot); len(ents) != 0 {
t.Errorf("failed Archive left %v behind", ents)
}
}
// TestTarLocalKeepsModesAndTimes: a restore gives files and directories back
// their permission bits (setuid, setgid and sticky dropped) and modification
// times, and what the archive cannot hold is reported rather than dropped
// silently.
func TestTarLocalKeepsModesAndTimes(t *testing.T) {
src, dst, backupRoot := t.TempDir(), t.TempDir(), t.TempDir()
writeTree(t, src, map[string]string{
"start.sh": "#!/bin/sh",
"secret.properties": "rcon",
"private/notes.txt": "n",
"setuid-bin": "x",
})
mtime := time.Date(2025, 3, 4, 5, 6, 7, 0, time.UTC)
modes := map[string]os.FileMode{
"start.sh": 0o755,
"secret.properties": 0o600,
"private/notes.txt": 0o640,
"setuid-bin": 0o755 | os.ModeSetuid,
"private": 0o700,
}
for rel, m := range modes {
p := filepath.Join(src, filepath.FromSlash(rel))
if err := os.Chmod(p, m); err != nil {
t.Fatal(err)
}
}
for _, rel := range []string{"start.sh", "secret.properties", "private/notes.txt", "setuid-bin", "private"} {
if err := os.Chtimes(filepath.Join(src, filepath.FromSlash(rel)), mtime, mtime); err != nil {
t.Fatal(err)
}
}
if err := os.Symlink("/etc/passwd", filepath.Join(src, "link")); err != nil {
t.Fatal(err)
}
archiver := &backup.TarLocal{BackupRoot: backupRoot, Resolve: backup.StaticResolver(map[string]string{"src": src, "dst": dst})}
ctx := context.Background()
a, err := archiver.Archive(ctx, "survival", "src")
if err != nil {
t.Fatalf("Archive: %v", err)
}
if !reflect.DeepEqual(a.Skipped, []string{"link"}) {
t.Errorf("Skipped = %v, want [link]", a.Skipped)
}
if err := archiver.Restore(ctx, a.Ref, "dst"); err != nil {
t.Fatalf("Restore: %v", err)
}
for rel, m := range modes {
info, err := os.Lstat(filepath.Join(dst, filepath.FromSlash(rel)))
if err != nil {
t.Errorf("%s: %v", rel, err)
continue
}
if got, want := info.Mode().Perm(), m.Perm(); got != want || info.Mode()&os.ModeSetuid != 0 {
t.Errorf("%s restored with mode %v, want %v", rel, info.Mode(), want)
}
if !info.ModTime().Equal(mtime) {
t.Errorf("%s restored with mtime %v, want %v", rel, info.ModTime(), mtime)
}
}
if _, err := os.Lstat(filepath.Join(dst, "link")); !os.IsNotExist(err) {
t.Errorf("symlink came back from the archive: %v", err)
}
}
// damage rewrites the archive at ref with f applied to its bytes.
func damage(t *testing.T, ref backup.ArchiveRef, f func([]byte) []byte) {
t.Helper()
b, err := os.ReadFile(string(ref))
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(string(ref), f(b), 0o644); err != nil {
t.Fatal(err)
}
}
// TestTarLocalVerifyFindsCorruption: a flipped byte, a truncated file, a digest
// that no longer matches and a missing file all come back as ErrCorrupt.
func TestTarLocalVerifyFindsCorruption(t *testing.T) {
src := t.TempDir()
writeTree(t, src, map[string]string{"level.dat": "seed", "region/r.0.0.mca": string(make([]byte, 64<<10))})
ctx := context.Background()
fresh := func(t *testing.T) (*backup.TarLocal, backup.Archived) {
archiver := &backup.TarLocal{BackupRoot: t.TempDir(), Resolve: backup.StaticResolver(map[string]string{"src": src})}
a, err := archiver.Archive(ctx, "survival", "src")
if err != nil {
t.Fatalf("Archive: %v", err)
}
return archiver, a
}
cases := map[string]struct {
change func(*testing.T, backup.Archived)
want func(backup.Archived) string
}{
"flipped byte": {
change: func(t *testing.T, a backup.Archived) {
damage(t, a.Ref, func(b []byte) []byte { b[len(b)/2] ^= 0xff; return b })
},
want: func(a backup.Archived) string { return "" },
},
"truncated": {
change: func(t *testing.T, a backup.Archived) {
damage(t, a.Ref, func(b []byte) []byte { return b[:len(b)-9] })
},
want: func(a backup.Archived) string { return "" },
},
"digest mismatch": {
change: func(*testing.T, backup.Archived) {},
want: func(backup.Archived) string { return hex.EncodeToString(make([]byte, 32)) },
},
"missing": {
change: func(t *testing.T, a backup.Archived) {
if err := os.Remove(string(a.Ref)); err != nil {
t.Fatal(err)
}
},
want: func(a backup.Archived) string { return a.SHA256 },
},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
archiver, a := fresh(t)
tc.change(t, a)
if _, err := archiver.Verify(ctx, a.Ref, tc.want(a)); !errors.Is(err, backup.ErrCorrupt) {
t.Errorf("Verify = %v, want ErrCorrupt", err)
}
})
}
}
// TestTarLocalRestoreRefusesCorruptArchive: the archive is read back before
// anything is extracted, so a corrupt one leaves the world exactly as it was.
func TestTarLocalRestoreRefusesCorruptArchive(t *testing.T) {
src, dst := t.TempDir(), t.TempDir()
writeTree(t, src, map[string]string{"level.dat": "archived", "region/r.0.0.mca": string(make([]byte, 64<<10))})
writeTree(t, dst, map[string]string{"level.dat": "live"})
archiver := &backup.TarLocal{BackupRoot: t.TempDir(), Resolve: backup.StaticResolver(map[string]string{"src": src, "dst": dst})}
ctx := context.Background()
a, err := archiver.Archive(ctx, "survival", "src")
if err != nil {
t.Fatalf("Archive: %v", err)
}
damage(t, a.Ref, func(b []byte) []byte { return b[:len(b)-9] })
if err := archiver.Restore(ctx, a.Ref, "dst"); !errors.Is(err, backup.ErrCorrupt) {
t.Fatalf("Restore = %v, want ErrCorrupt", err)
}
got, err := os.ReadFile(filepath.Join(dst, "level.dat"))
if err != nil || string(got) != "live" {
t.Errorf("world changed by a refused restore: %q, %v", got, err)
}
if _, err := os.Stat(filepath.Join(dst, "region")); !os.IsNotExist(err) {
t.Errorf("refused restore extracted entries: %v", err)
}
}
// TestTarLocalSweep removes old .partial files and orphaned archives past the
// orphan cutoff, reports younger orphans, and leaves everything else: fresh
// leftovers (an Archive may still be writing, or its record not yet inserted),
// claimed archives, and files it did not write.
func TestTarLocalSweep(t *testing.T) {
root := t.TempDir()
now := time.Now()
ancient, old, recent := now.Add(-100*24*time.Hour), now.Add(-7*time.Hour), now.Add(-time.Minute)
files := map[string]time.Time{
".survival-100.tar.gz.partial": old,
".survival-200.tar.gz.partial": recent,
"survival-300.tar.gz": ancient, // orphan past the cutoff
"survival-310.tar.gz": old, // orphan, kept and reported
"survival-400.tar.gz": ancient, // claimed
"survival-500.tar.gz": recent, // may be about to be claimed
"notes.txt": ancient,
"felis.dump": ancient,
}
for name, at := range files {
p := filepath.Join(root, name)
if err := os.WriteFile(p, []byte("xyz"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Chtimes(p, at, at); err != nil {
t.Fatal(err)
}
}
archiver := &backup.TarLocal{BackupRoot: root}
live := func(ref backup.ArchiveRef) bool {
return ref == backup.ArchiveRef(filepath.Join(root, "survival-400.tar.gz"))
}
got, err := archiver.Sweep(context.Background(), live, now.Add(-6*time.Hour), now.Add(-90*24*time.Hour))
if err != nil {
t.Fatalf("Sweep: %v", err)
}
sort.Strings(got.Removed)
want := backup.Swept{
Removed: []string{filepath.Join(root, ".survival-100.tar.gz.partial"), filepath.Join(root, "survival-300.tar.gz")},
Orphans: []string{filepath.Join(root, "survival-310.tar.gz")},
OrphanBytes: 3,
}
if !reflect.DeepEqual(got, want) {
t.Errorf("Sweep = %+v, want %+v", got, want)
}
ents, _ := os.ReadDir(root)
if len(ents) != len(files)-2 {
t.Errorf("%d files left, want %d", len(ents), len(files)-2)
}
// A store that does not exist yet has nothing to sweep.
if _, err := (&backup.TarLocal{BackupRoot: filepath.Join(root, "absent")}).Sweep(context.Background(), live, now, now); err != nil {
t.Errorf("Sweep of a missing root: %v", err)
}
}
+145 -2
View File
@@ -37,10 +37,10 @@ func (c *reclaimCluster) Stop(context.Context, string) error { return nil }
type reclaimArchiver struct{ archived []string } type reclaimArchiver struct{ archived []string }
func (a *reclaimArchiver) Archive(_ context.Context, server, _ string) (backup.ArchiveRef, int64, error) { func (a *reclaimArchiver) Archive(_ context.Context, server, _ string) (backup.Archived, error) {
ref := "/archives/" + server + "-new.tar.gz" ref := "/archives/" + server + "-new.tar.gz"
a.archived = append(a.archived, ref) a.archived = append(a.archived, ref)
return backup.ArchiveRef(ref), 42, nil return backup.Archived{Ref: backup.ArchiveRef(ref), Size: 42}, nil
} }
func (a *reclaimArchiver) Restore(context.Context, backup.ArchiveRef, string) error { return nil } func (a *reclaimArchiver) Restore(context.Context, backup.ArchiveRef, string) error { return nil }
@@ -244,3 +244,146 @@ func TestManualBackupRationing(t *testing.T) {
t.Fatalf("a request before since still counted: (%v, %v)", at, err) t.Fatalf("a request before since still counted: (%v, %v)", at, err)
} }
} }
// TestBackupReadBack pins the SQL behind data-durability-7/15/16: the digest a
// backup is written with survives a later read-back, a corrupt archive is no
// longer offered for reuse, restore or read-back but stays listed, and it is
// the first a keep-N prune removes.
func TestBackupReadBack(t *testing.T) {
ctx := context.Background()
st := reaper.NewPGStore(db)
name := "readback-" + suffix(t)
if _, err := db.ExecContext(ctx,
`INSERT INTO servers (name, cached_cpu_milli, cached_memory_mb, cached_storage_mb) VALUES ($1, 100, 128, 1)`,
name); err != nil {
t.Fatalf("seed server: %v", err)
}
now := time.Now()
sfx := suffix(t)
older, newer := "bk-old-"+sfx, "bk-new-"+sfx
for _, r := range []reaper.BackupRecord{
{ID: older, ServerName: name, BackupRef: "/archives/" + older + ".tar.gz", SizeBytes: 10,
Reason: "inactive_15d", ExpiresAt: now.Add(90 * reaper.Day), SHA256: "aa", SkippedEntries: 2},
{ID: newer, ServerName: name, BackupRef: "/archives/" + newer + ".tar.gz", SizeBytes: 10,
Reason: "inactive_15d", ExpiresAt: now.Add(90 * reaper.Day)},
} {
if err := st.InsertBackup(ctx, r); err != nil {
t.Fatalf("InsertBackup %s: %v", r.ID, err)
}
}
if _, err := db.ExecContext(ctx,
`UPDATE world_backups SET created_at = CASE id WHEN $1 THEN $3::timestamptz ELSE $4::timestamptz END WHERE id IN ($1, $2)`,
older, newer, now.Add(-2*time.Hour), now.Add(-time.Hour)); err != nil {
t.Fatalf("age backups: %v", err)
}
if f, ok, err := st.FreshBackup(ctx, name, now.Add(-reaper.Day)); err != nil || !ok || f.ID != newer || f.SHA256 != "" {
t.Fatalf("FreshBackup = (%+v, %v, %v); want %s with no digest yet", f, ok, err, newer)
}
// A read-back fills in a missing digest and never rewrites a recorded one.
checked := now.Add(-time.Minute)
for id, sum := range map[string]string{newer: "bb", older: "zz"} {
if err := st.MarkBackupVerified(ctx, id, sum, checked); err != nil {
t.Fatalf("MarkBackupVerified %s: %v", id, err)
}
}
due := func(before time.Time) map[string]string {
t.Helper()
bs, err := st.BackupsToVerify(ctx, before, 100000)
if err != nil {
t.Fatalf("BackupsToVerify: %v", err)
}
out := map[string]string{}
for _, b := range bs {
if b.ServerName == name {
out[b.ID] = b.SHA256
}
}
return out
}
if got := due(checked.Add(-time.Second)); len(got) != 0 {
t.Fatalf("due before their read-back = %v; want none", got)
}
if got := due(checked.Add(time.Second)); got[newer] != "bb" || got[older] != "aa" || len(got) != 2 {
t.Fatalf("due after their read-back = %v; want %s=bb and %s=aa", got, newer, older)
}
corruptAt := now.Add(-30 * time.Second)
if err := st.MarkBackupCorrupt(ctx, newer, corruptAt); err != nil {
t.Fatalf("MarkBackupCorrupt: %v", err)
}
if err := st.MarkBackupCorrupt(ctx, newer, now); err != nil {
t.Fatalf("MarkBackupCorrupt again: %v", err)
}
var first time.Time
if err := db.QueryRowContext(ctx, `SELECT corrupt_at FROM world_backups WHERE id = $1`, newer).Scan(&first); err != nil ||
!first.Equal(corruptAt.Truncate(time.Microsecond)) {
t.Fatalf("corrupt_at = (%v, %v); want the first finding %v kept", first, err, corruptAt)
}
if f, ok, err := st.FreshBackup(ctx, name, now.Add(-reaper.Day)); err != nil || !ok || f.ID != older || f.SHA256 != "aa" {
t.Fatalf("FreshBackup after corruption = (%+v, %v, %v); want the intact %s", f, ok, err, older)
}
if got := due(now.Add(reaper.Day)); len(got) != 1 || got[older] == "" {
t.Fatalf("due after corruption = %v; want only %s", got, older)
}
if b, err := repo.LatestBackup(ctx, name); err != nil || b.ID != older {
t.Fatalf("LatestBackup = (%+v, %v); want %s", b, err, older)
}
if b, err := repo.BackupByID(ctx, newer); err != nil || !b.Corrupt {
t.Fatalf("BackupByID(corrupt) = (%+v, %v); want Corrupt", b, err)
}
if b, err := repo.BackupByID(ctx, older); err != nil || b.Corrupt {
t.Fatalf("BackupByID(intact) = (%+v, %v); want not Corrupt", b, err)
}
views, err := repo.AllBackups(ctx)
if err != nil {
t.Fatalf("AllBackups: %v", err)
}
seen := 0
for _, v := range views {
switch v.ID {
case newer:
seen++
if !v.Corrupt || v.VerifiedAt == nil || v.SkippedEntries != 0 {
t.Fatalf("listed corrupt backup = %+v; want corrupt, verified_at set", v)
}
case older:
seen++
if v.Corrupt || v.VerifiedAt == nil || v.SkippedEntries != 2 {
t.Fatalf("listed intact backup = %+v; want verified_at set and 2 skipped entries", v)
}
}
}
if seen != 2 {
t.Fatalf("AllBackups listed %d of the 2 backups", seen)
}
if excess, err := st.ExcessBackups(ctx, name, "inactive_15d", 1, ""); err != nil || len(excess) != 1 || excess[0].ID != newer {
t.Fatalf("ExcessBackups(keep 1) = (%+v, %v); want the corrupt %s pruned first", excess, err, newer)
}
live := func() map[string]bool {
t.Helper()
refs, err := st.LiveBackupRefs(ctx)
if err != nil {
t.Fatalf("LiveBackupRefs: %v", err)
}
out := map[string]bool{}
for _, r := range refs {
out[r] = true
}
return out
}
if l := live(); !l["/archives/"+older+".tar.gz"] || !l["/archives/"+newer+".tar.gz"] {
t.Fatalf("live refs miss a present backup")
}
if err := st.MarkBackupDeleted(ctx, newer, now); err != nil {
t.Fatalf("MarkBackupDeleted: %v", err)
}
if l := live(); l["/archives/"+newer+".tar.gz"] || !l["/archives/"+older+".tar.gz"] {
t.Fatalf("live refs after deleting %s still claim its archive", newer)
}
}
+50 -11
View File
@@ -53,13 +53,14 @@ func (s *PGStore) FreshBackup(ctx context.Context, server string, since time.Tim
// Only the reaper's own archives count, and only those taken since the // Only the reaper's own archives count, and only those taken since the
// current owner claimed the server: a manual backup may predate a panel edit // current owner claimed the server: a manual backup may predate a panel edit
// that did not move last_active_at, and an archive from before the claim is // that did not move last_active_at, and an archive from before the claim is
// the previous owner's world. // the previous owner's world. One found corrupt is never reused.
const q = `SELECT b.backup_ref, b.offsite_at IS NOT NULL FROM world_backups b const q = `SELECT b.id, b.backup_ref, COALESCE(b.sha256, ''), b.offsite_at IS NOT NULL FROM world_backups b
WHERE b.server_name = $1 AND b.status = 'present' AND b.reason = 'inactive_15d' AND b.created_at >= $2 WHERE b.server_name = $1 AND b.status = 'present' AND b.reason = 'inactive_15d' AND b.created_at >= $2
AND b.corrupt_at IS NULL
AND b.created_at >= COALESCE((SELECT s.claimed_at FROM servers s WHERE s.name = $1 AND s.deleted_at IS NULL), '-infinity') AND b.created_at >= COALESCE((SELECT s.claimed_at FROM servers s WHERE s.name = $1 AND s.deleted_at IS NULL), '-infinity')
ORDER BY b.offsite_at IS NOT NULL DESC, b.created_at DESC LIMIT 1` ORDER BY b.offsite_at IS NOT NULL DESC, b.created_at DESC LIMIT 1`
var f Fresh var f Fresh
switch err := s.db.QueryRowContext(ctx, q, server, since).Scan(&f.Ref, &f.Offsite); { switch err := s.db.QueryRowContext(ctx, q, server, since).Scan(&f.ID, &f.Ref, &f.SHA256, &f.Offsite); {
case err == sql.ErrNoRows: case err == sql.ErrNoRows:
return Fresh{}, false, nil return Fresh{}, false, nil
case err != nil: case err != nil:
@@ -70,10 +71,11 @@ func (s *PGStore) FreshBackup(ctx context.Context, server string, since time.Tim
func (s *PGStore) InsertBackup(ctx context.Context, rec BackupRecord) error { func (s *PGStore) InsertBackup(ctx context.Context, rec BackupRecord) error {
const q = `INSERT INTO world_backups const q = `INSERT INTO world_backups
(id, server_name, former_owner, backup_ref, size_bytes, reason, status, created_at, expires_at) (id, server_name, former_owner, backup_ref, size_bytes, reason, status, created_at, expires_at, sha256, skipped_entries)
VALUES ($1, $2, NULLIF($3, ''), $4, $5, $6, 'present', now(), $7)` VALUES ($1, $2, NULLIF($3, ''), $4, $5, $6, 'present', now(), $7, NULLIF($8, ''), $9)`
_, err := s.db.ExecContext(ctx, q, _, err := s.db.ExecContext(ctx, q,
rec.ID, rec.ServerName, rec.FormerOwner, rec.BackupRef, rec.SizeBytes, rec.Reason, rec.ExpiresAt) rec.ID, rec.ServerName, rec.FormerOwner, rec.BackupRef, rec.SizeBytes, rec.Reason, rec.ExpiresAt,
rec.SHA256, rec.SkippedEntries)
return err return err
} }
@@ -107,7 +109,7 @@ func (s *PGStore) PresentBackupBytes(ctx context.Context) (int64, error) {
} }
func (s *PGStore) EvictableBackups(ctx context.Context) ([]StoredBackup, error) { func (s *PGStore) EvictableBackups(ctx context.Context) ([]StoredBackup, error) {
const q = `SELECT id, server_name, backup_ref, size_bytes, reason FROM world_backups const q = `SELECT id, server_name, backup_ref, size_bytes, reason, COALESCE(sha256, '') FROM world_backups
WHERE status = 'present' AND (reason <> 'inactive_15d' OR offsite_at IS NOT NULL) WHERE status = 'present' AND (reason <> 'inactive_15d' OR offsite_at IS NOT NULL)
ORDER BY reason = 'inactive_15d', created_at ASC` ORDER BY reason = 'inactive_15d', created_at ASC`
return s.queryBackups(ctx, q) return s.queryBackups(ctx, q)
@@ -118,9 +120,9 @@ func (s *PGStore) EvictableBackups(ctx context.Context) ([]StoredBackup, error)
// set, is a backup id left out of the list whatever its age (the one a chained // set, is a backup id left out of the list whatever its age (the one a chained
// restore is about to extract). // restore is about to extract).
func (s *PGStore) ExcessBackups(ctx context.Context, server, reason string, keep int, protect string) ([]StoredBackup, error) { func (s *PGStore) ExcessBackups(ctx context.Context, server, reason string, keep int, protect string) ([]StoredBackup, error) {
const q = `SELECT id, server_name, backup_ref, size_bytes, reason FROM world_backups const q = `SELECT id, server_name, backup_ref, size_bytes, reason, COALESCE(sha256, '') FROM world_backups
WHERE server_name = $1 AND status = 'present' AND reason = $2 AND id <> $4 WHERE server_name = $1 AND status = 'present' AND reason = $2 AND id <> $4
ORDER BY created_at DESC OFFSET $3` ORDER BY corrupt_at IS NULL DESC, created_at DESC OFFSET $3`
out, err := s.queryBackups(ctx, q, server, reason, keep, protect) out, err := s.queryBackups(ctx, q, server, reason, keep, protect)
for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 {
out[i], out[j] = out[j], out[i] out[i], out[j] = out[j], out[i]
@@ -129,7 +131,7 @@ func (s *PGStore) ExcessBackups(ctx context.Context, server, reason string, keep
} }
func (s *PGStore) ListExpiredBackups(ctx context.Context, now time.Time) ([]StoredBackup, error) { func (s *PGStore) ListExpiredBackups(ctx context.Context, now time.Time) ([]StoredBackup, error) {
const q = `SELECT id, server_name, backup_ref, size_bytes, reason FROM world_backups const q = `SELECT id, server_name, backup_ref, size_bytes, reason, COALESCE(sha256, '') FROM world_backups
WHERE status = 'present' AND expires_at < $1 ORDER BY expires_at ASC` WHERE status = 'present' AND expires_at < $1 ORDER BY expires_at ASC`
return s.queryBackups(ctx, q, now) return s.queryBackups(ctx, q, now)
} }
@@ -143,7 +145,7 @@ func (s *PGStore) queryBackups(ctx context.Context, q string, args ...any) ([]St
var out []StoredBackup var out []StoredBackup
for rows.Next() { for rows.Next() {
var b StoredBackup var b StoredBackup
if err := rows.Scan(&b.ID, &b.ServerName, &b.BackupRef, &b.SizeBytes, &b.Reason); err != nil { if err := rows.Scan(&b.ID, &b.ServerName, &b.BackupRef, &b.SizeBytes, &b.Reason, &b.SHA256); err != nil {
return nil, err return nil, err
} }
out = append(out, b) out = append(out, b)
@@ -151,6 +153,43 @@ func (s *PGStore) queryBackups(ctx context.Context, q string, args ...any) ([]St
return out, rows.Err() return out, rows.Err()
} }
func (s *PGStore) BackupsToVerify(ctx context.Context, checkedBefore time.Time, limit int) ([]StoredBackup, error) {
const q = `SELECT id, server_name, backup_ref, size_bytes, reason, COALESCE(sha256, '') FROM world_backups
WHERE status = 'present' AND corrupt_at IS NULL AND (verified_at IS NULL OR verified_at < $1)
ORDER BY verified_at NULLS FIRST, created_at LIMIT $2`
return s.queryBackups(ctx, q, checkedBefore, limit)
}
func (s *PGStore) MarkBackupVerified(ctx context.Context, id, sha256 string, at time.Time) error {
_, err := s.db.ExecContext(ctx,
`UPDATE world_backups SET verified_at = $3, sha256 = COALESCE(sha256, NULLIF($2, '')) WHERE id = $1`,
id, sha256, at)
return err
}
func (s *PGStore) MarkBackupCorrupt(ctx context.Context, id string, at time.Time) error {
_, err := s.db.ExecContext(ctx,
`UPDATE world_backups SET corrupt_at = $2 WHERE id = $1 AND corrupt_at IS NULL`, id, at)
return err
}
func (s *PGStore) LiveBackupRefs(ctx context.Context) ([]string, error) {
rows, err := s.db.QueryContext(ctx, `SELECT backup_ref FROM world_backups WHERE status <> 'deleted'`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []string
for rows.Next() {
var ref string
if err := rows.Scan(&ref); err != nil {
return nil, err
}
out = append(out, ref)
}
return out, rows.Err()
}
func (s *PGStore) MarkBackupDeleted(ctx context.Context, id string, at time.Time) error { func (s *PGStore) MarkBackupDeleted(ctx context.Context, id string, at time.Time) error {
_, err := s.db.ExecContext(ctx, _, err := s.db.ExecContext(ctx,
`UPDATE world_backups SET status = 'deleted', deleted_at = $2 WHERE id = $1`, id, at) `UPDATE world_backups SET status = 'deleted', deleted_at = $2 WHERE id = $1`, id, at)
+173 -19
View File
@@ -97,6 +97,14 @@ type Config struct {
// world is deleted on the first run after the copy lands, normally the // world is deleted on the first run after the copy lands, normally the
// next day. // next day.
RequireOffsite bool RequireOffsite bool
// VerifyEvery is how often each stored archive is read back in full, and
// VerifyPerRun caps how many one run reads (the ones checked longest ago
// first), so bit rot in a backup is found before a restore needs it.
VerifyEvery time.Duration
VerifyPerRun int
// PartialAfter is how old an unfinished archive file must be before it is
// swept: longer than any archive takes to write.
PartialAfter time.Duration
} }
// DefaultConfig is the spec's §24 default window set. // DefaultConfig is the spec's §24 default window set.
@@ -110,6 +118,10 @@ func DefaultConfig() Config {
ManualRetention: 30 * Day, ManualRetention: 30 * Day,
ManualKeep: 5, ManualKeep: 5,
ManualCooldown: 10 * time.Minute, ManualCooldown: 10 * time.Minute,
VerifyEvery: 7 * Day,
VerifyPerRun: 10,
PartialAfter: 6 * time.Hour,
} }
} }
@@ -149,11 +161,17 @@ type BackupRecord struct {
SizeBytes int64 SizeBytes int64
Reason string Reason string
ExpiresAt time.Time ExpiresAt time.Time
// SHA256 is the archive's digest as written ("" when the backend keeps none)
// and SkippedEntries the world entries it could not hold.
SHA256 string
SkippedEntries int
} }
// Fresh is the backup FreshBackup found. // Fresh is the backup FreshBackup found.
type Fresh struct { type Fresh struct {
Ref string ID string
Ref string
SHA256 string
// Offsite reports that the archive has its off-site copy (offsite_at). // Offsite reports that the archive has its off-site copy (offsite_at).
Offsite bool Offsite bool
} }
@@ -166,6 +184,7 @@ type StoredBackup struct {
BackupRef string BackupRef string
SizeBytes int64 SizeBytes int64
Reason string Reason string
SHA256 string // "" when none was recorded
} }
// AuditRecord is a reaper-sourced audit_logs entry. The PG binding fills // AuditRecord is a reaper-sourced audit_logs entry. The PG binding fills
@@ -186,9 +205,10 @@ type Store interface {
// FreshBackup reports an existing present reaper archive (reason // FreshBackup reports an existing present reaper archive (reason
// inactive_15d) for server whose world is still current — created at or // inactive_15d) for server whose world is still current — created at or
// after since (the world's last_active_at) and after the current claim, // after since (the world's last_active_at) and after the current claim,
// preferring one already copied off-site. It makes a reap idempotent across // preferring one already copied off-site, and never one found corrupt. It
// a DeletePVC failure, and across the wait for the off-site copy: the retry // makes a reap idempotent across a DeletePVC failure, and across the wait for
// reuses the archive instead of writing a duplicate. // the off-site copy: the retry reuses the archive instead of writing a
// duplicate.
FreshBackup(ctx context.Context, server string, since time.Time) (b Fresh, ok bool, err error) FreshBackup(ctx context.Context, server string, since time.Time) (b Fresh, ok bool, err error)
// InsertBackup records a world_backups row (status=present). // InsertBackup records a world_backups row (status=present).
@@ -218,6 +238,19 @@ type Store interface {
// MarkBackupDeleted flips a backup to status=deleted, deleted_at=at. // MarkBackupDeleted flips a backup to status=deleted, deleted_at=at.
MarkBackupDeleted(ctx context.Context, id string, at time.Time) error MarkBackupDeleted(ctx context.Context, id string, at time.Time) error
// BackupsToVerify lists up to limit present backups not found corrupt and
// not read back since checkedBefore, the ones never read back first, then
// the ones read back longest ago.
BackupsToVerify(ctx context.Context, checkedBefore time.Time, limit int) ([]StoredBackup, error)
// MarkBackupVerified records a read-back that matched at `at`, and the
// archive's digest when none was recorded yet.
MarkBackupVerified(ctx context.Context, id, sha256 string, at time.Time) error
// MarkBackupCorrupt records a read-back that failed: the backup is no
// longer reused for a reap or offered for a restore.
MarkBackupCorrupt(ctx context.Context, id string, at time.Time) error
// LiveBackupRefs lists the backup_ref of every backup not deleted.
LiveBackupRefs(ctx context.Context) ([]string, error)
// Audit appends a reaper-sourced audit_logs row. // Audit appends a reaper-sourced audit_logs row.
Audit(ctx context.Context, rec AuditRecord) error Audit(ctx context.Context, rec AuditRecord) error
} }
@@ -276,12 +309,27 @@ type Summary struct {
// AwaitingOffsite are idle worlds that are archived and kept until the // AwaitingOffsite are idle worlds that are archived and kept until the
// archive's off-site copy lands. // archive's off-site copy lands.
AwaitingOffsite int AwaitingOffsite int
// Verified are archives read back in full and found matching; Corrupt are
// the ones that were not (marked, and never reused or restored from);
// VerifyFailed are the ones that could not be read back at all this run.
Verified int
Corrupt int
VerifyFailed int
// Swept are leftover files of interrupted archives removed; OrphanArchives
// are finished archives no backup records, kept for now (see
// backup.Swept); SweepFailed reports that the sweep did not complete.
Swept int
OrphanArchives int
SweepFailed bool
} }
// Failed reports whether the run left work undone: a server it could not // Failed reports whether the run left work undone or found damage: a server it
// process, or an expired backup it could not remove. The world is safe either // could not process, an expired backup it could not remove, an archive that did
// way, but the run did not do its job and whoever operates it must hear. // not read back or could not be read, or a sweep that did not complete. The
func (s Summary) Failed() bool { return s.Skipped > 0 || s.ExpireFailed > 0 } // worlds are safe either way, but whoever operates the run must hear.
func (s Summary) Failed() bool {
return s.Skipped > 0 || s.ExpireFailed > 0 || s.Corrupt > 0 || s.VerifyFailed > 0 || s.SweepFailed
}
func (r *Reaper) now() time.Time { func (r *Reaper) now() time.Time {
if r.Now != nil { if r.Now != nil {
@@ -340,6 +388,8 @@ func (r *Reaper) RunOnce(ctx context.Context) (Summary, error) {
} }
r.expireBackups(ctx, now, &sum) r.expireBackups(ctx, now, &sum)
r.verifyBackups(ctx, now, &sum)
r.sweepArchives(ctx, now, &sum)
return sum, nil return sum, nil
} }
@@ -393,32 +443,49 @@ func (r *Reaper) reap(ctx context.Context, now time.Time, c Candidate, crd Serve
if err != nil { if err != nil {
return fmt.Errorf("lookup fresh backup: %w", err) return fmt.Errorf("lookup fresh backup: %w", err)
} }
if ok {
// The archive may have sat on disk for days (a delete that failed, the
// wait for its off-site copy); it is about to become the only copy, so it
// is read back first. One that fails is marked and replaced by a fresh
// archive of the world, which is still there.
good, err := r.verifyOne(ctx, now, fresh.ID, fresh.Ref, fresh.SHA256, sum)
if err != nil {
return fmt.Errorf("read back archive %s: %w", fresh.ID, err)
}
ok = good
}
ref, offsite := fresh.Ref, fresh.Offsite ref, offsite := fresh.Ref, fresh.Offsite
if !ok { if !ok {
aref, size, err := r.Archiver.Archive(ctx, c.Name, crd.PVC) a, err := r.Archiver.Archive(ctx, c.Name, crd.PVC)
if err != nil { if err != nil {
// Red line ④: archive failed → the PVC is untouched, the world // Red line ④: archive failed → the PVC is untouched, the world
// survives, and this server is retried next run. // survives, and this server is retried next run.
return fmt.Errorf("archive: %w", err) return fmt.Errorf("archive: %w", err)
} }
if len(a.Skipped) > 0 {
r.log().Warn("reaper: archive leaves out entries that are not plain files or directories",
"server", c.Name, "count", len(a.Skipped), "first", a.Skipped[:min(len(a.Skipped), 5)])
}
rec := BackupRecord{ rec := BackupRecord{
ID: r.id(), ID: r.id(),
ServerName: c.Name, ServerName: c.Name,
FormerOwner: c.OwnerID, FormerOwner: c.OwnerID,
BackupRef: string(aref), BackupRef: string(a.Ref),
SizeBytes: size, SizeBytes: a.Size,
Reason: ReasonInactive, Reason: ReasonInactive,
ExpiresAt: now.Add(r.Cfg.Retention), ExpiresAt: now.Add(r.Cfg.Retention),
SHA256: a.SHA256,
SkippedEntries: len(a.Skipped),
} }
if err := r.Store.InsertBackup(ctx, rec); err != nil { if err := r.Store.InsertBackup(ctx, rec); err != nil {
// The archive exists but is untracked. Delete the orphan so it does // The archive exists but is untracked. Delete the orphan so it does
// not leak, then fail without touching the PVC. // not leak, then fail without touching the PVC.
if derr := r.Archiver.Delete(ctx, aref); derr != nil { if derr := r.Archiver.Delete(ctx, a.Ref); derr != nil {
r.log().Error("reaper: orphan archive cleanup failed", "server", c.Name, "ref", aref, "err", derr) r.log().Error("reaper: orphan archive cleanup failed", "server", c.Name, "ref", a.Ref, "err", derr)
} }
return fmt.Errorf("insert backup: %w", err) return fmt.Errorf("insert backup: %w", err)
} }
ref, offsite = string(aref), false ref, offsite = string(a.Ref), false
} }
// With an off-site bucket configured, the archive on this node's disk is // With an off-site bucket configured, the archive on this node's disk is
@@ -570,6 +637,93 @@ func (r *Reaper) expireBackups(ctx context.Context, now time.Time, sum *Summary)
} }
} }
// verifyBackups is the read-back pass: up to VerifyPerRun present archives not
// read back within VerifyEvery are read end to end and checked against their
// recorded digest. A backend that cannot read its archives back skips it.
func (r *Reaper) verifyBackups(ctx context.Context, now time.Time, sum *Summary) {
if _, ok := r.Archiver.(backup.Verifier); !ok || r.Cfg.VerifyPerRun <= 0 {
return
}
list, err := r.Store.BackupsToVerify(ctx, now.Add(-r.Cfg.VerifyEvery), r.Cfg.VerifyPerRun)
if err != nil {
r.log().Error("reaper: list backups to read back", "err", err)
sum.VerifyFailed++
return
}
for _, b := range list {
if _, err := r.verifyOne(ctx, now, b.ID, b.BackupRef, b.SHA256, sum); err != nil {
r.log().Error("reaper: read back archive", "id", b.ID, "server", b.ServerName, "err", err)
sum.VerifyFailed++
}
}
}
// verifyOne reads one archive back and records the outcome. It reports whether
// the archive is good; an error means it could not be read back at all (the
// store is not mounted, the run was cancelled), which says nothing about the
// archive. A backend that cannot read its archives back reports every archive
// good, as before read-backs existed.
func (r *Reaper) verifyOne(ctx context.Context, now time.Time, id, ref, want string, sum *Summary) (bool, error) {
v, ok := r.Archiver.(backup.Verifier)
if !ok {
return true, nil
}
got, err := v.Verify(ctx, backup.ArchiveRef(ref), want)
switch {
case errors.Is(err, backup.ErrCorrupt):
sum.Corrupt++
r.log().Error("reaper: archive is corrupt; it will not be reused or offered for restore", "id", id, "ref", ref, "err", err)
if err := r.Store.MarkBackupCorrupt(ctx, id, now); err != nil {
return false, fmt.Errorf("mark corrupt: %w", err)
}
return false, nil
case err != nil:
return false, err
}
if err := r.Store.MarkBackupVerified(ctx, id, got, now); err != nil {
r.log().Error("reaper: record read-back", "id", id, "err", err)
}
sum.Verified++
return true, nil
}
// sweepArchives removes what interrupted archives left in the store (see
// backup.Sweeper): unfinished files older than PartialAfter, and finished ones
// no backup records once they are older than Retention, the longest any backup
// is kept. Younger unrecorded archives are reported and kept.
func (r *Reaper) sweepArchives(ctx context.Context, now time.Time, sum *Summary) {
sw, ok := r.Archiver.(backup.Sweeper)
if !ok {
return
}
refs, err := r.Store.LiveBackupRefs(ctx)
if err != nil {
// Without the list every archive would look unclaimed.
r.log().Error("reaper: list recorded archives; sweep skipped", "err", err)
sum.SweepFailed = true
return
}
live := make(map[backup.ArchiveRef]bool, len(refs))
for _, ref := range refs {
live[backup.ArchiveRef(ref)] = true
}
res, err := sw.Sweep(ctx, func(ref backup.ArchiveRef) bool { return live[ref] },
now.Add(-r.Cfg.PartialAfter), now.Add(-r.Cfg.Retention))
if err != nil {
r.log().Error("reaper: sweep archive store", "err", err)
sum.SweepFailed = true
}
for _, p := range res.Removed {
r.log().Info("reaper: removed leftover of an interrupted archive", "path", p)
}
sum.Swept += len(res.Removed)
sum.OrphanArchives += len(res.Orphans)
if len(res.Orphans) > 0 {
r.log().Warn("reaper: archives no backup records are kept until they are older than the retention",
"count", len(res.Orphans), "bytes", res.OrphanBytes, "first", res.Orphans[:min(len(res.Orphans), 5)])
}
}
// formatRemaining renders an offset as the human-facing time left before reap. // formatRemaining renders an offset as the human-facing time left before reap.
func formatRemaining(d time.Duration) string { func formatRemaining(d time.Duration) string {
if d%Day == 0 { if d%Day == 0 {
+255 -6
View File
@@ -40,14 +40,48 @@ type fakeArchiver struct {
seq int seq int
} }
func (f *fakeArchiver) Archive(_ context.Context, server, _ string) (backup.ArchiveRef, int64, error) { func (f *fakeArchiver) Archive(_ context.Context, server, _ string) (backup.Archived, error) {
if f.archiveErr != nil { if f.archiveErr != nil {
return "", 0, f.archiveErr return backup.Archived{}, f.archiveErr
} }
f.archives++ f.archives++
f.seq++ f.seq++
f.rec.add("archive") f.rec.add("archive")
return backup.ArchiveRef(fmt.Sprintf("ref-%s-%d", server, f.seq)), 10, nil ref := fmt.Sprintf("ref-%s-%d", server, f.seq)
return backup.Archived{Ref: backup.ArchiveRef(ref), Size: 10, SHA256: "sha-" + ref}, nil
}
// checkingArchiver is a fakeArchiver that also reads archives back
// (backup.Verifier) and sweeps its store (backup.Sweeper).
type checkingArchiver struct {
*fakeArchiver
corrupt map[string]bool // refs that no longer read back
verifyErr error // every read-back fails with this
verified []string
sweepErr error
sweepLive func(backup.ArchiveRef) bool
sweepCutoffs []time.Time
sweepRemoved []string
sweepOrphaned []string
}
func (c *checkingArchiver) Verify(_ context.Context, ref backup.ArchiveRef, want string) (string, error) {
if c.verifyErr != nil {
return "", c.verifyErr
}
c.verified = append(c.verified, string(ref))
c.rec.add("verify")
if c.corrupt[string(ref)] {
return "", fmt.Errorf("%w: %s", backup.ErrCorrupt, ref)
}
return "sha-" + string(ref), nil
}
func (c *checkingArchiver) Sweep(_ context.Context, live func(backup.ArchiveRef) bool, partialBefore, orphanBefore time.Time) (backup.Swept, error) {
c.sweepLive = live
c.sweepCutoffs = []time.Time{partialBefore, orphanBefore}
return backup.Swept{Removed: c.sweepRemoved, Orphans: c.sweepOrphaned, OrphanBytes: int64(len(c.sweepOrphaned))}, c.sweepErr
} }
func (f *fakeArchiver) Restore(context.Context, backup.ArchiveRef, string) error { return nil } func (f *fakeArchiver) Restore(context.Context, backup.ArchiveRef, string) error { return nil }
@@ -109,6 +143,10 @@ type fakeBackup struct {
status string // present | deleted status string // present | deleted
createdAt, expires time.Time createdAt, expires time.Time
offsite bool offsite bool
sha string
skipped int
verifiedAt time.Time
corruptAt time.Time
} }
type fakeStore struct { type fakeStore struct {
@@ -121,6 +159,7 @@ type fakeStore struct {
released []string released []string
listErr error listErr error
insertErr error insertErr error
liveErr error
idn int idn int
} }
@@ -138,7 +177,7 @@ func (s *fakeStore) ListActiveServers(context.Context) ([]Candidate, error) {
func (s *fakeStore) FreshBackup(_ context.Context, server string, since time.Time) (Fresh, bool, error) { func (s *fakeStore) FreshBackup(_ context.Context, server string, since time.Time) (Fresh, bool, error) {
var found *fakeBackup var found *fakeBackup
for _, b := range s.backups { for _, b := range s.backups {
if b.server == server && b.status == "present" && b.reason == ReasonInactive && !b.createdAt.Before(since) { if b.server == server && b.status == "present" && b.reason == ReasonInactive && !b.createdAt.Before(since) && b.corruptAt.IsZero() {
if found == nil || (b.offsite && !found.offsite) { if found == nil || (b.offsite && !found.offsite) {
found = b found = b
} }
@@ -147,7 +186,7 @@ func (s *fakeStore) FreshBackup(_ context.Context, server string, since time.Tim
if found == nil { if found == nil {
return Fresh{}, false, nil return Fresh{}, false, nil
} }
return Fresh{Ref: found.ref, Offsite: found.offsite}, true, nil return Fresh{ID: found.id, Ref: found.ref, SHA256: found.sha, Offsite: found.offsite}, true, nil
} }
func (s *fakeStore) InsertBackup(_ context.Context, rec BackupRecord) error { func (s *fakeStore) InsertBackup(_ context.Context, rec BackupRecord) error {
@@ -156,7 +195,7 @@ func (s *fakeStore) InsertBackup(_ context.Context, rec BackupRecord) error {
} }
s.backups = append(s.backups, &fakeBackup{ s.backups = append(s.backups, &fakeBackup{
id: rec.ID, server: rec.ServerName, ref: rec.BackupRef, reason: rec.Reason, size: rec.SizeBytes, id: rec.ID, server: rec.ServerName, ref: rec.BackupRef, reason: rec.Reason, size: rec.SizeBytes,
status: "present", createdAt: s.clock, expires: rec.ExpiresAt, status: "present", createdAt: s.clock, expires: rec.ExpiresAt, sha: rec.SHA256, skipped: rec.SkippedEntries,
}) })
s.rec.add("insert") s.rec.add("insert")
return nil return nil
@@ -233,6 +272,73 @@ func (s *fakeStore) MarkBackupDeleted(_ context.Context, id string, at time.Time
return fmt.Errorf("no backup %s", id) return fmt.Errorf("no backup %s", id)
} }
func (s *fakeStore) BackupsToVerify(_ context.Context, checkedBefore time.Time, limit int) ([]StoredBackup, error) {
var ps []*fakeBackup
for _, b := range s.backups {
if b.status == "present" && b.corruptAt.IsZero() && b.verifiedAt.Before(checkedBefore) {
ps = append(ps, b)
}
}
sort.SliceStable(ps, func(i, j int) bool {
if !ps[i].verifiedAt.Equal(ps[j].verifiedAt) {
return ps[i].verifiedAt.Before(ps[j].verifiedAt)
}
return ps[i].createdAt.Before(ps[j].createdAt)
})
var out []StoredBackup
for _, b := range ps {
if len(out) == limit {
break
}
out = append(out, StoredBackup{ID: b.id, ServerName: b.server, BackupRef: b.ref, SizeBytes: b.size, Reason: b.reason, SHA256: b.sha})
}
return out, nil
}
func (s *fakeStore) find(id string) (*fakeBackup, error) {
for _, b := range s.backups {
if b.id == id {
return b, nil
}
}
return nil, fmt.Errorf("no backup %s", id)
}
func (s *fakeStore) MarkBackupVerified(_ context.Context, id, sha string, at time.Time) error {
b, err := s.find(id)
if err != nil {
return err
}
b.verifiedAt = at
if b.sha == "" {
b.sha = sha
}
return nil
}
func (s *fakeStore) MarkBackupCorrupt(_ context.Context, id string, at time.Time) error {
b, err := s.find(id)
if err != nil {
return err
}
b.corruptAt = at
s.rec.add("corrupt")
return nil
}
func (s *fakeStore) LiveBackupRefs(context.Context) ([]string, error) {
if s.liveErr != nil {
return nil, s.liveErr
}
var out []string
for _, b := range s.backups {
if b.status != "deleted" {
out = append(out, b.ref)
}
}
return out, nil
}
func (s *fakeStore) Audit(_ context.Context, rec AuditRecord) error { func (s *fakeStore) Audit(_ context.Context, rec AuditRecord) error {
s.audits = append(s.audits, rec) s.audits = append(s.audits, rec)
s.rec.add("audit:" + rec.Action) s.rec.add("audit:" + rec.Action)
@@ -808,3 +914,146 @@ func TestListErrorAbortsBatch(t *testing.T) {
t.Fatal("expected a hard error when listing servers fails") t.Fatal("expected a hard error when listing servers fails")
} }
} }
// checking swaps the fixture's archiver for one that reads archives back and
// sweeps.
func checking(r *Reaper, ar *fakeArchiver) *checkingArchiver {
c := &checkingArchiver{fakeArchiver: ar, corrupt: map[string]bool{}}
r.Archiver = c
return c
}
// An archive left from an earlier run is read back before the world it holds is
// deleted, and the read-back is recorded.
func TestReapReadsBackReusedArchive(t *testing.T) {
r, st, cl, ar := newReaper(DefaultConfig(),
Candidate{Name: "kilo", OwnerID: "user-1", LastActiveAt: idleBy(20 * Day)})
ca := checking(r, ar)
st.backups = []*fakeBackup{{id: "old", server: "kilo", ref: "ref-old", reason: ReasonInactive, size: 5,
status: "present", createdAt: idleBy(Day), expires: testNow.Add(89 * Day), sha: "sha-ref-old"}}
sum := mustRun(t, r)
if sum.WorldsReaped != 1 || ar.archives != 0 || len(cl.deletedPVCs) != 1 {
t.Fatalf("summary %+v archives=%d deleted=%v: want the checked archive reused", sum, ar.archives, cl.deletedPVCs)
}
if st.rec.events[0] != "verify" || st.rec.events[1] != "deletePVC" {
t.Errorf("events = %v, want the read-back before the delete", st.rec.events)
}
if !st.backups[0].verifiedAt.Equal(testNow) || !reflect.DeepEqual(ca.verified, []string{"ref-old"}) {
t.Errorf("read-back not recorded: %+v", st.backups[0])
}
}
// A reused archive that no longer reads back is marked corrupt and the world,
// still there, is archived afresh before it is deleted. The run reports it.
func TestReapReplacesCorruptArchive(t *testing.T) {
r, st, cl, ar := newReaper(DefaultConfig(),
Candidate{Name: "lima", OwnerID: "user-1", LastActiveAt: idleBy(20 * Day)})
ca := checking(r, ar)
ca.corrupt["ref-rotten"] = true
st.backups = []*fakeBackup{{id: "rotten", server: "lima", ref: "ref-rotten", reason: ReasonInactive, size: 5,
status: "present", createdAt: idleBy(Day), expires: testNow.Add(89 * Day)}}
sum := mustRun(t, r)
if sum.WorldsReaped != 1 || sum.Corrupt != 1 || !sum.Failed() {
t.Fatalf("summary = %+v, want reaped with one corrupt archive reported", sum)
}
want := []string{"verify", "corrupt", "archive", "insert", "deletePVC"}
if !reflect.DeepEqual(st.rec.events[:len(want)], want) {
t.Errorf("events = %v, want %v first", st.rec.events, want)
}
if st.backups[0].corruptAt.IsZero() || len(st.backups) != 2 || st.backups[1].sha != "sha-ref-lima-1" {
t.Errorf("backups = %+v %+v", st.backups[0], st.backups[len(st.backups)-1])
}
if len(cl.deletedPVCs) != 1 {
t.Errorf("deleted = %v", cl.deletedPVCs)
}
}
// When the archive cannot be read back at all (the store is not mounted), the
// world is kept and nothing is marked: the error says nothing about the archive.
func TestReapKeepsWorldWhenReadBackFails(t *testing.T) {
r, st, cl, ar := newReaper(DefaultConfig(),
Candidate{Name: "mike", OwnerID: "user-1", LastActiveAt: idleBy(20 * Day)})
ca := checking(r, ar)
ca.verifyErr = errors.New("input/output error")
st.backups = []*fakeBackup{{id: "b", server: "mike", ref: "ref-b", reason: ReasonInactive, size: 5,
status: "present", createdAt: idleBy(Day), expires: testNow.Add(89 * Day)}}
sum := mustRun(t, r)
if sum.WorldsReaped != 0 || sum.Skipped != 1 || cl.deletePVCCalls != 0 || ar.archives != 0 {
t.Fatalf("summary %+v deletes=%d archives=%d: want the world kept", sum, cl.deletePVCCalls, ar.archives)
}
if !st.backups[0].corruptAt.IsZero() {
t.Error("an unreadable store marked the archive corrupt")
}
}
// The read-back pass takes VerifyPerRun archives, never-checked ones first, and
// records what it finds; a corrupt one is marked and fails the run once.
func TestVerifyPassSamplesArchives(t *testing.T) {
cfg := DefaultConfig()
cfg.VerifyPerRun = 2
r, st, _, ar := newReaper(cfg)
ca := checking(r, ar)
ca.corrupt["ref-c"] = true
st.backups = []*fakeBackup{
{id: "a", ref: "ref-a", status: "present", createdAt: idleBy(9 * Day), verifiedAt: idleBy(8 * Day), expires: testNow.Add(30 * Day)},
{id: "b", ref: "ref-b", status: "present", createdAt: idleBy(9 * Day), verifiedAt: idleBy(Day), expires: testNow.Add(30 * Day)},
{id: "c", ref: "ref-c", status: "present", createdAt: idleBy(5 * Day), expires: testNow.Add(30 * Day)},
{id: "d", ref: "ref-d", status: "deleted", createdAt: idleBy(5 * Day), expires: testNow.Add(30 * Day)},
}
sum := mustRun(t, r)
if !reflect.DeepEqual(ca.verified, []string{"ref-c", "ref-a"}) {
t.Errorf("read back %v, want [ref-c ref-a]", ca.verified)
}
if sum.Verified != 1 || sum.Corrupt != 1 || !sum.Failed() {
t.Errorf("summary = %+v", sum)
}
if st.backups[2].corruptAt.IsZero() || !st.backups[0].verifiedAt.Equal(testNow) || st.backups[0].sha != "sha-ref-a" {
t.Errorf("outcomes not recorded: %+v %+v", st.backups[0], st.backups[2])
}
// The next run skips the corrupt one and what was checked within
// VerifyEvery; a week on, b and a are due again, b first.
ca.verified = nil
if sum := mustRun(t, r); sum.Corrupt != 0 || len(ca.verified) != 0 {
t.Errorf("second run read back %v (%+v), want nothing", ca.verified, sum)
}
r.Now = func() time.Time { return testNow.Add(7*Day + time.Hour) }
if mustRun(t, r); !reflect.DeepEqual(ca.verified, []string{"ref-b", "ref-a"}) {
t.Errorf("a week later read back %v, want [ref-b ref-a]", ca.verified)
}
}
// The sweep claims every archive a backup records, removes leftovers past
// PartialAfter and orphans past the retention, and never runs without the list
// of recorded archives.
func TestSweepPass(t *testing.T) {
r, st, _, ar := newReaper(DefaultConfig())
ca := checking(r, ar)
ca.sweepRemoved = []string{"/archives/.x-1.tar.gz.partial"}
ca.sweepOrphaned = []string{"/archives/x-2.tar.gz"}
st.backups = []*fakeBackup{
{id: "a", ref: "ref-a", status: "present", expires: testNow.Add(Day), verifiedAt: testNow},
{id: "d", ref: "ref-d", status: "deleted", expires: testNow.Add(Day)},
}
sum := mustRun(t, r)
if sum.Swept != 1 || sum.OrphanArchives != 1 || sum.Failed() {
t.Errorf("summary = %+v", sum)
}
if !ca.sweepLive("ref-a") || ca.sweepLive("ref-d") {
t.Error("sweep did not get the recorded archives as live")
}
cfg := DefaultConfig()
if want := []time.Time{testNow.Add(-cfg.PartialAfter), testNow.Add(-cfg.Retention)}; !reflect.DeepEqual(ca.sweepCutoffs, want) {
t.Errorf("cutoffs = %v, want %v", ca.sweepCutoffs, want)
}
ca.sweepLive = nil
st.liveErr = errors.New("db down")
if sum := mustRun(t, r); !sum.SweepFailed || ca.sweepLive != nil {
t.Errorf("sweep ran without the recorded archives: %+v", sum)
}
}
@@ -0,0 +1,16 @@
-- Integrity of world archives. sha256 is the digest of the archive as written
-- (NULL for archives written before digests were kept; the first read-back
-- records one). verified_at is when the reaper last read the archive back in
-- full and found it matching; corrupt_at is when a read-back failed, after
-- which the archive is never reused for a reap or offered for a restore.
-- skipped_entries counts the world entries the archive could not hold
-- (symbolic links, devices, sockets).
ALTER TABLE world_backups
ADD COLUMN sha256 text,
ADD COLUMN verified_at timestamptz,
ADD COLUMN corrupt_at timestamptz,
ADD COLUMN skipped_entries integer NOT NULL DEFAULT 0;
-- The reaper's read-back work list: the present archives checked longest ago.
CREATE INDEX world_backups_verify_idx ON world_backups (verified_at NULLS FIRST, created_at)
WHERE status = 'present' AND corrupt_at IS NULL;
@@ -55,6 +55,14 @@
"restoring": "Restoring…", "restoring": "Restoring…",
"stop_timeout": "The server did not stop in time. Close this window and try again shortly.", "stop_timeout": "The server did not stop in time. Close this window and try again shortly.",
"expired_cannot_restore": "This backup has expired and can no longer be restored.", "expired_cannot_restore": "This backup has expired and can no longer be restored.",
"corrupt_badge": "Corrupt",
"corrupt_hint": "A read-back found this archive no longer matches what was written, so it can't be restored intact. It stays listed until it expires so the loss is visible.",
"corrupt_short": "Corrupt",
"corrupt_cannot_restore": "This backup failed its read-back check and can't be restored — pick another backup.",
"verified_at": "Verified {{when}}",
"skipped_entries_one": "{{count}} entry not archived",
"skipped_entries_other": "{{count}} entries not archived",
"skipped_entries_hint": "The world held symlinks or other entries that aren't plain files, so the archive skipped them and a restore won't bring them back.",
"col_created": "Backup Time", "col_created": "Backup Time",
"col_size": "Size", "col_size": "Size",
"col_expires": "Expires", "col_expires": "Expires",
@@ -16,6 +16,7 @@
"not_running": "The server isn't running — wake it before managing access.", "not_running": "The server isn't running — wake it before managing access.",
"console_unavailable": "Can't reach the server console right now — try again shortly.", "console_unavailable": "Can't reach the server console right now — try again shortly.",
"no_backup": "There's no restorable backup for this server yet.", "no_backup": "There's no restorable backup for this server yet.",
"backup_corrupt": "This backup failed its read-back check and can't be restored intact — pick another backup.",
"not_stopped": "Stop the server completely before restoring — a restore overwrites the live world volume.", "not_stopped": "Stop the server completely before restoring — a restore overwrites the live world volume.",
"maintenance_in_progress": "This server's world is busy with a restore, backup or file write — try again once it finishes, usually within a minute or two.", "maintenance_in_progress": "This server's world is busy with a restore, backup or file write — try again once it finishes, usually within a minute or two.",
"no_world_volume": "This server has no world volume yet — start it once so it is created, then retry.", "no_world_volume": "This server has no world volume yet — start it once so it is created, then retry.",
@@ -55,6 +55,13 @@
"restoring": "正在恢复备份……", "restoring": "正在恢复备份……",
"stop_timeout": "服务器停止超时。请关闭此窗口,稍后重试。", "stop_timeout": "服务器停止超时。请关闭此窗口,稍后重试。",
"expired_cannot_restore": "此备份已过期,无法恢复。", "expired_cannot_restore": "此备份已过期,无法恢复。",
"corrupt_badge": "已损坏",
"corrupt_hint": "回读校验发现这份归档与写入时不一致,已无法完整恢复。它会保留到过期,方便排查。",
"corrupt_short": "已损坏",
"corrupt_cannot_restore": "这份备份回读校验未通过,无法恢复,请选择另一份备份。",
"verified_at": "{{when}}已校验",
"skipped_entries": "{{count}} 个条目未归档",
"skipped_entries_hint": "世界目录里有符号链接等非普通文件,归档时跳过了它们,恢复后这些条目不会存在。",
"col_created": "创建时间", "col_created": "创建时间",
"col_size": "大小", "col_size": "大小",
"col_expires": "过期时间", "col_expires": "过期时间",
@@ -16,6 +16,7 @@
"not_running": "服务器未在运行——请先启动它再管理访问权限。", "not_running": "服务器未在运行——请先启动它再管理访问权限。",
"console_unavailable": "暂时无法连接服务器控制台,请稍后重试。", "console_unavailable": "暂时无法连接服务器控制台,请稍后重试。",
"no_backup": "这台服务器暂时没有可回档的备份。", "no_backup": "这台服务器暂时没有可回档的备份。",
"backup_corrupt": "这份备份回读校验未通过,已无法完整恢复——请选择另一份备份。",
"not_stopped": "回档会覆盖世界的实时存储卷,请先把服务器完全停止再回档。", "not_stopped": "回档会覆盖世界的实时存储卷,请先把服务器完全停止再回档。",
"maintenance_in_progress": "这台服务器的世界正在回档、备份或写入文件——等它完成后再试,通常不超过一两分钟。", "maintenance_in_progress": "这台服务器的世界正在回档、备份或写入文件——等它完成后再试,通常不超过一两分钟。",
"no_world_volume": "这台服务器还没有世界卷——先启动一次让它创建,然后再试。", "no_world_volume": "这台服务器还没有世界卷——先启动一次让它创建,然后再试。",
+4
View File
@@ -743,6 +743,10 @@ export function humanizeError(e: unknown): string {
// the restore subsystem may be unwired (503 restore_unavailable). // the restore subsystem may be unwired (503 restore_unavailable).
case "no_backup": case "no_backup":
return t("no_backup"); return t("no_backup");
// The backup failed a read-back (sha256 or gzip/tar parse), so the server
// refuses to extract it over the world.
case "backup_corrupt":
return t("backup_corrupt");
case "not_stopped": case "not_stopped":
return t("not_stopped"); return t("not_stopped");
// World-volume lock: a restore, backup or file write is running on this // World-volume lock: a restore, backup or file write is running on this
+15 -4
View File
@@ -142,10 +142,11 @@ export interface FleetServer {
* by handle; restore resolves the latest present backup server-side. * by handle; restore resolves the latest present backup server-side.
* *
* Only `status: "present"` rows are ever listed (the query filters them) and the * Only `status: "present"` rows are ever listed (the query filters them) and the
* list is created_at-descending, so the FIRST row for a given server is exactly * list is created_at-descending, so the first row for a given server that is not
* the one a restore would recover (LatestBackup's WHERE mirrors this) — the UI must * `corrupt` is exactly the one a restore would recover (LatestBackup's WHERE
* name that row, not a plausible proxy. `reason`/`status` cross an unvalidated JSON * mirrors this) — the UI must name that row, not a plausible proxy.
* boundary; render unknown values tolerantly. */ * `reason`/`status` cross an unvalidated JSON boundary; render unknown values
* tolerantly. */
export interface BackupView { export interface BackupView {
id: string; id: string;
server_name: string; server_name: string;
@@ -156,6 +157,16 @@ export interface BackupView {
status: string; status: string;
created_at: string; created_at: string;
expires_at: string; expires_at: string;
/** True once the archive failed a read-back (its sha256 or its gzip/tar did
* not check out). It stays listed so the loss is visible, and the API refuses
* to restore it (409 backup_corrupt). */
corrupt?: boolean;
/** When the reaper last read the archive back intact. Absent until the first
* read-back after it was written. */
verified_at?: string;
/** How many entries of the world were not plain files or directories
* (symlinks, sockets) and so are not in the archive. */
skipped_entries?: number;
} }
/** ServerJob is one row of GET /api/v1/servers/{name}/jobs — the observable /** ServerJob is one row of GET /api/v1/servers/{name}/jobs — the observable
+70 -12
View File
@@ -1,6 +1,7 @@
import { useEffect, useRef, useState } from "react"; import { useEffect, useRef, useState } from "react";
import { useParams } from "react-router-dom"; import { useParams } from "react-router-dom";
import { import {
AlertTriangle,
Archive, Archive,
CheckCircle2, CheckCircle2,
Clock, Clock,
@@ -8,6 +9,7 @@ import {
Loader2, Loader2,
RotateCcw, RotateCcw,
ShieldCheck, ShieldCheck,
ShieldX,
UserMinus, UserMinus,
XCircle, XCircle,
} from "lucide-react"; } from "lucide-react";
@@ -36,14 +38,12 @@ import { formatBytes, formatRelative, formatAbsolute, isExpired } from "@/lib/fo
import { cn } from "@/lib/utils"; import { cn } from "@/lib/utils";
import type { BackupView, ServerJob } from "@/lib/types"; import type { BackupView, ServerJob } from "@/lib/types";
/** LatestBackupCard renders the most-recent backup as the restore card — the one a /** BackupRow is one backup in the table, with its own restore action. `isLatest`
* restore actually recovers (the list is created_at-descending and the backend's * marks the row a restore with no pick recovers: the newest one that is not
* LatestBackup selects the same present row), with the restore action beneath. Older * corrupt, which is what the backend's LatestBackup selects. Under the reason it
* archives are shown separately as a compact, read-only history (HistoryRow): a restore * shows what the reaper's read-back found — corrupt (restore refused), verified,
* ALWAYS recovers this latest one, so giving an older backup an action card of its own * or entries the archive could not hold. `showOwner` surfaces the former owner
* would falsely imply you could restore (or delete) it — the backend offers neither. * (admins list every world's backups; a user only ever sees their own). */
* `showOwner` surfaces the former owner (admins list every world's backups; a user only
* ever sees their own). */
function BackupRow({ function BackupRow({
b, b,
isLatest, isLatest,
@@ -93,6 +93,7 @@ function BackupRow({
<Clock className="h-3 w-3 text-muted-foreground/60 shrink-0" /> <Clock className="h-3 w-3 text-muted-foreground/60 shrink-0" />
<span>{reasonLabel}</span> <span>{reasonLabel}</span>
</div> </div>
<IntegrityNote b={b} now={now} locale={locale} />
</div> </div>
</td> </td>
<td className="px-4 py-3.5 whitespace-nowrap text-muted-foreground"> <td className="px-4 py-3.5 whitespace-nowrap text-muted-foreground">
@@ -119,7 +120,11 @@ function BackupRow({
</td> </td>
)} )}
<td className="px-4 py-3.5 whitespace-nowrap text-right"> <td className="px-4 py-3.5 whitespace-nowrap text-right">
{!expired ? ( {b.corrupt ? (
<span className="text-xs text-destructive/70 font-medium px-3 py-1.5">
{t("corrupt_short")}
</span>
) : !expired ? (
<RestoreControls <RestoreControls
serverName={serverName} serverName={serverName}
backup={b} backup={b}
@@ -140,6 +145,48 @@ function BackupRow({
); );
} }
/** IntegrityNote is what the reaper's read-back says about one archive: corrupt
* (it no longer matches what was written, so it cannot be restored), when it
* was last read back intact, and how many entries of the world it could not
* hold. A backup not read back yet shows nothing, as before. */
function IntegrityNote({ b, now, locale }: { b: BackupView; now: number; locale: string }) {
const { t } = useTranslation("backups");
const skipped = b.skipped_entries ?? 0;
if (!b.corrupt && !b.verified_at && skipped === 0) return null;
return (
<div className="flex flex-wrap items-center gap-x-2.5 gap-y-0.5 text-[11px]">
{b.corrupt ? (
<span
className="inline-flex items-center gap-1 font-medium text-destructive"
title={t("corrupt_hint")}
>
<ShieldX className="h-3 w-3 shrink-0" />
{t("corrupt_badge")}
</span>
) : (
b.verified_at && (
<span
className="inline-flex items-center gap-1 text-muted-foreground"
title={formatAbsolute(b.verified_at, locale)}
>
<ShieldCheck className="h-3 w-3 shrink-0 text-emerald-500/80" />
{t("verified_at", { when: formatRelative(b.verified_at, now, locale) })}
</span>
)
)}
{skipped > 0 && (
<span
className="inline-flex items-center gap-1 text-amber-600 dark:text-amber-500"
title={t("skipped_entries_hint")}
>
<AlertTriangle className="h-3 w-3 shrink-0" />
{t("skipped_entries", { count: skipped })}
</span>
)}
</div>
);
}
/** JobStateBadge renders one async Job's state (backup/restore Job history). The /** JobStateBadge renders one async Job's state (backup/restore Job history). The
* state vocabulary is the API's ("running" | "succeeded" | "failed"); anything * state vocabulary is the API's ("running" | "succeeded" | "failed"); anything
* unknown is shown verbatim rather than hidden. */ * unknown is shown verbatim rather than hidden. */
@@ -265,6 +312,15 @@ function RestoreControls({
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
const [safety, setSafety] = useState(true); const [safety, setSafety] = useState(true);
if (backup.corrupt) {
if (layout === "row") return null;
return (
<p className="mt-4 border-t border-primary/20 pt-4 text-xs text-destructive">
{t("corrupt_cannot_restore")}
</p>
);
}
if (isExpired(backup.expires_at, now)) { if (isExpired(backup.expires_at, now)) {
if (layout === "row") return null; if (layout === "row") return null;
return ( return (
@@ -525,10 +581,12 @@ export function ServerBackups() {
const now = Date.now(); const now = Date.now();
const locale = i18n.language; const locale = i18n.language;
// The global list, narrowed to this server. Already created_at-descending from the // The global list, narrowed to this server. Already created_at-descending from the
// API, but re-sorted defensively so all[0] is unambiguously the restore target. // API, but re-sorted defensively; the newest backup that is not corrupt is the
// one a restore with no pick recovers.
const all = (backupsQ.data ?? []) const all = (backupsQ.data ?? [])
.filter((b) => b.server_name === name) .filter((b) => b.server_name === name)
.sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at)); .sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at));
const latestID = all.find((b) => !b.corrupt)?.id;
const header = ( const header = (
<PageHeader <PageHeader
@@ -597,11 +655,11 @@ export function ServerBackups() {
</tr> </tr>
</thead> </thead>
<tbody className="divide-y divide-border"> <tbody className="divide-y divide-border">
{all.map((b, idx) => ( {all.map((b) => (
<BackupRow <BackupRow
key={b.id} key={b.id}
b={b} b={b}
isLatest={idx === 0} isLatest={b.id === latestID}
now={now} now={now}
locale={locale} locale={locale}
showOwner={isAdmin} showOwner={isAdmin}