feat(backup): 归档先写 .partial 再 fsync 改名并记录 sha256,删除原件前回读校验,reaper 抽样巡检与清扫半截归档,保留权限与 mtime,面板标出损坏与已校验
This commit is contained in:
25 files changed
+1489
-125
No files matched your search
@@ -4,7 +4,11 @@
|
||||
// ArchiveRef is deliberately opaque.
|
||||
package backup
|
||||
|
||||
import "context"
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ArchiveRef is an opaque handle to a stored world archive. Depending on the
|
||||
// backend it may be a tar path, a VolumeSnapshot name, or a Longhorn backup URL.
|
||||
@@ -15,15 +19,64 @@ type ArchiveRef string
|
||||
// backends (VolumeSnapshot/Longhorn) cannot produce an io.Reader — they create
|
||||
// K8s objects referencing the source PVC (spec §19).
|
||||
type WorldArchiver interface {
|
||||
// Archive captures the world living on pvc for server and returns an opaque
|
||||
// ref plus the stored size in bytes.
|
||||
Archive(ctx context.Context, server, pvc string) (ref ArchiveRef, size int64, err error)
|
||||
// Archive captures the world living on pvc for server. It returns only once
|
||||
// the archive is complete and durable: a failed or interrupted Archive leaves
|
||||
// nothing that could be mistaken for a finished archive.
|
||||
Archive(ctx context.Context, server, pvc string) (Archived, error)
|
||||
// Restore writes a previously archived world into targetPVC.
|
||||
Restore(ctx context.Context, ref ArchiveRef, targetPVC string) error
|
||||
// Delete removes the archive identified by ref.
|
||||
Delete(ctx context.Context, ref ArchiveRef) error
|
||||
}
|
||||
|
||||
// Archived is what one Archive call stored.
|
||||
type Archived struct {
|
||||
Ref ArchiveRef
|
||||
Size int64 // stored bytes
|
||||
// SHA256 is the hex digest of the stored archive, "" when the backend keeps
|
||||
// none. world_backups.sha256 records it, so a later Verify tells an archive
|
||||
// that rotted on disk from a good one.
|
||||
SHA256 string
|
||||
// Skipped lists the world entries the archive leaves out (symbolic links,
|
||||
// devices, sockets, named pipes), relative to the world root.
|
||||
Skipped []string
|
||||
}
|
||||
|
||||
// ErrCorrupt marks an archive that cannot be read back in full, or whose bytes
|
||||
// no longer match the checksum recorded for it. It is the only Verify error that
|
||||
// condemns the archive: any other (a mount that is not there, a cancelled
|
||||
// context) says nothing about it.
|
||||
var ErrCorrupt = errors.New("backup: archive corrupt")
|
||||
|
||||
// Verifier is implemented by backends that can read a stored archive back end to
|
||||
// end. Verify returns the archive's SHA256 as read; want, when not "", is the
|
||||
// digest it must match (an archive recorded before checksums were kept has
|
||||
// none, and its read-back establishes one).
|
||||
type Verifier interface {
|
||||
Verify(ctx context.Context, ref ArchiveRef, want string) (sha256 string, err error)
|
||||
}
|
||||
|
||||
// Sweeper is implemented by backends that can find what interrupted archives
|
||||
// leave behind.
|
||||
type Sweeper interface {
|
||||
// Sweep removes every unfinished archive last written before partialBefore.
|
||||
// A finished archive that live does not claim (its record was never inserted)
|
||||
// is removed once it was last written before orphanBefore, and reported and
|
||||
// kept until then.
|
||||
Sweep(ctx context.Context, live func(ArchiveRef) bool, partialBefore, orphanBefore time.Time) (Swept, error)
|
||||
}
|
||||
|
||||
// Swept is what one Sweep found.
|
||||
type Swept struct {
|
||||
Removed []string // paths removed
|
||||
// Orphans are finished archives no record claims that are still young enough
|
||||
// to keep. A backup whose record insert failed looks like this, and so do the
|
||||
// archives of a store brought back before the database that records them:
|
||||
// removing them early could throw away the only copy of a world.
|
||||
Orphans []string
|
||||
OrphanBytes int64
|
||||
}
|
||||
|
||||
// PVCResolver maps a PVC name to the local filesystem path where it is mounted.
|
||||
// In production the reaper Job mounts the source/backup PVCs and supplies a
|
||||
// resolver over those mount points; tests supply temp dirs.
|
||||
|
||||
+314
-45
@@ -4,13 +4,19 @@ import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -34,38 +40,218 @@ func (t *TarLocal) now() time.Time {
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// partialSuffix marks an archive still being written. Its file is the final
|
||||
// name hidden behind a leading dot, so a listing of the store shows finished
|
||||
// archives only and a leftover still says whose it was.
|
||||
const partialSuffix = ".partial"
|
||||
|
||||
// archiveName matches the finished archives Archive names (<server>-<unix
|
||||
// nanoseconds>.tar.gz); partialName matches their unfinished form. Sweep touches
|
||||
// nothing else in the store.
|
||||
var (
|
||||
archiveName = regexp.MustCompile(`^[^.].*-[0-9]+\.tar\.gz$`)
|
||||
partialName = regexp.MustCompile(`^\..*-[0-9]+\.tar\.gz\.partial$`)
|
||||
)
|
||||
|
||||
// Archive tars+gzips the world on pvc into BackupRoot and returns the archive
|
||||
// path as the opaque ref plus its on-disk size.
|
||||
func (t *TarLocal) Archive(ctx context.Context, server, pvc string) (ArchiveRef, int64, error) {
|
||||
// path as the opaque ref, with its size and SHA256.
|
||||
//
|
||||
// The archive is written under its hidden .partial name, flushed to disk, read
|
||||
// back in full, and only then renamed to its final name and the rename flushed
|
||||
// too. A Job killed at its deadline or a node that loses power mid-write leaves
|
||||
// a .partial for Sweep, never a truncated file under a name that looks finished,
|
||||
// and the world is deleted only after an archive that has already been read
|
||||
// back whole.
|
||||
func (t *TarLocal) Archive(ctx context.Context, server, pvc string) (Archived, error) {
|
||||
srcDir, err := t.Resolve(pvc)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
return Archived{}, err
|
||||
}
|
||||
if err := os.MkdirAll(t.BackupRoot, 0o750); err != nil {
|
||||
return "", 0, fmt.Errorf("backup: mkdir backup root: %w", err)
|
||||
return Archived{}, fmt.Errorf("backup: mkdir backup root: %w", err)
|
||||
}
|
||||
name := fmt.Sprintf("%s-%d.tar.gz", server, t.now().UTC().UnixNano())
|
||||
dest := filepath.Join(t.BackupRoot, name)
|
||||
tmp := filepath.Join(t.BackupRoot, "."+name+partialSuffix)
|
||||
|
||||
f, err := os.Create(dest)
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("backup: create archive: %w", err)
|
||||
return Archived{}, fmt.Errorf("backup: create archive: %w", err)
|
||||
}
|
||||
if err := writeTarGz(ctx, f, srcDir); err != nil {
|
||||
f.Close()
|
||||
os.Remove(dest)
|
||||
return "", 0, err
|
||||
h := sha256.New()
|
||||
st, err := writeTarGz(ctx, io.MultiWriter(f, h), srcDir)
|
||||
if err == nil {
|
||||
if err = f.Sync(); err != nil {
|
||||
err = fmt.Errorf("backup: sync archive: %w", err)
|
||||
}
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(dest)
|
||||
return "", 0, fmt.Errorf("backup: close archive: %w", err)
|
||||
if cerr := f.Close(); err == nil && cerr != nil {
|
||||
err = fmt.Errorf("backup: close archive: %w", cerr)
|
||||
}
|
||||
if err != nil {
|
||||
os.Remove(tmp)
|
||||
return Archived{}, err
|
||||
}
|
||||
sum := hex.EncodeToString(h.Sum(nil))
|
||||
|
||||
// Read back what landed: the gzip CRC, every tar header and entry, and the
|
||||
// entry count must all come out as written.
|
||||
entries, _, err := verifyArchive(ctx, tmp, sum)
|
||||
if err == nil && entries != st.entries {
|
||||
err = fmt.Errorf("%w: %s: %d entries read back, %d written", ErrCorrupt, tmp, entries, st.entries)
|
||||
}
|
||||
if err != nil {
|
||||
os.Remove(tmp)
|
||||
return Archived{}, err
|
||||
}
|
||||
if err := os.Rename(tmp, dest); err != nil {
|
||||
os.Remove(tmp)
|
||||
return Archived{}, fmt.Errorf("backup: name archive: %w", err)
|
||||
}
|
||||
if err := syncDir(t.BackupRoot); err != nil {
|
||||
os.Remove(dest)
|
||||
return Archived{}, fmt.Errorf("backup: sync backup root: %w", err)
|
||||
}
|
||||
info, err := os.Stat(dest)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("backup: stat archive: %w", err)
|
||||
return Archived{}, fmt.Errorf("backup: stat archive: %w", err)
|
||||
}
|
||||
return ArchiveRef(dest), info.Size(), nil
|
||||
return Archived{Ref: ArchiveRef(dest), Size: info.Size(), SHA256: sum, Skipped: st.skipped}, nil
|
||||
}
|
||||
|
||||
// Verify reads the archive at ref back end to end (see verifyArchive).
|
||||
func (t *TarLocal) Verify(ctx context.Context, ref ArchiveRef, want string) (string, error) {
|
||||
_, sum, err := verifyArchive(ctx, string(ref), want)
|
||||
return sum, err
|
||||
}
|
||||
|
||||
// verifyArchive reads the archive at p through gzip and tar to the last byte and
|
||||
// returns its entry count and SHA256. Anything that does not read back — a
|
||||
// missing file, a torn gzip stream, a CRC or length mismatch, a bad tar header,
|
||||
// or a digest other than want (when want is not "") — is ErrCorrupt. Only an
|
||||
// error opening a file that is there, or the context, is not.
|
||||
func verifyArchive(ctx context.Context, p, want string) (int, string, error) {
|
||||
f, err := os.Open(p)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return 0, "", fmt.Errorf("%w: %s is missing", ErrCorrupt, p)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("backup: open archive: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
raw := io.TeeReader(f, h)
|
||||
corrupt := func(err error) (int, string, error) {
|
||||
if ctx.Err() != nil {
|
||||
return 0, "", ctx.Err()
|
||||
}
|
||||
return 0, "", fmt.Errorf("%w: %s: %v", ErrCorrupt, p, err)
|
||||
}
|
||||
|
||||
gz, err := gzip.NewReader(raw)
|
||||
if err != nil {
|
||||
return corrupt(err)
|
||||
}
|
||||
tr := tar.NewReader(gz)
|
||||
entries := 0
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return 0, "", ctx.Err()
|
||||
}
|
||||
_, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return corrupt(err)
|
||||
}
|
||||
if _, err := io.Copy(io.Discard, tr); err != nil {
|
||||
return corrupt(err)
|
||||
}
|
||||
entries++
|
||||
}
|
||||
// The tar end marker is not the end of the gzip member: reading on to EOF is
|
||||
// what checks the CRC and length in the gzip trailer.
|
||||
if _, err := io.Copy(io.Discard, gz); err != nil {
|
||||
return corrupt(err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
return corrupt(err)
|
||||
}
|
||||
if _, err := io.Copy(io.Discard, raw); err != nil {
|
||||
return corrupt(err)
|
||||
}
|
||||
sum := hex.EncodeToString(h.Sum(nil))
|
||||
if want != "" && sum != want {
|
||||
return 0, sum, fmt.Errorf("%w: %s: sha256 %s, recorded %s", ErrCorrupt, p, sum, want)
|
||||
}
|
||||
return entries, sum, nil
|
||||
}
|
||||
|
||||
// Sweep removes the leftovers of archives that never finished (see Sweeper). It
|
||||
// only looks at the top level of BackupRoot and only at names Archive writes;
|
||||
// partialBefore must leave room for the longest Archive, and orphanBefore for
|
||||
// the insert of the record that follows it at the very least.
|
||||
func (t *TarLocal) Sweep(ctx context.Context, live func(ArchiveRef) bool, partialBefore, orphanBefore time.Time) (Swept, error) {
|
||||
var out Swept
|
||||
ents, err := os.ReadDir(t.BackupRoot)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return out, nil
|
||||
}
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("backup: list backup root: %w", err)
|
||||
}
|
||||
var errs []error
|
||||
for _, e := range ents {
|
||||
if ctx.Err() != nil {
|
||||
return out, ctx.Err()
|
||||
}
|
||||
name := e.Name()
|
||||
if !e.Type().IsRegular() {
|
||||
continue
|
||||
}
|
||||
p := filepath.Join(t.BackupRoot, name)
|
||||
cutoff := partialBefore
|
||||
switch {
|
||||
case partialName.MatchString(name):
|
||||
case archiveName.MatchString(name):
|
||||
if live(ArchiveRef(p)) {
|
||||
continue
|
||||
}
|
||||
cutoff = orphanBefore
|
||||
default:
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil || !info.ModTime().Before(partialBefore) {
|
||||
continue
|
||||
}
|
||||
if !info.ModTime().Before(cutoff) {
|
||||
out.Orphans = append(out.Orphans, p)
|
||||
out.OrphanBytes += info.Size()
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(p); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
out.Removed = append(out.Removed, p)
|
||||
}
|
||||
return out, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// syncDir flushes a change to a directory's entries (a rename) to disk. A
|
||||
// filesystem that cannot sync a directory has no stronger promise to give.
|
||||
func syncDir(dir string) error {
|
||||
d, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer d.Close()
|
||||
if err := d.Sync(); err != nil && !errors.Is(err, syscall.EINVAL) && !errors.Is(err, syscall.ENOTSUP) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Restore extracts the archive at ref into the world mount for targetPVC,
|
||||
@@ -74,10 +260,16 @@ func (t *TarLocal) Archive(ctx context.Context, server, pvc string) (ArchiveRef,
|
||||
// griefer's chunks). It extracts over the target, then removes any pre-existing
|
||||
// entry the archive did not contain.
|
||||
//
|
||||
// The prune runs only after a fully successful extract: a corrupt or truncated
|
||||
// archive fails before the prune, leaving the target as a (recoverable) partial
|
||||
// overlay rather than a destroyed world. The archive is retained on restore, so
|
||||
// such a failure is recoverable by re-running the Job.
|
||||
// The archive is read back in full before anything is extracted, so a corrupt
|
||||
// or truncated archive fails with the world untouched. The prune runs only after
|
||||
// a fully successful extract: an extract that still fails (the volume fills up)
|
||||
// leaves the target as a recoverable partial overlay rather than a destroyed
|
||||
// world. The archive is retained on restore, so such a failure is recoverable by
|
||||
// re-running the Job.
|
||||
//
|
||||
// Files and directories get back the permission bits and modification times the
|
||||
// archive recorded. Ownership is left to the server: every start re-owns the
|
||||
// world volume to the game uid (felis init-volume).
|
||||
//
|
||||
// A top-level lost+found is never a prune target. It is a filesystem artifact
|
||||
// (root-owned, mode 0700) that the non-root restore Pod cannot delete anyway,
|
||||
@@ -90,17 +282,23 @@ func (t *TarLocal) Restore(ctx context.Context, ref ArchiveRef, targetPVC string
|
||||
if err := os.MkdirAll(dstDir, 0o750); err != nil {
|
||||
return fmt.Errorf("backup: mkdir restore target: %w", err)
|
||||
}
|
||||
if _, _, err := verifyArchive(ctx, string(ref), ""); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.Open(string(ref))
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: open archive: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
keep, err := readTarGz(ctx, f, dstDir)
|
||||
keep, dirs, err := readTarGz(ctx, f, dstDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return pruneToManifest(dstDir, keep)
|
||||
if err := pruneToManifest(dstDir, keep); err != nil {
|
||||
return err
|
||||
}
|
||||
return restoreDirMeta(dirs)
|
||||
}
|
||||
|
||||
// pruneToManifest removes every entry under dstDir whose archive-relative path
|
||||
@@ -151,7 +349,18 @@ func (t *TarLocal) Delete(_ context.Context, ref ArchiveRef) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeTarGz(ctx context.Context, w io.Writer, srcDir string) error {
|
||||
// tarStats is what writeTarGz put in the archive and what it left out.
|
||||
type tarStats struct {
|
||||
entries int
|
||||
skipped []string
|
||||
}
|
||||
|
||||
// writeTarGz archives srcDir (not the root entry itself) as gzip+tar. Each entry
|
||||
// keeps its permission bits, without setuid, setgid and sticky, and its
|
||||
// modification time. Entries other than regular files and directories are left
|
||||
// out and listed in the stats.
|
||||
func writeTarGz(ctx context.Context, w io.Writer, srcDir string) (tarStats, error) {
|
||||
var st tarStats
|
||||
gz := gzip.NewWriter(w)
|
||||
tw := tar.NewWriter(gz)
|
||||
|
||||
@@ -173,12 +382,17 @@ func writeTarGz(ctx context.Context, w io.Writer, srcDir string) error {
|
||||
// Normalize to forward slashes so archives are portable.
|
||||
name := filepath.ToSlash(rel)
|
||||
|
||||
mode := int64(info.Mode().Perm())
|
||||
switch {
|
||||
case info.IsDir():
|
||||
hdr := &tar.Header{Name: name + "/", Mode: 0o750, Typeflag: tar.TypeDir}
|
||||
return tw.WriteHeader(hdr)
|
||||
hdr := &tar.Header{Name: name + "/", Mode: mode, ModTime: info.ModTime(), Typeflag: tar.TypeDir}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
st.entries++
|
||||
return nil
|
||||
case info.Mode().IsRegular():
|
||||
hdr := &tar.Header{Name: name, Mode: 0o640, Size: info.Size(), Typeflag: tar.TypeReg}
|
||||
hdr := &tar.Header{Name: name, Mode: mode, ModTime: info.ModTime(), Size: info.Size(), Typeflag: tar.TypeReg}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -187,24 +401,37 @@ func writeTarGz(ctx context.Context, w io.Writer, srcDir string) error {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
_, err = io.Copy(tw, src)
|
||||
return err
|
||||
if _, err := io.Copy(tw, src); err != nil {
|
||||
return err
|
||||
}
|
||||
st.entries++
|
||||
return nil
|
||||
default:
|
||||
// Skip symlinks/devices/sockets: a world directory should be plain
|
||||
// files, and refusing the rest avoids surprising archive contents.
|
||||
// A symlink could point anywhere on the node, and a device or socket
|
||||
// has no bytes to keep: a world is plain files. What is left out is
|
||||
// reported, so a backup never drops something silently.
|
||||
st.skipped = append(st.skipped, name)
|
||||
return nil
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup: tar walk: %w", err)
|
||||
return st, fmt.Errorf("backup: tar walk: %w", err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return fmt.Errorf("backup: close tar: %w", err)
|
||||
return st, fmt.Errorf("backup: close tar: %w", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
return fmt.Errorf("backup: close gzip: %w", err)
|
||||
return st, fmt.Errorf("backup: close gzip: %w", err)
|
||||
}
|
||||
return nil
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// dirMeta is a directory's recorded permission bits and modification time,
|
||||
// applied once nothing more is written into it.
|
||||
type dirMeta struct {
|
||||
path string
|
||||
mode fs.FileMode
|
||||
mtime time.Time
|
||||
}
|
||||
|
||||
// readTarGz extracts the gzip+tar stream into dstDir and returns the keep-set:
|
||||
@@ -213,33 +440,39 @@ func writeTarGz(ctx context.Context, w io.Writer, srcDir string) error {
|
||||
// target files for replace semantics. On any error the keep-set is incomplete
|
||||
// and must not be used to prune (a partial manifest would delete live files the
|
||||
// stream had not yet reached).
|
||||
func readTarGz(ctx context.Context, r io.Reader, dstDir string) (map[string]struct{}, error) {
|
||||
//
|
||||
// Files get their recorded mode and modification time as they are written. A
|
||||
// directory's are returned instead (restoreDirMeta): extracting and pruning
|
||||
// inside it would move its mtime again, and a read-only mode would stop the
|
||||
// extract from writing into it.
|
||||
func readTarGz(ctx context.Context, r io.Reader, dstDir string) (map[string]struct{}, []dirMeta, error) {
|
||||
gz, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: open gzip: %w", err)
|
||||
return nil, nil, fmt.Errorf("backup: open gzip: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
tr := tar.NewReader(gz)
|
||||
|
||||
keep := make(map[string]struct{})
|
||||
var dirs []dirMeta
|
||||
cleanDst := filepath.Clean(dstDir)
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
return nil, nil, ctx.Err()
|
||||
}
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return keep, nil
|
||||
return keep, dirs, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("backup: read tar: %w", err)
|
||||
return nil, nil, fmt.Errorf("backup: read tar: %w", err)
|
||||
}
|
||||
|
||||
// Guard against path traversal (zip-slip): the resolved target must stay
|
||||
// within dstDir.
|
||||
target := filepath.Join(cleanDst, filepath.FromSlash(hdr.Name))
|
||||
if target != cleanDst && !strings.HasPrefix(target, cleanDst+string(os.PathSeparator)) {
|
||||
return nil, fmt.Errorf("backup: archive entry escapes target: %q", hdr.Name)
|
||||
return nil, nil, fmt.Errorf("backup: archive entry escapes target: %q", hdr.Name)
|
||||
}
|
||||
|
||||
// Record this entry and its ancestors in the keep-set. Names are stored
|
||||
@@ -247,25 +480,39 @@ func readTarGz(ctx context.Context, r io.Reader, dstDir string) (map[string]stru
|
||||
// relative paths pruneToManifest derives from the on-disk walk.
|
||||
rememberKept(keep, hdr.Name)
|
||||
|
||||
mode := fs.FileMode(hdr.Mode).Perm()
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(target, 0o750); err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
if target != cleanDst {
|
||||
dirs = append(dirs, dirMeta{path: target, mode: mode, mtime: hdr.ModTime})
|
||||
}
|
||||
case tar.TypeReg:
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o640)
|
||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
if _, err := io.Copy(out, tr); err != nil {
|
||||
out.Close()
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
// Chmod rather than the create mode: the umask would narrow it, and
|
||||
// a file that already existed keeps its old mode through O_TRUNC.
|
||||
if err := os.Chmod(target, mode); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if recordedTime(hdr.ModTime) {
|
||||
if err := os.Chtimes(target, hdr.ModTime, hdr.ModTime); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
default:
|
||||
// Ignore entry types tarLocal never writes.
|
||||
@@ -273,6 +520,28 @@ func readTarGz(ctx context.Context, r io.Reader, dstDir string) (map[string]stru
|
||||
}
|
||||
}
|
||||
|
||||
// restoreDirMeta applies the directories' recorded modes and times, deepest
|
||||
// first, so setting a parent's time comes after every change inside it.
|
||||
func restoreDirMeta(dirs []dirMeta) error {
|
||||
sort.SliceStable(dirs, func(i, j int) bool { return len(dirs[i].path) > len(dirs[j].path) })
|
||||
for _, d := range dirs {
|
||||
if err := os.Chmod(d.path, d.mode); err != nil {
|
||||
return fmt.Errorf("backup: restore mode of %s: %w", d.path, err)
|
||||
}
|
||||
if recordedTime(d.mtime) {
|
||||
if err := os.Chtimes(d.path, d.mtime, d.mtime); err != nil {
|
||||
return fmt.Errorf("backup: restore time of %s: %w", d.path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// recordedTime reports whether an archive recorded a modification time. Archives
|
||||
// written before times were kept carry the Unix epoch, which is left alone
|
||||
// rather than stamped onto every file.
|
||||
func recordedTime(t time.Time) bool { return t.Unix() > 0 }
|
||||
|
||||
// rememberKept adds an archive entry name and every ancestor directory to keep,
|
||||
// normalized to a cleaned forward-slash path with no trailing slash. Adding
|
||||
// ancestors guards against archives that list a file without an explicit entry
|
||||
|
||||
@@ -4,9 +4,15 @@ import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/backup"
|
||||
)
|
||||
@@ -47,10 +53,11 @@ func TestTarLocalRoundTrip(t *testing.T) {
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
ref, size, err := archiver.Archive(ctx, "survival", "src-pvc")
|
||||
a, err := archiver.Archive(ctx, "survival", "src-pvc")
|
||||
if err != nil {
|
||||
t.Fatalf("Archive: %v", err)
|
||||
}
|
||||
ref, size := a.Ref, a.Size
|
||||
if size <= 0 {
|
||||
t.Errorf("archive size = %d, want > 0", size)
|
||||
}
|
||||
@@ -139,10 +146,11 @@ func TestTarLocalRestoreReplacesTarget(t *testing.T) {
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
ref, _, err := archiver.Archive(ctx, "survival", "src-pvc")
|
||||
a, err := archiver.Archive(ctx, "survival", "src-pvc")
|
||||
if err != nil {
|
||||
t.Fatalf("Archive: %v", err)
|
||||
}
|
||||
ref := a.Ref
|
||||
if err := archiver.Restore(ctx, ref, "dst-pvc"); err != nil {
|
||||
t.Fatalf("Restore: %v", err)
|
||||
}
|
||||
@@ -181,7 +189,7 @@ func TestTarLocalUnknownPVC(t *testing.T) {
|
||||
BackupRoot: t.TempDir(),
|
||||
Resolve: backup.StaticResolver(map[string]string{}),
|
||||
}
|
||||
if _, _, err := archiver.Archive(context.Background(), "x", "missing"); err == nil {
|
||||
if _, err := archiver.Archive(context.Background(), "x", "missing"); err == nil {
|
||||
t.Fatal("expected error for unknown pvc")
|
||||
}
|
||||
}
|
||||
@@ -222,3 +230,268 @@ func TestTarLocalRejectsZipSlip(t *testing.T) {
|
||||
t.Errorf("zip-slip wrote outside target: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTarLocalArchiveIsDurableAndChecksummed pins the write path: the archive
|
||||
// lands under its final name only, no .partial is left beside it, and the
|
||||
// SHA256 it reports is the one of the bytes on disk, which Verify reads back.
|
||||
func TestTarLocalArchiveIsDurableAndChecksummed(t *testing.T) {
|
||||
src, backupRoot := t.TempDir(), t.TempDir()
|
||||
writeTree(t, src, map[string]string{"level.dat": "seed", "region/r.0.0.mca": "chunk"})
|
||||
archiver := &backup.TarLocal{BackupRoot: backupRoot, Resolve: backup.StaticResolver(map[string]string{"src": src})}
|
||||
ctx := context.Background()
|
||||
|
||||
a, err := archiver.Archive(ctx, "survival", "src")
|
||||
if err != nil {
|
||||
t.Fatalf("Archive: %v", err)
|
||||
}
|
||||
ents, err := os.ReadDir(backupRoot)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ents) != 1 || filepath.Join(backupRoot, ents[0].Name()) != string(a.Ref) {
|
||||
t.Fatalf("backup root holds %v, want only %s", ents, a.Ref)
|
||||
}
|
||||
body, err := os.ReadFile(string(a.Ref))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if a.SHA256 != hex.EncodeToString(sum[:]) || a.Size != int64(len(body)) {
|
||||
t.Errorf("Archived = %+v, want sha256 %x size %d", a, sum, len(body))
|
||||
}
|
||||
got, err := archiver.Verify(ctx, a.Ref, a.SHA256)
|
||||
if err != nil || got != a.SHA256 {
|
||||
t.Errorf("Verify = %q, %v; want %q, nil", got, err, a.SHA256)
|
||||
}
|
||||
// With no digest recorded, Verify still reads it through and reports one.
|
||||
if got, err := archiver.Verify(ctx, a.Ref, ""); err != nil || got != a.SHA256 {
|
||||
t.Errorf("Verify without a digest = %q, %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTarLocalFailedArchiveLeavesNothing: an Archive that fails part way (here
|
||||
// a cancelled context) removes its .partial, so no leftover can pass for an
|
||||
// archive.
|
||||
func TestTarLocalFailedArchiveLeavesNothing(t *testing.T) {
|
||||
src, backupRoot := t.TempDir(), t.TempDir()
|
||||
writeTree(t, src, map[string]string{"level.dat": "seed"})
|
||||
archiver := &backup.TarLocal{BackupRoot: backupRoot, Resolve: backup.StaticResolver(map[string]string{"src": src})}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := archiver.Archive(ctx, "survival", "src"); err == nil {
|
||||
t.Fatal("Archive with a cancelled context succeeded")
|
||||
}
|
||||
if ents, _ := os.ReadDir(backupRoot); len(ents) != 0 {
|
||||
t.Errorf("failed Archive left %v behind", ents)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTarLocalKeepsModesAndTimes: a restore gives files and directories back
|
||||
// their permission bits (setuid, setgid and sticky dropped) and modification
|
||||
// times, and what the archive cannot hold is reported rather than dropped
|
||||
// silently.
|
||||
func TestTarLocalKeepsModesAndTimes(t *testing.T) {
|
||||
src, dst, backupRoot := t.TempDir(), t.TempDir(), t.TempDir()
|
||||
writeTree(t, src, map[string]string{
|
||||
"start.sh": "#!/bin/sh",
|
||||
"secret.properties": "rcon",
|
||||
"private/notes.txt": "n",
|
||||
"setuid-bin": "x",
|
||||
})
|
||||
mtime := time.Date(2025, 3, 4, 5, 6, 7, 0, time.UTC)
|
||||
modes := map[string]os.FileMode{
|
||||
"start.sh": 0o755,
|
||||
"secret.properties": 0o600,
|
||||
"private/notes.txt": 0o640,
|
||||
"setuid-bin": 0o755 | os.ModeSetuid,
|
||||
"private": 0o700,
|
||||
}
|
||||
for rel, m := range modes {
|
||||
p := filepath.Join(src, filepath.FromSlash(rel))
|
||||
if err := os.Chmod(p, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, rel := range []string{"start.sh", "secret.properties", "private/notes.txt", "setuid-bin", "private"} {
|
||||
if err := os.Chtimes(filepath.Join(src, filepath.FromSlash(rel)), mtime, mtime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.Symlink("/etc/passwd", filepath.Join(src, "link")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
archiver := &backup.TarLocal{BackupRoot: backupRoot, Resolve: backup.StaticResolver(map[string]string{"src": src, "dst": dst})}
|
||||
ctx := context.Background()
|
||||
a, err := archiver.Archive(ctx, "survival", "src")
|
||||
if err != nil {
|
||||
t.Fatalf("Archive: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(a.Skipped, []string{"link"}) {
|
||||
t.Errorf("Skipped = %v, want [link]", a.Skipped)
|
||||
}
|
||||
if err := archiver.Restore(ctx, a.Ref, "dst"); err != nil {
|
||||
t.Fatalf("Restore: %v", err)
|
||||
}
|
||||
for rel, m := range modes {
|
||||
info, err := os.Lstat(filepath.Join(dst, filepath.FromSlash(rel)))
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", rel, err)
|
||||
continue
|
||||
}
|
||||
if got, want := info.Mode().Perm(), m.Perm(); got != want || info.Mode()&os.ModeSetuid != 0 {
|
||||
t.Errorf("%s restored with mode %v, want %v", rel, info.Mode(), want)
|
||||
}
|
||||
if !info.ModTime().Equal(mtime) {
|
||||
t.Errorf("%s restored with mtime %v, want %v", rel, info.ModTime(), mtime)
|
||||
}
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(dst, "link")); !os.IsNotExist(err) {
|
||||
t.Errorf("symlink came back from the archive: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// damage rewrites the archive at ref with f applied to its bytes.
|
||||
func damage(t *testing.T, ref backup.ArchiveRef, f func([]byte) []byte) {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(string(ref))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(string(ref), f(b), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTarLocalVerifyFindsCorruption: a flipped byte, a truncated file, a digest
|
||||
// that no longer matches and a missing file all come back as ErrCorrupt.
|
||||
func TestTarLocalVerifyFindsCorruption(t *testing.T) {
|
||||
src := t.TempDir()
|
||||
writeTree(t, src, map[string]string{"level.dat": "seed", "region/r.0.0.mca": string(make([]byte, 64<<10))})
|
||||
ctx := context.Background()
|
||||
fresh := func(t *testing.T) (*backup.TarLocal, backup.Archived) {
|
||||
archiver := &backup.TarLocal{BackupRoot: t.TempDir(), Resolve: backup.StaticResolver(map[string]string{"src": src})}
|
||||
a, err := archiver.Archive(ctx, "survival", "src")
|
||||
if err != nil {
|
||||
t.Fatalf("Archive: %v", err)
|
||||
}
|
||||
return archiver, a
|
||||
}
|
||||
cases := map[string]struct {
|
||||
change func(*testing.T, backup.Archived)
|
||||
want func(backup.Archived) string
|
||||
}{
|
||||
"flipped byte": {
|
||||
change: func(t *testing.T, a backup.Archived) {
|
||||
damage(t, a.Ref, func(b []byte) []byte { b[len(b)/2] ^= 0xff; return b })
|
||||
},
|
||||
want: func(a backup.Archived) string { return "" },
|
||||
},
|
||||
"truncated": {
|
||||
change: func(t *testing.T, a backup.Archived) {
|
||||
damage(t, a.Ref, func(b []byte) []byte { return b[:len(b)-9] })
|
||||
},
|
||||
want: func(a backup.Archived) string { return "" },
|
||||
},
|
||||
"digest mismatch": {
|
||||
change: func(*testing.T, backup.Archived) {},
|
||||
want: func(backup.Archived) string { return hex.EncodeToString(make([]byte, 32)) },
|
||||
},
|
||||
"missing": {
|
||||
change: func(t *testing.T, a backup.Archived) {
|
||||
if err := os.Remove(string(a.Ref)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
want: func(a backup.Archived) string { return a.SHA256 },
|
||||
},
|
||||
}
|
||||
for name, tc := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
archiver, a := fresh(t)
|
||||
tc.change(t, a)
|
||||
if _, err := archiver.Verify(ctx, a.Ref, tc.want(a)); !errors.Is(err, backup.ErrCorrupt) {
|
||||
t.Errorf("Verify = %v, want ErrCorrupt", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestTarLocalRestoreRefusesCorruptArchive: the archive is read back before
|
||||
// anything is extracted, so a corrupt one leaves the world exactly as it was.
|
||||
func TestTarLocalRestoreRefusesCorruptArchive(t *testing.T) {
|
||||
src, dst := t.TempDir(), t.TempDir()
|
||||
writeTree(t, src, map[string]string{"level.dat": "archived", "region/r.0.0.mca": string(make([]byte, 64<<10))})
|
||||
writeTree(t, dst, map[string]string{"level.dat": "live"})
|
||||
archiver := &backup.TarLocal{BackupRoot: t.TempDir(), Resolve: backup.StaticResolver(map[string]string{"src": src, "dst": dst})}
|
||||
ctx := context.Background()
|
||||
a, err := archiver.Archive(ctx, "survival", "src")
|
||||
if err != nil {
|
||||
t.Fatalf("Archive: %v", err)
|
||||
}
|
||||
damage(t, a.Ref, func(b []byte) []byte { return b[:len(b)-9] })
|
||||
if err := archiver.Restore(ctx, a.Ref, "dst"); !errors.Is(err, backup.ErrCorrupt) {
|
||||
t.Fatalf("Restore = %v, want ErrCorrupt", err)
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(dst, "level.dat"))
|
||||
if err != nil || string(got) != "live" {
|
||||
t.Errorf("world changed by a refused restore: %q, %v", got, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dst, "region")); !os.IsNotExist(err) {
|
||||
t.Errorf("refused restore extracted entries: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTarLocalSweep removes old .partial files and orphaned archives past the
|
||||
// orphan cutoff, reports younger orphans, and leaves everything else: fresh
|
||||
// leftovers (an Archive may still be writing, or its record not yet inserted),
|
||||
// claimed archives, and files it did not write.
|
||||
func TestTarLocalSweep(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
now := time.Now()
|
||||
ancient, old, recent := now.Add(-100*24*time.Hour), now.Add(-7*time.Hour), now.Add(-time.Minute)
|
||||
files := map[string]time.Time{
|
||||
".survival-100.tar.gz.partial": old,
|
||||
".survival-200.tar.gz.partial": recent,
|
||||
"survival-300.tar.gz": ancient, // orphan past the cutoff
|
||||
"survival-310.tar.gz": old, // orphan, kept and reported
|
||||
"survival-400.tar.gz": ancient, // claimed
|
||||
"survival-500.tar.gz": recent, // may be about to be claimed
|
||||
"notes.txt": ancient,
|
||||
"felis.dump": ancient,
|
||||
}
|
||||
for name, at := range files {
|
||||
p := filepath.Join(root, name)
|
||||
if err := os.WriteFile(p, []byte("xyz"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chtimes(p, at, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
archiver := &backup.TarLocal{BackupRoot: root}
|
||||
live := func(ref backup.ArchiveRef) bool {
|
||||
return ref == backup.ArchiveRef(filepath.Join(root, "survival-400.tar.gz"))
|
||||
}
|
||||
got, err := archiver.Sweep(context.Background(), live, now.Add(-6*time.Hour), now.Add(-90*24*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep: %v", err)
|
||||
}
|
||||
sort.Strings(got.Removed)
|
||||
want := backup.Swept{
|
||||
Removed: []string{filepath.Join(root, ".survival-100.tar.gz.partial"), filepath.Join(root, "survival-300.tar.gz")},
|
||||
Orphans: []string{filepath.Join(root, "survival-310.tar.gz")},
|
||||
OrphanBytes: 3,
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("Sweep = %+v, want %+v", got, want)
|
||||
}
|
||||
ents, _ := os.ReadDir(root)
|
||||
if len(ents) != len(files)-2 {
|
||||
t.Errorf("%d files left, want %d", len(ents), len(files)-2)
|
||||
}
|
||||
// A store that does not exist yet has nothing to sweep.
|
||||
if _, err := (&backup.TarLocal{BackupRoot: filepath.Join(root, "absent")}).Sweep(context.Background(), live, now, now); err != nil {
|
||||
t.Errorf("Sweep of a missing root: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user