Felis/internal/backup/tarlocal.go

package backup

import (
	"archive/tar"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"syscall"
	"time"
)

// TarLocal is the zero-storageClass-requirement backend (spec §19): it mounts
// the source PVC, tars+gzips it, and writes the archive into the backup PVC.
// It runs on any StorageClass, including hostPath-style local-path, where the
// snapshot backends cannot.
type TarLocal struct {
	// BackupRoot is the directory (backup PVC mount) archives are written into.
	BackupRoot string
	// Resolve maps a PVC name to its mounted filesystem path.
	Resolve PVCResolver
	// Now is injectable for deterministic archive names in tests.
	Now func() time.Time
}

func (t *TarLocal) now() time.Time {
	if t.Now != nil {
		return t.Now()
	}
	return time.Now()
}

// partialSuffix marks an archive still being written. Its file is the final
// name hidden behind a leading dot, so a listing of the store shows finished
// archives only and a leftover still says whose it was.
const partialSuffix = ".partial"

// archiveName matches the finished archives Archive names (<server>-<unix
// nanoseconds>.tar.gz); partialName matches their unfinished form. Sweep touches
// nothing else in the store.
var (
	archiveName = regexp.MustCompile(`^[^.].*-[0-9]+\.tar\.gz$`)
	partialName = regexp.MustCompile(`^\..*-[0-9]+\.tar\.gz\.partial$`)
)

// Archive tars+gzips the world on pvc into BackupRoot and returns the archive
// path as the opaque ref, with its size and SHA256.
//
// The archive is written under its hidden .partial name, flushed to disk, read
// back in full, and only then renamed to its final name and the rename flushed
// too. A Job killed at its deadline or a node that loses power mid-write leaves
// a .partial for Sweep, never a truncated file under a name that looks finished,
// and the world is deleted only after an archive that has already been read
// back whole.
func (t *TarLocal) Archive(ctx context.Context, server, pvc string) (Archived, error) {
	srcDir, err := t.Resolve(pvc)
	if err != nil {
		return Archived{}, err
	}
	if err := os.MkdirAll(t.BackupRoot, 0o750); err != nil {
		return Archived{}, fmt.Errorf("backup: mkdir backup root: %w", err)
	}
	name := fmt.Sprintf("%s-%d.tar.gz", server, t.now().UTC().UnixNano())
	dest := filepath.Join(t.BackupRoot, name)
	tmp := filepath.Join(t.BackupRoot, "."+name+partialSuffix)

	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
	if err != nil {
		return Archived{}, fmt.Errorf("backup: create archive: %w", err)
	}
	h := sha256.New()
	st, err := writeTarGz(ctx, io.MultiWriter(f, h), srcDir)
	if err == nil {
		if err = f.Sync(); err != nil {
			err = fmt.Errorf("backup: sync archive: %w", err)
		}
	}
	if cerr := f.Close(); err == nil && cerr != nil {
		err = fmt.Errorf("backup: close archive: %w", cerr)
	}
	if err != nil {
		os.Remove(tmp)
		return Archived{}, err
	}
	sum := hex.EncodeToString(h.Sum(nil))

	// Read back what landed: the gzip CRC, every tar header and entry, and the
	// entry count must all come out as written.
	entries, _, err := verifyArchive(ctx, tmp, sum)
	if err == nil && entries != st.entries {
		err = fmt.Errorf("%w: %s: %d entries read back, %d written", ErrCorrupt, tmp, entries, st.entries)
	}
	if err != nil {
		os.Remove(tmp)
		return Archived{}, err
	}
	if err := os.Rename(tmp, dest); err != nil {
		os.Remove(tmp)
		return Archived{}, fmt.Errorf("backup: name archive: %w", err)
	}
	if err := syncDir(t.BackupRoot); err != nil {
		os.Remove(dest)
		return Archived{}, fmt.Errorf("backup: sync backup root: %w", err)
	}
	info, err := os.Stat(dest)
	if err != nil {
		return Archived{}, fmt.Errorf("backup: stat archive: %w", err)
	}
	return Archived{Ref: ArchiveRef(dest), Size: info.Size(), SHA256: sum, Skipped: st.skipped}, nil
}

// Verify reads the archive at ref back end to end (see verifyArchive).
func (t *TarLocal) Verify(ctx context.Context, ref ArchiveRef, want string) (string, error) {
	_, sum, err := verifyArchive(ctx, string(ref), want)
	return sum, err
}

// verifyArchive reads the archive at p through gzip and tar to the last byte and
// returns its entry count and SHA256. Anything that does not read back — a
// missing file, a torn gzip stream, a CRC or length mismatch, a bad tar header,
// or a digest other than want (when want is not "") — is ErrCorrupt. Only an
// error opening a file that is there, or the context, is not.
func verifyArchive(ctx context.Context, p, want string) (int, string, error) {
	f, err := os.Open(p)
	if errors.Is(err, fs.ErrNotExist) {
		return 0, "", fmt.Errorf("%w: %s is missing", ErrCorrupt, p)
	}
	if err != nil {
		return 0, "", fmt.Errorf("backup: open archive: %w", err)
	}
	defer f.Close()
	h := sha256.New()
	raw := io.TeeReader(f, h)
	corrupt := func(err error) (int, string, error) {
		if ctx.Err() != nil {
			return 0, "", ctx.Err()
		}
		return 0, "", fmt.Errorf("%w: %s: %v", ErrCorrupt, p, err)
	}

	gz, err := gzip.NewReader(raw)
	if err != nil {
		return corrupt(err)
	}
	tr := tar.NewReader(gz)
	entries := 0
	for {
		if ctx.Err() != nil {
			return 0, "", ctx.Err()
		}
		_, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return corrupt(err)
		}
		if _, err := io.Copy(io.Discard, tr); err != nil {
			return corrupt(err)
		}
		entries++
	}
	// The tar end marker is not the end of the gzip member: reading on to EOF is
	// what checks the CRC and length in the gzip trailer.
	if _, err := io.Copy(io.Discard, gz); err != nil {
		return corrupt(err)
	}
	if err := gz.Close(); err != nil {
		return corrupt(err)
	}
	if _, err := io.Copy(io.Discard, raw); err != nil {
		return corrupt(err)
	}
	sum := hex.EncodeToString(h.Sum(nil))
	if want != "" && sum != want {
		return 0, sum, fmt.Errorf("%w: %s: sha256 %s, recorded %s", ErrCorrupt, p, sum, want)
	}
	return entries, sum, nil
}

// Sweep removes the leftovers of archives that never finished (see Sweeper). It
// only looks at the top level of BackupRoot and only at names Archive writes;
// partialBefore must leave room for the longest Archive, and orphanBefore for
// the insert of the record that follows it at the very least.
func (t *TarLocal) Sweep(ctx context.Context, live func(ArchiveRef) bool, partialBefore, orphanBefore time.Time) (Swept, error) {
	var out Swept
	ents, err := os.ReadDir(t.BackupRoot)
	if errors.Is(err, fs.ErrNotExist) {
		return out, nil
	}
	if err != nil {
		return out, fmt.Errorf("backup: list backup root: %w", err)
	}
	var errs []error
	for _, e := range ents {
		if ctx.Err() != nil {
			return out, ctx.Err()
		}
		name := e.Name()
		if !e.Type().IsRegular() {
			continue
		}
		p := filepath.Join(t.BackupRoot, name)
		cutoff := partialBefore
		switch {
		case partialName.MatchString(name):
		case archiveName.MatchString(name):
			if live(ArchiveRef(p)) {
				continue
			}
			cutoff = orphanBefore
		default:
			continue
		}
		info, err := e.Info()
		if err != nil || !info.ModTime().Before(partialBefore) {
			continue
		}
		if !info.ModTime().Before(cutoff) {
			out.Orphans = append(out.Orphans, p)
			out.OrphanBytes += info.Size()
			continue
		}
		if err := os.Remove(p); err != nil && !errors.Is(err, fs.ErrNotExist) {
			errs = append(errs, err)
			continue
		}
		out.Removed = append(out.Removed, p)
	}
	return out, errors.Join(errs...)
}

// syncDir flushes a change to a directory's entries (a rename) to disk. A
// filesystem that cannot sync a directory has no stronger promise to give.
func syncDir(dir string) error {
	d, err := os.Open(dir)
	if err != nil {
		return err
	}
	defer d.Close()
	if err := d.Sync(); err != nil && !errors.Is(err, syscall.EINVAL) && !errors.Is(err, syscall.ENOTSUP) {
		return err
	}
	return nil
}

// Restore extracts the archive at ref into the world mount for targetPVC,
// replacing the target's contents so the world equals the archive (spec §466
// "restore PVC": a rollback must not leave stale files the backup lacks — e.g. a
// griefer's chunks). It extracts over the target, then removes any pre-existing
// entry the archive did not contain.
//
// The archive is read back in full before anything is extracted, so a corrupt
// or truncated archive fails with the world untouched. The prune runs only after
// a fully successful extract: an extract that still fails (the volume fills up)
// leaves the target as a recoverable partial overlay rather than a destroyed
// world. The archive is retained on restore, so such a failure is recoverable by
// re-running the Job.
//
// Files and directories get back the permission bits and modification times the
// archive recorded. Ownership is left to the server: every start re-owns the
// world volume to the game uid (felis init-volume).
//
// A top-level lost+found is never a prune target. It is a filesystem artifact
// (root-owned, mode 0700) that the non-root restore Pod cannot delete anyway,
// and writeTarGz includes it in the archive, so it is preserved on both axes.
func (t *TarLocal) Restore(ctx context.Context, ref ArchiveRef, targetPVC string) error {
	dstDir, err := t.Resolve(targetPVC)
	if err != nil {
		return err
	}
	if err := os.MkdirAll(dstDir, 0o750); err != nil {
		return fmt.Errorf("backup: mkdir restore target: %w", err)
	}
	if _, _, err := verifyArchive(ctx, string(ref), ""); err != nil {
		return err
	}
	f, err := os.Open(string(ref))
	if err != nil {
		return fmt.Errorf("backup: open archive: %w", err)
	}
	defer f.Close()

	keep, dirs, err := readTarGz(ctx, f, dstDir)
	if err != nil {
		return err
	}
	if err := pruneToManifest(dstDir, keep); err != nil {
		return err
	}
	return restoreDirMeta(dirs)
}

// pruneToManifest removes every entry under dstDir whose archive-relative path
// is absent from keep, giving Restore replace semantics. keep holds cleaned,
// forward-slash relative paths (no trailing slash) for every archive entry plus
// all of their ancestor directories, so a kept file's parent dirs are never
// removed. A top-level lost+found is always kept. dstDir (the mount root) is
// never removed.
func pruneToManifest(dstDir string, keep map[string]struct{}) error {
	cleanDst := filepath.Clean(dstDir)
	return filepath.WalkDir(cleanDst, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if p == cleanDst {
			return nil // never remove the mount root itself
		}
		relNative, err := filepath.Rel(cleanDst, p)
		if err != nil {
			return err
		}
		rel := filepath.ToSlash(relNative)
		if rel == "lost+found" {
			if d.IsDir() {
				return filepath.SkipDir // filesystem artifact: keep and don't descend
			}
			return nil
		}
		if _, ok := keep[rel]; ok {
			return nil // the archive contained this path: keep it
		}
		// Stale: present in the target but absent from the archive.
		if err := os.RemoveAll(p); err != nil {
			return fmt.Errorf("backup: prune stale entry %q: %w", rel, err)
		}
		if d.IsDir() {
			return filepath.SkipDir // already removed; don't descend into it
		}
		return nil
	})
}

// Delete removes the tar archive at ref.
func (t *TarLocal) Delete(_ context.Context, ref ArchiveRef) error {
	if err := os.Remove(string(ref)); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("backup: delete archive: %w", err)
	}
	return nil
}

// tarStats is what writeTarGz put in the archive and what it left out.
type tarStats struct {
	entries int
	skipped []string
}

// writeTarGz archives srcDir (not the root entry itself) as gzip+tar. Each entry
// keeps its permission bits, without setuid, setgid and sticky, and its
// modification time. Entries other than regular files and directories are left
// out and listed in the stats.
func writeTarGz(ctx context.Context, w io.Writer, srcDir string) (tarStats, error) {
	var st tarStats
	gz := gzip.NewWriter(w)
	tw := tar.NewWriter(gz)

	root := filepath.Clean(srcDir)
	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if ctx.Err() != nil {
			return ctx.Err()
		}
		rel, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if rel == "." {
			return nil // don't archive the root entry itself
		}
		// Normalize to forward slashes so archives are portable.
		name := filepath.ToSlash(rel)

		mode := int64(info.Mode().Perm())
		switch {
		case info.IsDir():
			hdr := &tar.Header{Name: name + "/", Mode: mode, ModTime: info.ModTime(), Typeflag: tar.TypeDir}
			if err := tw.WriteHeader(hdr); err != nil {
				return err
			}
			st.entries++
			return nil
		case info.Mode().IsRegular():
			hdr := &tar.Header{Name: name, Mode: mode, ModTime: info.ModTime(), Size: info.Size(), Typeflag: tar.TypeReg}
			if err := tw.WriteHeader(hdr); err != nil {
				return err
			}
			src, err := os.Open(path)
			if err != nil {
				return err
			}
			defer src.Close()
			if _, err := io.Copy(tw, src); err != nil {
				return err
			}
			st.entries++
			return nil
		default:
			// A symlink could point anywhere on the node, and a device or socket
			// has no bytes to keep: a world is plain files. What is left out is
			// reported, so a backup never drops something silently.
			st.skipped = append(st.skipped, name)
			return nil
		}
	})
	if err != nil {
		return st, fmt.Errorf("backup: tar walk: %w", err)
	}
	if err := tw.Close(); err != nil {
		return st, fmt.Errorf("backup: close tar: %w", err)
	}
	if err := gz.Close(); err != nil {
		return st, fmt.Errorf("backup: close gzip: %w", err)
	}
	return st, nil
}

// dirMeta is a directory's recorded permission bits and modification time,
// applied once nothing more is written into it.
type dirMeta struct {
	path  string
	mode  fs.FileMode
	mtime time.Time
}

// readTarGz extracts the gzip+tar stream into dstDir and returns the keep-set:
// the cleaned, forward-slash relative path of every entry the archive contained
// plus all of their ancestor directories. The caller uses it to prune stale
// target files for replace semantics. On any error the keep-set is incomplete
// and must not be used to prune (a partial manifest would delete live files the
// stream had not yet reached).
//
// Files get their recorded mode and modification time as they are written. A
// directory's are returned instead (restoreDirMeta): extracting and pruning
// inside it would move its mtime again, and a read-only mode would stop the
// extract from writing into it.
func readTarGz(ctx context.Context, r io.Reader, dstDir string) (map[string]struct{}, []dirMeta, error) {
	gz, err := gzip.NewReader(r)
	if err != nil {
		return nil, nil, fmt.Errorf("backup: open gzip: %w", err)
	}
	defer gz.Close()
	tr := tar.NewReader(gz)

	keep := make(map[string]struct{})
	var dirs []dirMeta
	cleanDst := filepath.Clean(dstDir)
	for {
		if ctx.Err() != nil {
			return nil, nil, ctx.Err()
		}
		hdr, err := tr.Next()
		if err == io.EOF {
			return keep, dirs, nil
		}
		if err != nil {
			return nil, nil, fmt.Errorf("backup: read tar: %w", err)
		}

		// Guard against path traversal (zip-slip): the resolved target must stay
		// within dstDir.
		target := filepath.Join(cleanDst, filepath.FromSlash(hdr.Name))
		if target != cleanDst && !strings.HasPrefix(target, cleanDst+string(os.PathSeparator)) {
			return nil, nil, fmt.Errorf("backup: archive entry escapes target: %q", hdr.Name)
		}

		// Record this entry and its ancestors in the keep-set. Names are stored
		// in archive form (forward slash, no trailing slash) to match the
		// relative paths pruneToManifest derives from the on-disk walk.
		rememberKept(keep, hdr.Name)

		mode := fs.FileMode(hdr.Mode).Perm()
		switch hdr.Typeflag {
		case tar.TypeDir:
			if err := os.MkdirAll(target, 0o750); err != nil {
				return nil, nil, err
			}
			if target != cleanDst {
				dirs = append(dirs, dirMeta{path: target, mode: mode, mtime: hdr.ModTime})
			}
		case tar.TypeReg:
			if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil {
				return nil, nil, err
			}
			out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
			if err != nil {
				return nil, nil, err
			}
			if _, err := io.Copy(out, tr); err != nil {
				out.Close()
				return nil, nil, err
			}
			if err := out.Close(); err != nil {
				return nil, nil, err
			}
			// Chmod rather than the create mode: the umask would narrow it, and
			// a file that already existed keeps its old mode through O_TRUNC.
			if err := os.Chmod(target, mode); err != nil {
				return nil, nil, err
			}
			if recordedTime(hdr.ModTime) {
				if err := os.Chtimes(target, hdr.ModTime, hdr.ModTime); err != nil {
					return nil, nil, err
				}
			}
		default:
			// Ignore entry types tarLocal never writes.
		}
	}
}

// restoreDirMeta applies the directories' recorded modes and times, deepest
// first, so setting a parent's time comes after every change inside it.
func restoreDirMeta(dirs []dirMeta) error {
	sort.SliceStable(dirs, func(i, j int) bool { return len(dirs[i].path) > len(dirs[j].path) })
	for _, d := range dirs {
		if err := os.Chmod(d.path, d.mode); err != nil {
			return fmt.Errorf("backup: restore mode of %s: %w", d.path, err)
		}
		if recordedTime(d.mtime) {
			if err := os.Chtimes(d.path, d.mtime, d.mtime); err != nil {
				return fmt.Errorf("backup: restore time of %s: %w", d.path, err)
			}
		}
	}
	return nil
}

// recordedTime reports whether an archive recorded a modification time. Archives
// written before times were kept carry the Unix epoch, which is left alone
// rather than stamped onto every file.
func recordedTime(t time.Time) bool { return t.Unix() > 0 }

// rememberKept adds an archive entry name and every ancestor directory to keep,
// normalized to a cleaned forward-slash path with no trailing slash. Adding
// ancestors guards against archives that list a file without an explicit entry
// for its parent dir: the dir must still survive the prune.
func rememberKept(keep map[string]struct{}, name string) {
	rel := path.Clean(strings.TrimSuffix(name, "/"))
	for rel != "." && rel != "/" && rel != "" {
		keep[rel] = struct{}{}
		rel = path.Dir(rel)
	}
}