127 lines
4.0 KiB
Go
127 lines
4.0 KiB
Go
package backup
|
|
|
|
import (
|
|
"archive/tar"
|
|
"compress/gzip"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
)
|
|
|
|
// Filter decides, for one regular file on its way out of a world into an
|
|
// export, whether it is left out (withhold) or has its bytes replaced (rewrite
|
|
// non-nil). name is the archive path; info is the file's, or nil when the file
|
|
// is an entry of a stored archive, which has only names. A backup is not
|
|
// filtered: it stays on the platform, and a restore must bring the world back
|
|
// whole.
|
|
type Filter func(name string, info fs.FileInfo) (withhold bool, rewrite func([]byte) []byte)
|
|
|
|
// maxRewrite bounds a file a Filter rewrites, which is read whole into memory.
|
|
// The files it rewrites are small configs; one larger than this is withheld
|
|
// rather than sent unrewritten.
|
|
const maxRewrite = 1 << 20
|
|
|
|
// WriteTarGz archives srcDir into w laid out exactly as Archive lays out a
|
|
// backup, so an exported world restores like any other archive. It returns the
|
|
// entries a tar cannot hold and the files filter withheld. The world export Job
|
|
// streams it straight into its upload.
|
|
func WriteTarGz(ctx context.Context, w io.Writer, srcDir string, filter Filter) (skipped, withheld []string, err error) {
|
|
st, err := writeTarGz(ctx, w, srcDir, filter)
|
|
return st.skipped, st.withheld, err
|
|
}
|
|
|
|
// FilterTarGz copies the gzip+tar archive read from r into w, passing every
|
|
// regular file through filter by name. It is how a stored backup leaves the
|
|
// platform: the archive is the world as it was, secrets included, so it is
|
|
// re-written on the way out rather than handed over as stored.
|
|
//
|
|
// r is read to its very end, past the tar trailer, before w's archive is
|
|
// closed. A reader that checks a digest when it reaches EOF therefore fails the
|
|
// copy while w still lacks the end of its archive, and a receiver never holds a
|
|
// complete-looking copy of a corrupt backup.
|
|
func FilterTarGz(ctx context.Context, w io.Writer, r io.Reader, filter Filter) (withheld []string, err error) {
|
|
zr, err := gzip.NewReader(r)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("backup: open archive: %w", err)
|
|
}
|
|
tr := tar.NewReader(zr)
|
|
zw := gzip.NewWriter(w)
|
|
tw := tar.NewWriter(zw)
|
|
for {
|
|
if err := ctx.Err(); err != nil {
|
|
return withheld, err
|
|
}
|
|
hdr, err := tr.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return withheld, fmt.Errorf("backup: read archive: %w", err)
|
|
}
|
|
if !hdr.FileInfo().Mode().IsRegular() {
|
|
if err := tw.WriteHeader(hdr); err != nil {
|
|
return withheld, err
|
|
}
|
|
continue
|
|
}
|
|
withhold, rewrite := filter(hdr.Name, nil)
|
|
if withhold {
|
|
withheld = append(withheld, hdr.Name)
|
|
continue
|
|
}
|
|
if rewrite == nil {
|
|
if err := tw.WriteHeader(hdr); err != nil {
|
|
return withheld, err
|
|
}
|
|
if _, err := io.Copy(tw, tr); err != nil {
|
|
return withheld, fmt.Errorf("backup: copy %s: %w", hdr.Name, err)
|
|
}
|
|
continue
|
|
}
|
|
content, err := io.ReadAll(io.LimitReader(tr, maxRewrite+1))
|
|
if err != nil {
|
|
return withheld, fmt.Errorf("backup: read %s: %w", hdr.Name, err)
|
|
}
|
|
if len(content) > maxRewrite {
|
|
withheld = append(withheld, hdr.Name)
|
|
continue
|
|
}
|
|
content = rewrite(content)
|
|
hdr.Size = int64(len(content))
|
|
if err := tw.WriteHeader(hdr); err != nil {
|
|
return withheld, err
|
|
}
|
|
if _, err := tw.Write(content); err != nil {
|
|
return withheld, err
|
|
}
|
|
}
|
|
// Through the gzip trailer to r's EOF (see above).
|
|
if _, err := io.Copy(io.Discard, zr); err != nil {
|
|
return withheld, fmt.Errorf("backup: read archive: %w", err)
|
|
}
|
|
if err := tw.Close(); err != nil {
|
|
return withheld, fmt.Errorf("backup: close tar: %w", err)
|
|
}
|
|
if err := zw.Close(); err != nil {
|
|
return withheld, fmt.Errorf("backup: close gzip: %w", err)
|
|
}
|
|
return withheld, nil
|
|
}
|
|
|
|
// readRewritable reads a file a Filter rewrites, reporting false when it is
|
|
// larger than maxRewrite.
|
|
func readRewritable(path string) ([]byte, bool, error) {
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
defer f.Close()
|
|
b, err := io.ReadAll(io.LimitReader(f, maxRewrite+1))
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
return b, len(b) <= maxRewrite, nil
|
|
}
|