287 lines
11 KiB
Go
287 lines
11 KiB
Go
package offsite
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"felis.lolicon.best/internal/dbbackup"
|
|
)
|
|
|
|
// snapshotter stands in for `felis db backup -label offsite`: each call writes
|
|
// a bundle stamped with the syncer's clock into its DBDir.
|
|
type snapshotter struct {
|
|
t *testing.T
|
|
s *Syncer
|
|
calls int
|
|
err error
|
|
}
|
|
|
|
func (sn *snapshotter) take(context.Context) error {
|
|
sn.calls++
|
|
if sn.err != nil {
|
|
return sn.err
|
|
}
|
|
name := dbbackup.BundleName(sn.s.now(), dbbackup.LabelOffsite)
|
|
data := dbBundle(sn.t, sn.s.now(), &dbbackup.Counts{Users: 3, Servers: 2}, 64)
|
|
return os.WriteFile(filepath.Join(sn.s.DBDir, name), data, 0o600)
|
|
}
|
|
|
|
func withSnapshot(t *testing.T, s *Syncer) *snapshotter {
|
|
sn := &snapshotter{t: t, s: s}
|
|
s.Snapshot = sn.take
|
|
return sn
|
|
}
|
|
|
|
func dbKeys(b *memBucket) []string {
|
|
var keys []string
|
|
for k := range b.objs {
|
|
if strings.HasPrefix(k, dbDir) {
|
|
keys = append(keys, strings.TrimSuffix(strings.TrimPrefix(k, dbDir), objExt))
|
|
}
|
|
}
|
|
sort.Strings(keys)
|
|
return keys
|
|
}
|
|
|
|
// TestSyncSnapshotsAfterCopyingWorlds: a pass that copies an archive leaves a
|
|
// bundle in the bucket taken after the copy, which a restore takes as the
|
|
// latest and whose rows list the archive; the pass counts the bucket's bundles
|
|
// once, with that one newest. A pass that copies nothing takes none, and the
|
|
// next daily bundle replaces it.
|
|
func TestSyncSnapshotsAfterCopyingWorlds(t *testing.T) {
|
|
cat := &fakeCatalog{rows: []*row{
|
|
{WorldBackup: WorldBackup{ID: "b1", Server: "alpha", Ref: "/a/alpha-1.tar.gz"}, status: "present"},
|
|
}}
|
|
s, b := newSyncer(t, cat)
|
|
sn := withSnapshot(t, s)
|
|
writeFile(t, s.ArchiveDir, "alpha-1.tar.gz", 100)
|
|
daily := "felis-db-20260924T030000Z-daily.tar"
|
|
writeFile(t, s.DBDir, daily, 50)
|
|
|
|
res, err := s.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("run: %v (%+v)", err, res)
|
|
}
|
|
snap := dbbackup.BundleName(now, dbbackup.LabelOffsite)
|
|
if sn.calls != 1 {
|
|
t.Fatalf("snapshots taken = %d, want 1 after copying an archive", sn.calls)
|
|
}
|
|
if got := dbKeys(b); !slices.Equal(got, []string{daily, snap}) {
|
|
t.Fatalf("bucket bundles = %v, want the daily and the snapshot", got)
|
|
}
|
|
if res.NewestDB != snap || res.RemoteDB != 2 || res.DBUploaded != 2 || res.WorldsUploaded != 1 {
|
|
t.Fatalf("result = %+v, want the snapshot newest of 2 bundles, 2 bundles and 1 archive copied", res)
|
|
}
|
|
name, _, err := ChooseDB(context.Background(), b, s.Key)
|
|
if err != nil || name != snap {
|
|
t.Fatalf("fetch-db latest = %q, %v; want the snapshot", name, err)
|
|
}
|
|
|
|
if res, err := s.Run(context.Background()); err != nil || sn.calls != 1 || res.NewestDB != snap || res.RemoteDB != 2 {
|
|
t.Fatalf("a pass that copied nothing: snapshots %d, result %+v, err %v; want no new snapshot", sn.calls, res, err)
|
|
}
|
|
|
|
next := now.Add(15 * time.Hour)
|
|
s.Now = func() time.Time { return next }
|
|
nextDaily := dbbackup.BundleName(next.Add(-time.Minute), dbbackup.LabelDaily)
|
|
writeFile(t, s.DBDir, nextDaily, 50)
|
|
res, err = s.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := dbKeys(b); !slices.Equal(got, []string{daily, nextDaily}) || sn.calls != 1 || res.DBPruned != 1 {
|
|
t.Fatalf("after the next daily: bucket %v, snapshots %d, result %+v; want the snapshot pruned, both dailies kept", got, sn.calls, res)
|
|
}
|
|
}
|
|
|
|
// TestSyncSnapshotOnlyWhenACopyIsNewer: no snapshot while the newest bundle in
|
|
// the bucket was taken after the last copy, in the same second included (bundle
|
|
// names have one-second resolution); one when a copy is newer, or when the
|
|
// bucket has no bundle at all; none on an install that never copied an archive.
|
|
func TestSyncSnapshotOnlyWhenACopyIsNewer(t *testing.T) {
|
|
stamped := func(at time.Time) string { return dbbackup.BundleName(at, dbbackup.LabelDaily) }
|
|
cases := []struct {
|
|
name string
|
|
copied time.Time
|
|
bundle string // in the bucket before the pass; "" for none
|
|
want int
|
|
wantNew string
|
|
}{
|
|
{name: "never copied", want: 0},
|
|
{name: "bundle after the copy", copied: now.Add(-time.Hour), bundle: stamped(now.Add(-time.Minute)), want: 0},
|
|
{name: "same second", copied: now.Add(400 * time.Millisecond), bundle: stamped(now), want: 0},
|
|
{name: "copy after the bundle", copied: now.Add(-time.Hour), bundle: stamped(now.Add(-2 * time.Hour)), want: 1},
|
|
{name: "no bundle", copied: now.Add(-time.Hour), want: 1},
|
|
}
|
|
for _, c := range cases {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
cat := &fakeCatalog{rows: []*row{
|
|
{WorldBackup: WorldBackup{ID: "b1", Ref: "/a/alpha-1.tar.gz"}, status: "present", offsite: c.copied},
|
|
}}
|
|
if c.copied.IsZero() {
|
|
cat.rows[0].status = "deleted"
|
|
}
|
|
s, b := newSyncer(t, cat)
|
|
sn := withSnapshot(t, s)
|
|
if c.bundle != "" {
|
|
b.objs[DBKey(c.bundle)] = seal(t, []byte("bundle"), s.Key)
|
|
}
|
|
res, err := s.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("run: %v", err)
|
|
}
|
|
if sn.calls != c.want {
|
|
t.Fatalf("snapshots taken = %d, want %d (newest %s)", sn.calls, c.want, res.NewestDB)
|
|
}
|
|
if c.want == 1 && res.NewestDB != dbbackup.BundleName(now, dbbackup.LabelOffsite) {
|
|
t.Fatalf("newest bundle = %s, want the snapshot", res.NewestDB)
|
|
}
|
|
})
|
|
}
|
|
|
|
// A pass that copies no bundles (-db-dir "") takes none either.
|
|
cat := &fakeCatalog{rows: []*row{
|
|
{WorldBackup: WorldBackup{ID: "b1", Ref: "/a/alpha-1.tar.gz"}, status: "present", offsite: now.Add(-time.Hour)},
|
|
}}
|
|
s, _ := newSyncer(t, cat)
|
|
s.DBDir = ""
|
|
calls := 0
|
|
s.Snapshot = func(context.Context) error { calls++; return errors.New("no bundle directory") }
|
|
if _, err := s.Run(context.Background()); err != nil || calls != 0 {
|
|
t.Fatalf("no bundle directory: snapshots %d, err %v; want none", calls, err)
|
|
}
|
|
}
|
|
|
|
// TestSyncSnapshotFailureIsReported: a snapshot that fails fails the pass, which
|
|
// the watchdog reports, and the bundles already there still count.
|
|
func TestSyncSnapshotFailureIsReported(t *testing.T) {
|
|
cat := &fakeCatalog{rows: []*row{
|
|
{WorldBackup: WorldBackup{ID: "b1", Server: "alpha", Ref: "/a/alpha-1.tar.gz"}, status: "present"},
|
|
}}
|
|
s, _ := newSyncer(t, cat)
|
|
sn := withSnapshot(t, s)
|
|
sn.err = errors.New("pg_dump: connection refused")
|
|
writeFile(t, s.ArchiveDir, "alpha-1.tar.gz", 100)
|
|
daily := "felis-db-20260924T030000Z-daily.tar"
|
|
writeFile(t, s.DBDir, daily, 50)
|
|
|
|
res, err := s.Run(context.Background())
|
|
if err == nil || !strings.Contains(err.Error(), "pg_dump: connection refused") {
|
|
t.Fatalf("run error = %v, want the snapshot's failure", err)
|
|
}
|
|
if res.NewestDB != daily || res.RemoteDB != 1 || res.WorldsUploaded != 1 {
|
|
t.Fatalf("result = %+v, want the daily still counted and the archive copied", res)
|
|
}
|
|
}
|
|
|
|
// TestSyncBucketKeepsDailiesPastSnapshots: snapshots do not count against
|
|
// DBKeep, so hourly ones leave the daily restore points alone, and only the
|
|
// newest snapshot stays, while it is the newest bundle of all.
|
|
func TestSyncBucketKeepsDailiesPastSnapshots(t *testing.T) {
|
|
s, b := newSyncer(t, &fakeCatalog{})
|
|
for _, name := range []string{
|
|
"felis-db-20260922T030000Z-daily.tar",
|
|
"felis-db-20260923T030000Z-daily.tar",
|
|
"felis-db-20260924T030000Z-daily.tar",
|
|
} {
|
|
b.objs[DBKey(name)] = []byte("daily")
|
|
}
|
|
for h := 4; h <= 9; h++ {
|
|
name := dbbackup.BundleName(time.Date(2026, 9, 24, h, 0, 0, 0, time.UTC), dbbackup.LabelOffsite)
|
|
b.objs[DBKey(name)] = []byte("snapshot")
|
|
}
|
|
b.objs[DBKey("felis-db-20260923T120000Z-offsite.tar")] = []byte("older than a daily")
|
|
|
|
res, err := s.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := []string{"felis-db-20260923T030000Z-daily.tar", "felis-db-20260924T030000Z-daily.tar", "felis-db-20260924T090000Z-offsite.tar"}
|
|
if got := dbKeys(b); !slices.Equal(got, want) {
|
|
t.Fatalf("bucket bundles = %v, want %v", got, want)
|
|
}
|
|
if res.RemoteDB != 3 || res.DBPruned != 7 || res.NewestDB != want[2] {
|
|
t.Fatalf("result = %+v, want 3 held, 7 pruned, the snapshot newest", res)
|
|
}
|
|
|
|
got := keptBundles([]string{"felis-db-20260925T030000Z-daily.tar", "felis-db-20260924T090000Z-offsite.tar", "felis-db-20260924T030000Z-daily.tar"}, 2)
|
|
if len(got) != 2 || got["felis-db-20260924T090000Z-offsite.tar"] {
|
|
t.Fatalf("kept = %v, want the two dailies: a snapshot older than a daily lists nothing it does not", got)
|
|
}
|
|
}
|
|
|
|
// TestSyncSweepsUnrecordedWorlds: a world object no row keeps goes once it is
|
|
// older than OrphanAfter; a younger one, one whose age the bucket does not
|
|
// report, one a row keeps however old, and anything that is not an archive
|
|
// object stay.
|
|
func TestSyncSweepsUnrecordedWorlds(t *testing.T) {
|
|
day := 24 * time.Hour
|
|
rows := func() []*row {
|
|
return []*row{
|
|
{WorldBackup: WorldBackup{ID: "kept", Ref: "/a/kept.tar.gz"}, status: "present", offsite: now.Add(-200 * day)},
|
|
{WorldBackup: WorldBackup{ID: "evicted", Ref: "/a/evicted.tar.gz"}, status: "deleted", expires: now.Add(day), offsite: now.Add(-100 * day)},
|
|
// Copied, but the run died before MarkOffsite, and the row has
|
|
// since expired: ExpiredRefs never lists it.
|
|
{WorldBackup: WorldBackup{ID: "unmarked", Ref: "/a/unmarked.tar.gz"}, status: "deleted", expires: now.Add(-day)},
|
|
}
|
|
}
|
|
ages := map[string]time.Duration{
|
|
"worlds/kept.tar.gz.fenc": 200 * day,
|
|
"worlds/evicted.tar.gz.fenc": 100 * day,
|
|
"worlds/unmarked.tar.gz.fenc": 95 * day,
|
|
"worlds/restored-old.tar.gz.fenc": 91 * day,
|
|
"worlds/restored-new.tar.gz.fenc": 10 * day,
|
|
"worlds/no-age.tar.gz.fenc": 0,
|
|
"worlds/README": 300 * day,
|
|
}
|
|
setup := func(t *testing.T, cat *fakeCatalog) (*Syncer, *memBucket, *bytes.Buffer) {
|
|
s, b := newSyncer(t, cat)
|
|
s.OrphanAfter = 90 * day
|
|
var log bytes.Buffer
|
|
s.Log = &log
|
|
b.modified = map[string]time.Time{}
|
|
for k, age := range ages {
|
|
b.objs[k] = []byte("x")
|
|
if age > 0 {
|
|
b.modified[k] = now.Add(-age)
|
|
}
|
|
}
|
|
return s, b, &log
|
|
}
|
|
|
|
s, b, log := setup(t, &fakeCatalog{rows: rows()})
|
|
res, err := s.Run(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
removed := slices.Sorted(slices.Values(b.removed))
|
|
if want := []string{"worlds/restored-old.tar.gz.fenc", "worlds/unmarked.tar.gz.fenc"}; !slices.Equal(removed, want) {
|
|
t.Fatalf("removed %v, want %v", removed, want)
|
|
}
|
|
if res.WorldsExpired != 2 || res.RemoteWorlds != len(ages)-2 {
|
|
t.Fatalf("result = %+v, want 2 expired, %d left", res, len(ages)-2)
|
|
}
|
|
if !strings.Contains(log.String(), "2 world archives in the bucket have no backup record") {
|
|
t.Errorf("the young unrecorded archives were not reported:\n%s", log.String())
|
|
}
|
|
|
|
// A database not restored yet keeps nothing: nothing is swept on it.
|
|
s, b, _ = setup(t, &fakeCatalog{})
|
|
if _, err := s.Run(context.Background()); err != nil || len(b.removed) != 0 {
|
|
t.Fatalf("an empty catalog: removed %v, err %v; want nothing", b.removed, err)
|
|
}
|
|
s, b, _ = setup(t, &fakeCatalog{rows: rows()})
|
|
s.OrphanAfter = 0
|
|
if _, err := s.Run(context.Background()); err != nil || len(b.removed) != 0 {
|
|
t.Fatalf("no OrphanAfter: removed %v, err %v; want nothing", b.removed, err)
|
|
}
|
|
}
|