fix(offsite): 数据库包与桶内已有份数合并排名，只传会留下的，不再每小时重传再剪掉 (5d1da31a) · Commits · FelisMC / Felis · GitLab

internal/offsite/offsite_test.go

+27 −0

Changes for internal/offsite/offsite_test.go: 27 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -392,6 +392,33 @@ func TestSyncDBKeepsNewest(t *testing.T) {
		}
		}

		// TestSyncDBRanksWithTheBucket: an old local bundle (kept here by its label's
		// own retention) that newer bundles in the bucket outrank is not sent, so a
		// pass does not upload what it then prunes, and the next pass the same again.
		func TestSyncDBRanksWithTheBucket(t *testing.T) {
		s, b := newSyncer(t, &fakeCatalog{})
		writeFile(t, s.DBDir, "felis-db-20260910T030000Z-pre-migrate.tar", 50)
		writeFile(t, s.DBDir, "felis-db-20260924T030000Z-daily.tar", 50)
		b.objs["db/felis-db-20260923T030000Z-daily.tar.fenc"] = []byte("gone here, kept there")

		for pass := 1; pass <= 2; pass++ {
		res, err := s.Run(context.Background())
		if err != nil {
		t.Fatal(err)
		}
		want := 0
		if pass == 1 {
		want = 1
		}
		if res.DBUploaded != want \|\| res.DBPruned != 0 \|\| res.RemoteDB != 2 {
		t.Fatalf("pass %d: result = %+v, want %d uploaded, none pruned, 2 held", pass, res, want)
		}
		}
		if _, ok := b.objs["db/felis-db-20260910T030000Z-pre-migrate.tar.fenc"]; ok {
		t.Fatal("the outranked bundle was sent")
		}
		}

		// TestFetchWorldsRestoresVolume: after a rebuild, every present archive the
		// volume lacks comes back byte for byte; ones already there are left alone.
		func TestFetchWorldsRestoresVolume(t *testing.T) {

internal/offsite/sync.go

+38 −20

Changes for internal/offsite/sync.go: 38 added lines, 20 removed lines.
Original line number	Diff line number	Diff line
		@@ -17,9 +17,11 @@ import (
		"errors"
		"fmt"
		"io"
		"maps"
		"os"
		"path"
		"path/filepath"
		"slices"
		"sort"
		"strings"
		"syscall"
		@@ -240,12 +242,32 @@ func (s Syncer) syncDB(ctx context.Context, res Result, fail func(string, ...a
		fail("list %s in the bucket: %v", dbDir, err)
		return
		}
		// Only the newest keep bundles are worth sending: older ones would be
		// pruned again at the end of this very pass.
		if len(local) > keep {
		local = local[:keep]
		// The bucket keeps the newest keep bundles of what it holds and what is
		// here together. Local retention is per label, so an old pre-migrate
		// bundle can outlive newer dailies here that the bucket still has:
		// sending it would only see it pruned again at the end of this pass, and
		// sent again on the next.
		names := map[string]bool{}
		for key := range remote {
		name := strings.TrimSuffix(strings.TrimPrefix(key, dbDir), objExt)
		if _, _, ok := dbbackup.ParseBundleName(name); ok && strings.HasSuffix(key, objExt) {
		names[name] = true
		}
		}
		for _, b := range local {
		names[b.Name] = true
		}
		// Bundle names start with their UTC stamp, so reversed order is newest first.
		ranked := slices.Sorted(maps.Keys(names))
		slices.Reverse(ranked)
		kept := map[string]bool{}
		for _, name := range ranked[:min(keep, len(ranked))] {
		kept[name] = true
		}
		for _, b := range local {
		if !kept[b.Name] {
		continue
		}
		key := DBKey(b.Name)
		want := SealedSize(b.Size)
		if size, ok := remote[key]; ok && size == want {
		@@ -260,29 +282,25 @@ func (s Syncer) syncDB(ctx context.Context, res Result, fail func(string, ...a
		res.BytesUploaded += b.Size
		s.logf("copied database bundle %s (%s)", b.Name, HumanBytes(b.Size))
		}

		var names []string
		for key := range remote {
		name := strings.TrimSuffix(strings.TrimPrefix(key, dbDir), objExt)
		if _, _, ok := dbbackup.ParseBundleName(name); ok && strings.HasSuffix(key, objExt) {
		names = append(names, name)
		}
		}
		// Bundle names start with their UTC stamp, so newest sorts last.
		sort.Sort(sort.Reverse(sort.StringSlice(names)))
		for i, name := range names {
		if i < keep {
		for _, name := range ranked {
		key := DBKey(name)
		if _, ok := remote[key]; !ok \|\| kept[name] {
		continue
		}
		if err := s.Bucket.Remove(ctx, DBKey(name)); err != nil {
		if err := s.Bucket.Remove(ctx, key); err != nil {
		fail("prune database bundle %s: %v", name, err)
		continue
		}
		delete(remote, key)
		res.DBPruned++
		}
		res.RemoteDB = min(len(names), keep)
		if len(names) > 0 {
		res.NewestDB = names[0]
		for _, name := range ranked {
		if _, ok := remote[DBKey(name)]; ok {
		res.RemoteDB++
		if res.NewestDB == "" {
		res.NewestDB = name
		}
		}
		}
		}