From 5d1da31a484bca8422daf9bc522389ccaf0f4e7e Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Fri, 25 Sep 2026 04:32:42 +0800 Subject: [PATCH] =?UTF-8?q?fix(offsite):=20=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E5=8C=85=E4=B8=8E=E6=A1=B6=E5=86=85=E5=B7=B2=E6=9C=89=E4=BB=BD?= =?UTF-8?q?=E6=95=B0=E5=90=88=E5=B9=B6=E6=8E=92=E5=90=8D=EF=BC=8C=E5=8F=AA?= =?UTF-8?q?=E4=BC=A0=E4=BC=9A=E7=95=99=E4=B8=8B=E7=9A=84=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E6=AF=8F=E5=B0=8F=E6=97=B6=E9=87=8D=E4=BC=A0=E5=86=8D?= =?UTF-8?q?=E5=89=AA=E6=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/offsite/offsite_test.go | 27 +++++++++++++++ internal/offsite/sync.go | 58 +++++++++++++++++++++----------- 2 files changed, 65 insertions(+), 20 deletions(-) diff --git a/internal/offsite/offsite_test.go b/internal/offsite/offsite_test.go index 3eafcd2..720f7d1 100644 --- a/internal/offsite/offsite_test.go +++ b/internal/offsite/offsite_test.go @@ -392,6 +392,33 @@ func TestSyncDBKeepsNewest(t *testing.T) { } } +// TestSyncDBRanksWithTheBucket: an old local bundle (kept here by its label's +// own retention) that newer bundles in the bucket outrank is not sent, so a +// pass does not upload what it then prunes, and the next pass the same again. +func TestSyncDBRanksWithTheBucket(t *testing.T) { + s, b := newSyncer(t, &fakeCatalog{}) + writeFile(t, s.DBDir, "felis-db-20260910T030000Z-pre-migrate.tar", 50) + writeFile(t, s.DBDir, "felis-db-20260924T030000Z-daily.tar", 50) + b.objs["db/felis-db-20260923T030000Z-daily.tar.fenc"] = []byte("gone here, kept there") + + for pass := 1; pass <= 2; pass++ { + res, err := s.Run(context.Background()) + if err != nil { + t.Fatal(err) + } + want := 0 + if pass == 1 { + want = 1 + } + if res.DBUploaded != want || res.DBPruned != 0 || res.RemoteDB != 2 { + t.Fatalf("pass %d: result = %+v, want %d uploaded, none pruned, 2 held", pass, res, want) + } + } + if _, ok := b.objs["db/felis-db-20260910T030000Z-pre-migrate.tar.fenc"]; ok { + t.Fatal("the outranked bundle was sent") + } +} + // TestFetchWorldsRestoresVolume: after a rebuild, every present archive the // volume lacks comes back byte for byte; ones already there are left alone. func TestFetchWorldsRestoresVolume(t *testing.T) { diff --git a/internal/offsite/sync.go b/internal/offsite/sync.go index 11931e3..a30dd70 100644 --- a/internal/offsite/sync.go +++ b/internal/offsite/sync.go @@ -17,9 +17,11 @@ import ( "errors" "fmt" "io" + "maps" "os" "path" "path/filepath" + "slices" "sort" "strings" "syscall" @@ -240,12 +242,32 @@ func (s *Syncer) syncDB(ctx context.Context, res *Result, fail func(string, ...a fail("list %s in the bucket: %v", dbDir, err) return } - // Only the newest keep bundles are worth sending: older ones would be - // pruned again at the end of this very pass. - if len(local) > keep { - local = local[:keep] + // The bucket keeps the newest keep bundles of what it holds and what is + // here together. Local retention is per label, so an old pre-migrate + // bundle can outlive newer dailies here that the bucket still has: + // sending it would only see it pruned again at the end of this pass, and + // sent again on the next. + names := map[string]bool{} + for key := range remote { + name := strings.TrimSuffix(strings.TrimPrefix(key, dbDir), objExt) + if _, _, ok := dbbackup.ParseBundleName(name); ok && strings.HasSuffix(key, objExt) { + names[name] = true + } } for _, b := range local { + names[b.Name] = true + } + // Bundle names start with their UTC stamp, so reversed order is newest first. + ranked := slices.Sorted(maps.Keys(names)) + slices.Reverse(ranked) + kept := map[string]bool{} + for _, name := range ranked[:min(keep, len(ranked))] { + kept[name] = true + } + for _, b := range local { + if !kept[b.Name] { + continue + } key := DBKey(b.Name) want := SealedSize(b.Size) if size, ok := remote[key]; ok && size == want { @@ -260,29 +282,25 @@ func (s *Syncer) syncDB(ctx context.Context, res *Result, fail func(string, ...a res.BytesUploaded += b.Size s.logf("copied database bundle %s (%s)", b.Name, HumanBytes(b.Size)) } - - var names []string - for key := range remote { - name := strings.TrimSuffix(strings.TrimPrefix(key, dbDir), objExt) - if _, _, ok := dbbackup.ParseBundleName(name); ok && strings.HasSuffix(key, objExt) { - names = append(names, name) - } - } - // Bundle names start with their UTC stamp, so newest sorts last. - sort.Sort(sort.Reverse(sort.StringSlice(names))) - for i, name := range names { - if i < keep { + for _, name := range ranked { + key := DBKey(name) + if _, ok := remote[key]; !ok || kept[name] { continue } - if err := s.Bucket.Remove(ctx, DBKey(name)); err != nil { + if err := s.Bucket.Remove(ctx, key); err != nil { fail("prune database bundle %s: %v", name, err) continue } + delete(remote, key) res.DBPruned++ } - res.RemoteDB = min(len(names), keep) - if len(names) > 0 { - res.NewestDB = names[0] + for _, name := range ranked { + if _, ok := remote[DBKey(name)]; ok { + res.RemoteDB++ + if res.NewestDB == "" { + res.NewestDB = name + } + } } }