feat(offsite): 异地副本加入提交上传的整合包，按内容去重加密，索引按版本保留 14 天，新增 fetch-uploads 恢复
This commit is contained in:
11 files changed
+874
-72
No files matched your search
@@ -0,0 +1,198 @@
|
||||
package offsite
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func newUploadsSyncer(t *testing.T) (*Syncer, *memBucket, *time.Time, string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
b := newMemBucket()
|
||||
clock := now
|
||||
return &Syncer{
|
||||
Bucket: b, Catalog: &fakeCatalog{}, Key: testKey(t), UploadsDir: dir,
|
||||
Now: func() time.Time { return clock },
|
||||
}, b, &clock, dir
|
||||
}
|
||||
|
||||
func putContext(t *testing.T, dir, id string, data []byte, mtime time.Time) {
|
||||
t.Helper()
|
||||
sub := filepath.Join(dir, id)
|
||||
if err := os.MkdirAll(sub, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := filepath.Join(sub, UploadContextFile)
|
||||
if err := os.WriteFile(p, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chtimes(p, mtime, mtime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func runSync(t *testing.T, s *Syncer) Result {
|
||||
t.Helper()
|
||||
res, err := s.Run(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v (%v)", err, res.Errors)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// TestSyncUploadsCopiesContexts: every submission's context is copied, two
|
||||
// identical ones once; a directory without its context and a name that is not
|
||||
// a submission id are left out. A second run with nothing new sends nothing.
|
||||
func TestSyncUploadsCopiesContexts(t *testing.T) {
|
||||
s, b, _, dir := newUploadsSyncer(t)
|
||||
pack := bytes.Repeat([]byte("modpack"), 5000)
|
||||
putContext(t, dir, "sub-aaa", pack, now.Add(-time.Hour))
|
||||
putContext(t, dir, "sub-bbb", pack, now.Add(-time.Hour))
|
||||
putContext(t, dir, "sub-ccc", []byte("another pack"), now.Add(-time.Hour))
|
||||
if err := os.MkdirAll(filepath.Join(dir, "sub-writing"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "lost+found"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
res := runSync(t, s)
|
||||
if res.Uploads != 3 || res.UploadsUploaded != 2 {
|
||||
t.Fatalf("uploads=%d uploaded=%d, want 3 contexts in 2 objects", res.Uploads, res.UploadsUploaded)
|
||||
}
|
||||
if got := keysUnder(b, uploadBlobsDir); len(got) != 2 {
|
||||
t.Fatalf("blobs %v, want 2", got)
|
||||
}
|
||||
if got := keysUnder(b, uploadIndexDir); len(got) != 1 {
|
||||
t.Fatalf("index versions %v, want 1", got)
|
||||
}
|
||||
x, err := LoadUploadIndex(context.Background(), b, s.Key, res.UploadIndex)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(x.Contexts) != 3 || x.Contexts["sub-aaa"].Digest != x.Contexts["sub-bbb"].Digest {
|
||||
t.Fatalf("index %+v", x.Contexts)
|
||||
}
|
||||
|
||||
puts := b.puts
|
||||
res = runSync(t, s)
|
||||
if b.puts != puts || res.UploadsUploaded != 0 {
|
||||
t.Fatalf("second run put %d objects, uploaded %d; want nothing", b.puts-puts, res.UploadsUploaded)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSyncUploadsEmptyVolumeKeepsCopies: a volume that comes back empty (a
|
||||
// rebuilt host before its restore) records an empty version, keeps every
|
||||
// context for UploadHistory, and a restore refuses the empty version until
|
||||
// told which one to use. Past the history both go.
|
||||
func TestSyncUploadsEmptyVolumeKeepsCopies(t *testing.T) {
|
||||
s, b, clock, dir := newUploadsSyncer(t)
|
||||
putContext(t, dir, "sub-aaa", []byte("pack a"), now.Add(-time.Hour))
|
||||
putContext(t, dir, "sub-bbb", []byte("pack b"), now.Add(-time.Hour))
|
||||
first := runSync(t, s).UploadIndex
|
||||
|
||||
for _, id := range []string{"sub-aaa", "sub-bbb"} {
|
||||
if err := os.RemoveAll(filepath.Join(dir, id)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
*clock = clock.Add(time.Hour)
|
||||
res := runSync(t, s)
|
||||
if res.Uploads != 0 || res.UploadObjectsPruned != 0 {
|
||||
t.Fatalf("empty volume: uploads=%d pruned=%d", res.Uploads, res.UploadObjectsPruned)
|
||||
}
|
||||
if got := keysUnder(b, uploadBlobsDir); len(got) != 2 {
|
||||
t.Fatalf("blobs %v, want both kept", got)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if _, _, err := ChooseUploadIndex(ctx, b, s.Key, ""); err == nil || !strings.Contains(err.Error(), first) {
|
||||
t.Fatalf("choosing the empty newest version: %v, want a refusal naming %s", err, first)
|
||||
}
|
||||
stamp, x, err := ChooseUploadIndex(ctx, b, s.Key, first)
|
||||
if err != nil || stamp != first || len(x.Contexts) != 2 {
|
||||
t.Fatalf("choose %s: %s %v %v", first, stamp, x, err)
|
||||
}
|
||||
|
||||
*clock = clock.Add(UploadHistory + time.Hour)
|
||||
res = runSync(t, s)
|
||||
if got := keysUnder(b, uploadBlobsDir); len(got) != 0 {
|
||||
t.Fatalf("blobs %v after the history, want none", got)
|
||||
}
|
||||
if got := keysUnder(b, uploadIndexDir); len(got) != 1 {
|
||||
t.Fatalf("index versions %v after the history, want the newest only", got)
|
||||
}
|
||||
if res.UploadObjectsPruned != 3 {
|
||||
t.Fatalf("pruned %d, want the old version and its 2 contexts", res.UploadObjectsPruned)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSyncUploadsReplacedContext: a context rewritten in place is copied again;
|
||||
// the old bytes stay while the version that names them is kept.
|
||||
func TestSyncUploadsReplacedContext(t *testing.T) {
|
||||
s, b, clock, dir := newUploadsSyncer(t)
|
||||
putContext(t, dir, "sub-aaa", []byte("version one"), now.Add(-time.Hour))
|
||||
runSync(t, s)
|
||||
|
||||
putContext(t, dir, "sub-aaa", []byte("version two"), now)
|
||||
*clock = clock.Add(time.Hour)
|
||||
res := runSync(t, s)
|
||||
if res.UploadsUploaded != 1 {
|
||||
t.Fatalf("uploaded %d, want the new version", res.UploadsUploaded)
|
||||
}
|
||||
if got := keysUnder(b, uploadBlobsDir); len(got) != 2 {
|
||||
t.Fatalf("blobs %v, want old and new", got)
|
||||
}
|
||||
|
||||
*clock = clock.Add(UploadHistory + time.Hour)
|
||||
runSync(t, s)
|
||||
if got := keysUnder(b, uploadBlobsDir); len(got) != 1 {
|
||||
t.Fatalf("blobs %v after the history, want the new one only", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFetchUploadsRestores: a restore writes every context back in place,
|
||||
// leaves one that is already right alone, and rewrites one whose bytes differ.
|
||||
func TestFetchUploadsRestores(t *testing.T) {
|
||||
s, b, _, dir := newUploadsSyncer(t)
|
||||
a, c := bytes.Repeat([]byte("a"), 70000), []byte("pack c")
|
||||
putContext(t, dir, "sub-aaa", a, now.Add(-time.Hour))
|
||||
putContext(t, dir, "sub-ccc", c, now.Add(-time.Hour))
|
||||
stamp := runSync(t, s).UploadIndex
|
||||
|
||||
ctx := context.Background()
|
||||
x, err := LoadUploadIndex(ctx, b, s.Key, stamp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target := t.TempDir()
|
||||
res, err := FetchUploads(ctx, b, s.Key, x, target, -1, -1, nil)
|
||||
if err != nil || res.Written != 2 || res.Bytes != int64(len(a)+len(c)) {
|
||||
t.Fatalf("fetch: %+v %v", res, err)
|
||||
}
|
||||
for id, want := range map[string][]byte{"sub-aaa": a, "sub-ccc": c} {
|
||||
got, err := os.ReadFile(filepath.Join(target, id, UploadContextFile))
|
||||
if err != nil || !bytes.Equal(got, want) {
|
||||
t.Fatalf("%s restored as %d bytes (%v)", id, len(got), err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.WriteFile(filepath.Join(target, "sub-ccc", UploadContextFile), []byte("pack X"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res, err = FetchUploads(ctx, b, s.Key, x, target, -1, -1, nil)
|
||||
if err != nil || res.Written != 1 || res.Present != 1 {
|
||||
t.Fatalf("second fetch: %+v %v, want the damaged one rewritten", res, err)
|
||||
}
|
||||
if got, _ := os.ReadFile(filepath.Join(target, "sub-ccc", UploadContextFile)); !bytes.Equal(got, c) {
|
||||
t.Fatalf("sub-ccc is %q after the second fetch", got)
|
||||
}
|
||||
leftovers, _ := filepath.Glob(filepath.Join(target, "*", "*.restore"))
|
||||
if len(leftovers) != 0 {
|
||||
t.Fatalf("temporary files left behind: %v", leftovers)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user