feat(backup): 归档先写 .partial 再 fsync 改名并记录 sha256,删除原件前回读校验,reaper 抽样巡检与清扫半截归档,保留权限与 mtime,面板标出损坏与已校验
This commit is contained in:
25 files changed
+1489
-125
No files matched your search
+10
-1
@@ -7,6 +7,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/backup"
|
||||
@@ -91,11 +92,16 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
|
||||
return 1
|
||||
}
|
||||
|
||||
ref, size, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
|
||||
a, err := archiver.Archive(ctx, *server, naming.WorldPVCName(*server))
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis backup: archive: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
ref, size := a.Ref, a.Size
|
||||
if len(a.Skipped) > 0 {
|
||||
fmt.Fprintf(stderr, "felis backup: %d entries are not plain files or directories and are not in the archive: %s\n",
|
||||
len(a.Skipped), strings.Join(a.Skipped[:min(len(a.Skipped), 10)], ", "))
|
||||
}
|
||||
|
||||
drv, err := store.Open(ctx, cfg.Database.URL)
|
||||
if err != nil {
|
||||
@@ -112,6 +118,9 @@ func cmdBackup(args []string, stdout, stderr io.Writer) int {
|
||||
SizeBytes: size,
|
||||
Reason: *reason,
|
||||
ExpiresAt: time.Now().Add(rcfg.ManualRetention),
|
||||
|
||||
SHA256: a.SHA256,
|
||||
SkippedEntries: len(a.Skipped),
|
||||
}
|
||||
st := reaper.NewPGStore(drv.DB())
|
||||
if err := st.InsertBackup(ctx, rec); err != nil {
|
||||
|
||||
+14
-4
@@ -138,14 +138,24 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
|
||||
// FelisWorldJobFailed rule) reach the operator: a world that cannot be archived
|
||||
// is kept, and without this nobody would learn that it is never reaped.
|
||||
func reportReaperRun(sum reaper.Summary, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d\n",
|
||||
fmt.Fprintf(stdout, "felis reaper: evaluated=%d reaped=%d awaiting_offsite=%d warned=%d skipped=%d store_full=%d evicted=%d expired=%d expire_failed=%d verified=%d corrupt=%d verify_failed=%d swept=%d orphan_archives=%d\n",
|
||||
sum.Evaluated, sum.WorldsReaped, sum.AwaitingOffsite, sum.Warned, sum.Skipped, sum.StoreFull,
|
||||
sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed)
|
||||
sum.EvictedEarly, sum.BackupsExpired, sum.ExpireFailed,
|
||||
sum.Verified, sum.Corrupt, sum.VerifyFailed, sum.Swept, sum.OrphanArchives)
|
||||
if !sum.Failed() {
|
||||
return 0
|
||||
}
|
||||
fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
|
||||
sum.Skipped, sum.StoreFull, sum.ExpireFailed)
|
||||
if sum.Skipped > 0 || sum.ExpireFailed > 0 {
|
||||
fmt.Fprintf(stderr, "felis reaper: %d servers failed (%d kept because the backup store is full) and %d expired backups were not removed; the errors are above, and each is retried next run\n",
|
||||
sum.Skipped, sum.StoreFull, sum.ExpireFailed)
|
||||
}
|
||||
if sum.Corrupt > 0 {
|
||||
fmt.Fprintf(stderr, "felis reaper: %d archives did not read back and are marked corrupt; they are no longer offered for restore (the errors are above)\n", sum.Corrupt)
|
||||
}
|
||||
if sum.VerifyFailed > 0 || sum.SweepFailed {
|
||||
fmt.Fprintf(stderr, "felis reaper: %d archives could not be read back and the store sweep completed=%t; both are retried next run\n",
|
||||
sum.VerifyFailed, !sum.SweepFailed)
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
|
||||
@@ -30,6 +30,10 @@ func TestReportReaperRunFailsTheJob(t *testing.T) {
|
||||
{"server failed", reaper.Summary{Evaluated: 3, Skipped: 1}, 1},
|
||||
{"store full", reaper.Summary{Evaluated: 3, Skipped: 1, StoreFull: 1}, 1},
|
||||
{"expiry failed", reaper.Summary{Evaluated: 3, ExpireFailed: 2}, 1},
|
||||
{"corrupt archive", reaper.Summary{Evaluated: 3, Verified: 4, Corrupt: 1}, 1},
|
||||
{"read-back failed", reaper.Summary{Evaluated: 3, VerifyFailed: 1}, 1},
|
||||
{"sweep failed", reaper.Summary{Evaluated: 3, SweepFailed: true}, 1},
|
||||
{"orphans kept", reaper.Summary{Evaluated: 3, Swept: 2, OrphanArchives: 1}, 0},
|
||||
} {
|
||||
var out, errb bytes.Buffer
|
||||
if got := reportReaperRun(tc.sum, &out, &errb); got != tc.want {
|
||||
|
||||
@@ -32,11 +32,11 @@ func archiveTempWorld(t *testing.T, files map[string]string) (ref string, backup
|
||||
BackupRoot: backupRoot,
|
||||
Resolve: func(string) (string, error) { return srcDir, nil },
|
||||
}
|
||||
got, _, err := ar.Archive(context.Background(), "survival", "world-survival-0")
|
||||
got, err := ar.Archive(context.Background(), "survival", "world-survival-0")
|
||||
if err != nil {
|
||||
t.Fatalf("Archive: %v", err)
|
||||
}
|
||||
return string(got), backupRoot
|
||||
return string(got.Ref), backupRoot
|
||||
}
|
||||
|
||||
// The restore subcommand must extract the archived world into the target world
|
||||
|
||||
Reference in new issue
Block a user