fix(reaper): 有服务器处理失败或过期备份删不掉时以非零码退出,让 Job 失败告警触达运维
This commit is contained in:
6 files changed
+119
-11
No files matched your search
@@ -675,8 +675,8 @@ func TestCapacityStillFullSkipsReap(t *testing.T) {
|
||||
r.Archiver.(*fakeArchiver).deleteErr = errors.New("evict unavailable")
|
||||
|
||||
sum := mustRun(t, r)
|
||||
if sum.WorldsReaped != 0 || sum.Skipped != 1 {
|
||||
t.Fatalf("summary = %+v, want 0 reaped / 1 skipped (store full)", sum)
|
||||
if sum.WorldsReaped != 0 || sum.Skipped != 1 || sum.StoreFull != 1 || !sum.Failed() {
|
||||
t.Fatalf("summary = %+v, want 0 reaped / 1 skipped, store full, failed", sum)
|
||||
}
|
||||
if cl.deletePVCCalls != 0 {
|
||||
t.Fatalf("world was deleted while the store was full")
|
||||
@@ -714,6 +714,24 @@ func TestExpiredBackupsDeleted(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// An expired backup the backend cannot delete stays present and marks the run
|
||||
// failed, so the Job reports it instead of succeeding every day.
|
||||
func TestExpiryFailureFailsTheRun(t *testing.T) {
|
||||
r, st, _, _ := newReaper(DefaultConfig())
|
||||
st.backups = []*fakeBackup{
|
||||
{id: "gone", server: "s1", ref: "ref-gone", size: 5, status: "present", createdAt: idleBy(120 * Day), expires: idleBy(1 * Day)},
|
||||
}
|
||||
r.Archiver.(*fakeArchiver).deleteErr = errors.New("permission denied")
|
||||
|
||||
sum := mustRun(t, r)
|
||||
if sum.BackupsExpired != 0 || sum.ExpireFailed != 1 || !sum.Failed() {
|
||||
t.Fatalf("summary = %+v, want 0 expired / 1 expire_failed, failed", sum)
|
||||
}
|
||||
if st.backups[0].status != "present" {
|
||||
t.Fatalf("the undeleted archive's row was marked %s", st.backups[0].status)
|
||||
}
|
||||
}
|
||||
|
||||
// A servers row whose CRD has been deleted is skipped (not a failure) — the
|
||||
// reaper never deletes world data it cannot first inspect for the exemption.
|
||||
func TestMissingCRDSkipped(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user