fix(dbbackup): 备份新鲜度按最新的 daily 包算,手动或异地包不再掩盖停掉的定时任务
This commit is contained in:
18 files changed
+262
-59
No files matched your search
@@ -341,8 +341,12 @@ func PostgresDown(err error) Finding {
|
||||
}
|
||||
}
|
||||
|
||||
// BackupFinding reports a control-plane database backup older than a day, or
|
||||
// none at all, in dir, and a fresh one that would restore no servers.
|
||||
// BackupFinding reports a daily control-plane database backup older than a
|
||||
// day, or none at all, in dir, and a fresh one that would restore no servers.
|
||||
// The age goes by the daily bundles alone: a manual or off-site bundle taken
|
||||
// since would keep a stopped timer quiet until it too is a day old. The
|
||||
// servers check reads the newest bundle of any label, the one a lost host
|
||||
// restores from.
|
||||
func BackupFinding(dir string, now time.Time) *Finding {
|
||||
bundles, err := dbbackup.List(dir)
|
||||
if err != nil {
|
||||
@@ -353,19 +357,28 @@ func BackupFinding(dir string, now time.Time) *Finding {
|
||||
Hint: "docs/troubleshooting.md §16",
|
||||
}
|
||||
}
|
||||
if len(bundles) > 0 && now.Sub(bundles[0].Created) <= maxBackupAge {
|
||||
daily, ok := dbbackup.Newest(bundles, dbbackup.LabelDaily)
|
||||
if ok && now.Sub(daily.Created) <= maxBackupAge {
|
||||
return serversFinding(bundles[0])
|
||||
}
|
||||
f := &Finding{
|
||||
Key: "db-backup", Severity: Critical, For: backupFor,
|
||||
Summary: fmt.Sprintf("%s 里没有任何控制面数据库备份", dir),
|
||||
SummaryEN: fmt.Sprintf("no control-plane database backup in %s", dir),
|
||||
Hint: "journalctl -u felis-db-backup -n 50; take one now with `sudo felis db backup` (docs/troubleshooting.md §16)",
|
||||
Hint: "journalctl -u felis-db-backup -n 50; once fixed, run it now with `sudo systemctl start felis-db-backup` (docs/troubleshooting.md §16)",
|
||||
}
|
||||
if len(bundles) > 0 {
|
||||
age := roundHours(now.Sub(bundles[0].Created))
|
||||
f.Summary = fmt.Sprintf("最新的控制面数据库备份已是 %s 前(%s)", age, bundles[0].Name)
|
||||
f.SummaryEN = fmt.Sprintf("the newest control-plane database backup is %s old (%s)", age, bundles[0].Name)
|
||||
switch {
|
||||
case ok:
|
||||
age := roundHours(now.Sub(daily.Created))
|
||||
f.Summary = fmt.Sprintf("最新的每日控制面数据库备份已是 %s 前(%s)", age, daily.Name)
|
||||
f.SummaryEN = fmt.Sprintf("the newest daily control-plane database backup is %s old (%s)", age, daily.Name)
|
||||
case len(bundles) > 0:
|
||||
f.Summary = fmt.Sprintf("%s 里没有每日定时的控制面数据库备份", dir)
|
||||
f.SummaryEN = fmt.Sprintf("no daily control-plane database backup in %s", dir)
|
||||
}
|
||||
if len(bundles) > 0 && bundles[0].Label != dbbackup.LabelDaily {
|
||||
f.Summary += fmt.Sprintf(";更新的 %s 来自手动或其他任务,定时任务修好之前它会一天天变旧", bundles[0].Name)
|
||||
f.SummaryEN += fmt.Sprintf("; the newer %s came from a manual run or another job and ages while the timer stays broken", bundles[0].Name)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
@@ -149,16 +149,34 @@ func TestBackupFinding(t *testing.T) {
|
||||
if f := BackupFinding(dir, t0); f == nil || !strings.Contains(f.SummaryEN, "no control-plane database backup") {
|
||||
t.Fatalf("empty dir: %+v", f)
|
||||
}
|
||||
touch := func(at time.Time) {
|
||||
if err := os.WriteFile(filepath.Join(dir, dbbackup.BundleName(at, "daily")), []byte("x"), 0o600); err != nil {
|
||||
touch := func(at time.Time, label string) {
|
||||
if err := os.WriteFile(filepath.Join(dir, dbbackup.BundleName(at, label)), []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
touch(t0.Add(-30 * time.Hour))
|
||||
if f := BackupFinding(dir, t0); f == nil || !strings.Contains(f.SummaryEN, "30h old") {
|
||||
t.Fatalf("stale: %+v", f)
|
||||
// A manual bundle alone: the timer has never written one.
|
||||
touch(t0.Add(-time.Hour), "manual")
|
||||
if f := BackupFinding(dir, t0); f == nil || f.Key != "db-backup" ||
|
||||
f.SummaryEN != "no daily control-plane database backup in "+dir+"; the newer felis-db-20260924T110000Z-manual.tar came from a manual run or another job and ages while the timer stays broken" ||
|
||||
f.Summary != dir+" 里没有每日定时的控制面数据库备份;更新的 felis-db-20260924T110000Z-manual.tar 来自手动或其他任务,定时任务修好之前它会一天天变旧" {
|
||||
t.Fatalf("manual only: %+v", f)
|
||||
}
|
||||
touch(t0.Add(-2 * time.Hour))
|
||||
// A stale daily bundle under that fresh manual one: still the timer's alarm.
|
||||
touch(t0.Add(-30*time.Hour), "daily")
|
||||
if f := BackupFinding(dir, t0); f == nil || f.Key != "db-backup" ||
|
||||
f.SummaryEN != "the newest daily control-plane database backup is 30h old (felis-db-20260923T060000Z-daily.tar); the newer felis-db-20260924T110000Z-manual.tar came from a manual run or another job and ages while the timer stays broken" ||
|
||||
!strings.Contains(f.Hint, "sudo systemctl start felis-db-backup") {
|
||||
t.Fatalf("stale daily under a fresh manual: %+v", f)
|
||||
}
|
||||
stale := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(stale, dbbackup.BundleName(t0.Add(-30*time.Hour), "daily")), []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f := BackupFinding(stale, t0); f == nil || f.SummaryEN != "the newest daily control-plane database backup is 30h old (felis-db-20260923T060000Z-daily.tar)" ||
|
||||
f.Summary != "最新的每日控制面数据库备份已是 30h 前(felis-db-20260923T060000Z-daily.tar)" {
|
||||
t.Fatalf("stale daily alone: %+v", f)
|
||||
}
|
||||
touch(t0.Add(-2*time.Hour), "daily")
|
||||
if f := BackupFinding(dir, t0); f != nil {
|
||||
t.Fatalf("fresh backup reported: %+v", f)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user