diff --git a/cmd/felis/db.go b/cmd/felis/db.go index ab5ae17..e7b9208 100644 --- a/cmd/felis/db.go +++ b/cmd/felis/db.go @@ -415,10 +415,11 @@ func humanBytes(n int64) string { return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) } -// dbCheck is the freshness probe: exit 1 when the newest bundle is missing or -// older than -max-age, for a monitor or the break-glass console to act on. +// dbCheck is the freshness probe: exit 1 when the newest daily bundle is +// missing or older than -max-age, for a monitor or the break-glass console to +// act on. func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writer) int { - maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest bundle") + maxAge := fs.Duration("max-age", dbbackup.StaleAfter, "oldest acceptable newest daily bundle") if err := fs.Parse(args); err != nil { return 2 } @@ -427,7 +428,7 @@ func dbCheck(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Wri fmt.Fprintf(stderr, "felis db check: %v\n", err) return 1 } - fmt.Fprintf(stdout, "felis db check: ok, newest backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created))) + fmt.Fprintf(stdout, "felis db check: ok, newest daily backup %s (%s ago)\n", b.Name, dbbackup.Age(time.Since(b.Created))) return 0 } diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e882c68..b643e2c 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -382,9 +382,19 @@ components: Why the bundle lacks the MinecraftServer objects (the cluster did not answer the export), when it does. A restore from it brings back the database but no servers. + daily_at: + type: string + format: date-time + description: > + When the newest daily bundle (felis-db-backup.timer) in dir was + written, as of this record; absent when dir held none. Equal to + at when this record is a daily one. stale: type: boolean - description: True when there is no record or it is older than max_age_seconds. + description: > + True when there is no record, no daily bundle, or the newest daily + bundle is older than max_age_seconds. A newer manual, pre-migrate or + off-site bundle leaves it as it is: the daily timer has still stopped. max_age_seconds: type: integer format: int64 @@ -3684,8 +3694,8 @@ paths: description: >- What the host's felis-db-backup.timer (or a manual `felis db backup`) last recorded in platform_settings. last is null before the first - backup; stale is true then, and whenever the newest backup is older than - max_age_seconds. Read-only: backups run on the host, never through the API. + backup; stale is true then, and whenever the newest daily backup + (last.daily_at) is missing or older than max_age_seconds. Read-only: backups run on the host, never through the API. x-felis-face: [external] x-felis-tier: admin security: [{ sessionCookie: [] }] diff --git a/docs/operations.md b/docs/operations.md index 93f21b9..49298e5 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -690,7 +690,7 @@ production install: - **Rehearse the rebuild** once on a spare VM: troubleshooting.md §16 "Rebuild on a new host", every step but 8 (take-over) and 11 (the tunnel), then its checks: sign in with an email code, restore one world and join it. `felis offsite status` and `felis db check` exit - non-zero when the copy or the newest bundle is stale; wire them into your monitoring, + non-zero when the copy or the newest daily bundle is stale; wire them into your monitoring, or rely on the watchdog's mail. ### Moving to another host (planned) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c6a3d9e..1d698df 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1665,7 +1665,7 @@ Every two minutes the host checks: | Node `NotReady`, or kubelet reports Disk/Memory/PID pressure (§13b) | 2–5 min | critical | | PostgreSQL unreachable | 3 min | critical | | The game proxy (`felis-velocity`) refuses connections on the game port | 3 min | critical | -| Newest control-plane database backup over 26h old, or none (§16) | 10 min | critical | +| Newest daily control-plane database backup over 26h old, or none (§16); a newer manual or off-site bundle leaves it standing | 10 min | critical | | A watched filesystem below 15% free (below 5%: critical) | 15 min (5 min) | warning | | Host memory available below 10% | 15 min | warning | | The host no longer holds the address the install was made on (§13c) | 5 min | critical | @@ -2187,13 +2187,18 @@ Installer knobs: `FELIS_DB_BACKUP_DIR`, `FELIS_DB_BACKUP_KEEP`, ### Is the newest backup fresh? -Three places answer, all with the same 26 h limit: +Four places answer, all with the same 26 h limit on the newest **daily** +bundle, the one `felis-db-backup.timer` writes. A manual, `pre-migrate` or +`offsite` bundle taken since counts for a restore and leaves the alarm +standing: the timer has still stopped, and that bundle only ages from here. - The panel: **管理 → 维护与备份** shows the newest backup, its kind and size, - and turns red with the fix commands when it is missing or overdue (read from - the `db_backup_last` platform setting each backup writes). + and turns red with the fix commands when the daily one is missing or overdue + (read from the `db_backup_last` platform setting each backup writes; its + `daily_at` is the newest daily bundle on disk when it was written). - `sudo felis db check` exits 1 with the reason; `sudo felis db list` shows every bundle with its age. +- The watchdog mails the owners (§14). - Prometheus: `FelisDBBackupStale` (critical) and `FelisDBBackupMetricMissing` (warning) in `deploy/alerts/`. They read `felis_db_backup_last_success_timestamp_seconds`, which each daily run writes @@ -2207,7 +2212,7 @@ When a backup is overdue: ``` sudo systemctl status felis-db-backup.timer # enabled? next run? sudo journalctl -u felis-db-backup -n 50 --no-pager # why the last run failed -sudo felis db backup # take one now (label manual) +sudo systemctl start felis-db-backup.service # run the daily backup now; clears the alarm ``` **A bundle without the MinecraftServer objects.** When the cluster does not diff --git a/internal/api/handlers_dbbackup.go b/internal/api/handlers_dbbackup.go index 401c2a4..0cf4cf5 100644 --- a/internal/api/handlers_dbbackup.go +++ b/internal/api/handlers_dbbackup.go @@ -16,8 +16,8 @@ import ( // something on this install right now". // dbBackupView is the wire shape. Last is null until the first backup has been -// recorded; Stale is true for a missing record too, so the panel has a single -// flag for "nobody could restore today's state". +// recorded; Stale is true for a missing record or daily backup too, so the +// panel has a single flag for "the daily backups have stopped". type dbBackupView struct { Last *dbbackup.Status `json:"last"` Stale bool `json:"stale"` @@ -43,7 +43,14 @@ func (a *API) handleGetDBBackup(w http.ResponseWriter, r *http.Request) { writeError(w, r, err) return } + // Staleness goes by the daily timer's newest bundle: a manual or + // pre-migrate one recorded since would hide a timer that has stopped. A + // daily record is its own daily bundle, also when written before daily_at + // existed. No daily bundle leaves the zero time, centuries past the limit. + if st.Label == dbbackup.LabelDaily { + st.DailyAt = st.At + } view.Last = &st - view.Stale = st.At.IsZero() || a.now().Sub(st.At) > dbbackup.StaleAfter + view.Stale = a.now().Sub(st.DailyAt) > dbbackup.StaleAfter writeJSON(w, http.StatusOK, view) } diff --git a/internal/api/handlers_dbbackup_test.go b/internal/api/handlers_dbbackup_test.go index c865470..28dc809 100644 --- a/internal/api/handlers_dbbackup_test.go +++ b/internal/api/handlers_dbbackup_test.go @@ -69,6 +69,41 @@ func TestDBBackupFreshness(t *testing.T) { } } +// TestDBBackupDailyDecides: a fresh manual record says nothing about the +// timer, so the daily bundle it carries decides; a daily record from before +// daily_at existed is its own daily bundle. +func TestDBBackupDailyDecides(t *testing.T) { + now := time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + label string + dailyAge time.Duration // 0: no daily_at in the record + stale bool + wantDaily time.Time + }{ + {"manual over a 30h-old daily", "manual", 30 * time.Hour, true, now.Add(-30 * time.Hour)}, + {"manual over this morning's daily", "manual", 8 * time.Hour, false, now.Add(-8 * time.Hour)}, + {"manual with no daily at all", "pre-migrate", 0, true, time.Time{}}, + {"a daily record written before daily_at", "daily", 0, false, now.Add(-time.Hour)}, + } { + t.Run(tc.name, func(t *testing.T) { + api, repo := seedUpdatesAPI(t) + api.Now = func() time.Time { return now } + st := dbbackup.Status{At: now.Add(-time.Hour), Name: "felis-db-x-" + tc.label + ".tar", Label: tc.label, Dir: "/var/lib/felis/db-backups"} + if tc.dailyAge > 0 { + st.DailyAt = now.Add(-tc.dailyAge) + } + raw, _ := json.Marshal(st) + repo.settings[dbbackup.StatusKey] = raw + + code, v := getDBBackup(t, api) + if code != http.StatusOK || v.Last == nil || v.Stale != tc.stale || !v.Last.DailyAt.Equal(tc.wantDaily) || !v.Last.At.Equal(st.At) { + t.Fatalf("code %d, view %+v, want stale %v daily_at %s", code, v, tc.stale, tc.wantDaily) + } + }) + } +} + func TestDBBackupStoreOutageIsAnError(t *testing.T) { api, repo := seedUpdatesAPI(t) repo.failGetSetting = errors.New("connection reset") diff --git a/internal/api/openapi_parity_test.go b/internal/api/openapi_parity_test.go index d05537d..6e04657 100644 --- a/internal/api/openapi_parity_test.go +++ b/internal/api/openapi_parity_test.go @@ -17,7 +17,7 @@ import ( // response bodies. Each named schema is compared with the Go struct the handler // actually encodes: the property set must equal the struct's JSON field set, and // `required` must list exactly the fields that are always on the wire (no -// omitempty). The panel's types are checked against the same schemas at compile +// omitempty or omitzero). The panel's types are checked against the same schemas at compile // time (panel/src/lib/types.parity.ts), so a field added here without the docs // fails in Go, and one added to the docs without the panel fails in tsc. func TestOpenAPISchemasMatchWireStructs(t *testing.T) { @@ -125,7 +125,10 @@ func wireFields(t reflect.Type) map[string]wireField { if name == "" { name = f.Name } - out[name] = wireField{omitempty: strings.Contains(","+opts+",", ",omitempty,"), typ: f.Type} + // omitzero leaves a field out too, and is the one that does for a + // struct such as time.Time. + opts = "," + opts + "," + out[name] = wireField{omitempty: strings.Contains(opts, ",omitempty,") || strings.Contains(opts, ",omitzero,"), typ: f.Type} } return out } diff --git a/internal/dbbackup/dbbackup.go b/internal/dbbackup/dbbackup.go index 44f6c49..eea1f6a 100644 --- a/internal/dbbackup/dbbackup.go +++ b/internal/dbbackup/dbbackup.go @@ -157,9 +157,10 @@ const StatusKey = "db_backup_last" // failed every try: a restore from it brings back no servers. var ErrServersMissing = errors.New("the bundle holds the database but not the MinecraftServer objects") -// StaleAfter is how old the newest backup may get before it counts as missed: -// a day plus the timer's randomized delay and a slow dump. `felis db check`, -// the admin panel and the FelisDBBackupStale alert (deploy/alerts) share it. +// StaleAfter is how old the newest daily backup may get before it counts as +// missed: a day plus the timer's randomized delay and a slow dump. `felis db +// check`, the watchdog, the admin panel and the FelisDBBackupStale alert +// (deploy/alerts) share it. const StaleAfter = 26 * time.Hour // Status is the value stored under StatusKey. @@ -174,6 +175,11 @@ type Status struct { // ServersError is why the bundle lacks the MinecraftServer objects, when // it does. ServersError string `json:"servers_error,omitempty"` + // DailyAt is when the newest daily bundle in Dir was written, as of this + // record: the timer's own freshness, which a manual, pre-migrate or + // off-site bundle taken since leaves as it was. Zero when Dir held none, + // and in records written before the field existed. + DailyAt time.Time `json:"daily_at,omitzero"` } // Manifest describes a bundle. @@ -402,6 +408,17 @@ func List(dir string) ([]Bundle, error) { return out, nil } +// Newest is the first bundle of label in bundles, which List orders newest +// first. +func Newest(bundles []Bundle, label string) (Bundle, bool) { + for _, b := range bundles { + if b.Label == label { + return b, true + } + } + return Bundle{}, false +} + // Prune deletes all but the newest keep bundles of label (and their sidecars) // and returns what it removed. keep <= 0 removes nothing. func Prune(dir, label string, keep int) ([]string, error) { @@ -626,9 +643,15 @@ func listArchive(ctx context.Context, c conn, t Tools, path string) error { return err } -// record upserts st into platform_settings. The JSON travels as a psql -// variable, quoted by psql itself, over stdin (-c does not interpolate). +// record upserts st into platform_settings, with DailyAt read off st.Dir. The +// JSON travels as a psql variable, quoted by psql itself, over stdin (-c does +// not interpolate). func record(ctx context.Context, c conn, t Tools, st Status) error { + if all, err := List(st.Dir); err == nil { + if d, ok := Newest(all, LabelDaily); ok { + st.DailyAt = d.Created + } + } v, err := json.Marshal(st) if err != nil { return err @@ -835,19 +858,21 @@ func Age(d time.Duration) string { } } -// Check reports the newest bundle in dir and an error when there is none or it -// is older than maxAge. +// Check reports the newest daily bundle in dir and an error when there is none +// or it is older than maxAge. The daily timer is what keeps the backups +// current, so a manual, pre-migrate or off-site bundle taken since is left out: +// it would hide a timer that has stopped. func Check(dir string, maxAge time.Duration, now time.Time) (*Bundle, error) { all, err := List(dir) if err != nil { return nil, err } - if len(all) == 0 { - return nil, fmt.Errorf("no database backup in %s", dir) + daily, ok := Newest(all, LabelDaily) + if !ok { + return nil, fmt.Errorf("no daily database backup in %s (felis-db-backup.timer writes them)", dir) } - newest := all[0] - if age := now.Sub(newest.Created); age > maxAge { - return &newest, fmt.Errorf("newest database backup %s is %s old (limit %s)", newest.Name, Age(age), maxAge) + if age := now.Sub(daily.Created); age > maxAge { + return &daily, fmt.Errorf("newest daily database backup %s is %s old (limit %s)", daily.Name, Age(age), maxAge) } - return &newest, nil + return &daily, nil } diff --git a/internal/dbbackup/dbbackup_test.go b/internal/dbbackup/dbbackup_test.go index 862e90d..feda0a4 100644 --- a/internal/dbbackup/dbbackup_test.go +++ b/internal/dbbackup/dbbackup_test.go @@ -290,9 +290,19 @@ func TestBackupRecordsFreshness(t *testing.T) { } st := pg.recorded(t) info, _ := os.Stat(path) - if st.Name != filepath.Base(path) || st.Label != LabelDaily || !st.At.Equal(t0) || st.SizeBytes != info.Size() || st.SchemaVersion != 21 { + if st.Name != filepath.Base(path) || st.Label != LabelDaily || !st.At.Equal(t0) || st.SizeBytes != info.Size() || st.SchemaVersion != 21 || !st.DailyAt.Equal(t0) { t.Fatalf("recorded %+v", st) } + // A manual bundle an hour later is the newest record, and carries the + // daily one's time for the panel to judge the timer by. + if _, err := Backup(context.Background(), BackupOptions{ + DatabaseURL: testURL, Dir: dir, Label: LabelManual, Tools: pg.tools, Now: at(t0.Add(time.Hour)), Record: true, + }); err != nil { + t.Fatal(err) + } + if st := pg.recorded(t); st.Label != LabelManual || !st.At.Equal(t0.Add(time.Hour)) || !st.DailyAt.Equal(t0) { + t.Fatalf("recorded after a manual backup %+v, want daily_at %s", st, t0) + } prom, err := os.ReadFile(metrics) if err != nil { @@ -701,15 +711,30 @@ func TestListPruneCheck(t *testing.T) { t.Error("a pruned bundle's sidecar survived") } - if b, err := Check(dir, 26*time.Hour, t0); err != nil || b.Label != LabelPreMigrate { + // Check goes by the newest daily bundle: the pre-migrate one taken an hour + // ago says nothing about the timer. + if b, err := Check(dir, 26*time.Hour, t0); err != nil || b.Name != "felis-db-20260923T033000Z-daily.tar" { t.Errorf("Check fresh = %v, %v", b, err) } - if _, err := Check(dir, 26*time.Hour, t0.Add(30*time.Hour)); err == nil { - t.Error("Check accepted a 31h-old newest bundle") + if b, err := Check(dir, 26*time.Hour, t0.Add(3*time.Hour)); err == nil || b.Name != "felis-db-20260923T033000Z-daily.tar" || + err.Error() != "newest daily database backup felis-db-20260923T033000Z-daily.tar is 27h0m old (limit 26h0m0s)" { + t.Errorf("Check with a fresh pre-migrate bundle over a 27h-old daily one = %v, %v", b, err) } if _, err := Check(filepath.Join(dir, "none"), time.Hour, t0); err == nil { t.Error("Check accepted an empty directory") } + only := t.TempDir() + if _, err := Backup(context.Background(), BackupOptions{DatabaseURL: testURL, Dir: only, Label: LabelManual, Tools: pg.tools, Now: at(t0), Record: true}); err != nil { + t.Fatal(err) + } + // No daily bundle: the record leaves daily_at out (a zero time would read + // as year 1), so the panel shows none. + if args, _ := os.ReadFile(filepath.Join(pg.dir, "record.args")); !strings.Contains(string(args), `"label":"manual"`) || strings.Contains(string(args), "daily_at") { + t.Errorf("record without a daily bundle = %s", args) + } + if _, err := Check(only, 26*time.Hour, t0); err == nil || err.Error() != "no daily database backup in "+only+" (felis-db-backup.timer writes them)" { + t.Errorf("Check with a fresh manual bundle alone = %v", err) + } } func TestParseBundleName(t *testing.T) { diff --git a/internal/watchdog/probes.go b/internal/watchdog/probes.go index a3fb085..b35af30 100644 --- a/internal/watchdog/probes.go +++ b/internal/watchdog/probes.go @@ -341,8 +341,12 @@ func PostgresDown(err error) Finding { } } -// BackupFinding reports a control-plane database backup older than a day, or -// none at all, in dir, and a fresh one that would restore no servers. +// BackupFinding reports a daily control-plane database backup older than a +// day, or none at all, in dir, and a fresh one that would restore no servers. +// The age goes by the daily bundles alone: a manual or off-site bundle taken +// since would keep a stopped timer quiet until it too is a day old. The +// servers check reads the newest bundle of any label, the one a lost host +// restores from. func BackupFinding(dir string, now time.Time) *Finding { bundles, err := dbbackup.List(dir) if err != nil { @@ -353,19 +357,28 @@ func BackupFinding(dir string, now time.Time) *Finding { Hint: "docs/troubleshooting.md §16", } } - if len(bundles) > 0 && now.Sub(bundles[0].Created) <= maxBackupAge { + daily, ok := dbbackup.Newest(bundles, dbbackup.LabelDaily) + if ok && now.Sub(daily.Created) <= maxBackupAge { return serversFinding(bundles[0]) } f := &Finding{ Key: "db-backup", Severity: Critical, For: backupFor, Summary: fmt.Sprintf("%s 里没有任何控制面数据库备份", dir), SummaryEN: fmt.Sprintf("no control-plane database backup in %s", dir), - Hint: "journalctl -u felis-db-backup -n 50; take one now with `sudo felis db backup` (docs/troubleshooting.md §16)", + Hint: "journalctl -u felis-db-backup -n 50; once fixed, run it now with `sudo systemctl start felis-db-backup` (docs/troubleshooting.md §16)", } - if len(bundles) > 0 { - age := roundHours(now.Sub(bundles[0].Created)) - f.Summary = fmt.Sprintf("最新的控制面数据库备份已是 %s 前（%s）", age, bundles[0].Name) - f.SummaryEN = fmt.Sprintf("the newest control-plane database backup is %s old (%s)", age, bundles[0].Name) + switch { + case ok: + age := roundHours(now.Sub(daily.Created)) + f.Summary = fmt.Sprintf("最新的每日控制面数据库备份已是 %s 前（%s）", age, daily.Name) + f.SummaryEN = fmt.Sprintf("the newest daily control-plane database backup is %s old (%s)", age, daily.Name) + case len(bundles) > 0: + f.Summary = fmt.Sprintf("%s 里没有每日定时的控制面数据库备份", dir) + f.SummaryEN = fmt.Sprintf("no daily control-plane database backup in %s", dir) + } + if len(bundles) > 0 && bundles[0].Label != dbbackup.LabelDaily { + f.Summary += fmt.Sprintf("；更新的 %s 来自手动或其他任务，定时任务修好之前它会一天天变旧", bundles[0].Name) + f.SummaryEN += fmt.Sprintf("; the newer %s came from a manual run or another job and ages while the timer stays broken", bundles[0].Name) } return f } diff --git a/internal/watchdog/probes_test.go b/internal/watchdog/probes_test.go index 1ed3c33..8e2240d 100644 --- a/internal/watchdog/probes_test.go +++ b/internal/watchdog/probes_test.go @@ -149,16 +149,34 @@ func TestBackupFinding(t *testing.T) { if f := BackupFinding(dir, t0); f == nil || !strings.Contains(f.SummaryEN, "no control-plane database backup") { t.Fatalf("empty dir: %+v", f) } - touch := func(at time.Time) { - if err := os.WriteFile(filepath.Join(dir, dbbackup.BundleName(at, "daily")), []byte("x"), 0o600); err != nil { + touch := func(at time.Time, label string) { + if err := os.WriteFile(filepath.Join(dir, dbbackup.BundleName(at, label)), []byte("x"), 0o600); err != nil { t.Fatal(err) } } - touch(t0.Add(-30 * time.Hour)) - if f := BackupFinding(dir, t0); f == nil || !strings.Contains(f.SummaryEN, "30h old") { - t.Fatalf("stale: %+v", f) + // A manual bundle alone: the timer has never written one. + touch(t0.Add(-time.Hour), "manual") + if f := BackupFinding(dir, t0); f == nil || f.Key != "db-backup" || + f.SummaryEN != "no daily control-plane database backup in "+dir+"; the newer felis-db-20260924T110000Z-manual.tar came from a manual run or another job and ages while the timer stays broken" || + f.Summary != dir+" 里没有每日定时的控制面数据库备份；更新的 felis-db-20260924T110000Z-manual.tar 来自手动或其他任务，定时任务修好之前它会一天天变旧" { + t.Fatalf("manual only: %+v", f) } - touch(t0.Add(-2 * time.Hour)) + // A stale daily bundle under that fresh manual one: still the timer's alarm. + touch(t0.Add(-30*time.Hour), "daily") + if f := BackupFinding(dir, t0); f == nil || f.Key != "db-backup" || + f.SummaryEN != "the newest daily control-plane database backup is 30h old (felis-db-20260923T060000Z-daily.tar); the newer felis-db-20260924T110000Z-manual.tar came from a manual run or another job and ages while the timer stays broken" || + !strings.Contains(f.Hint, "sudo systemctl start felis-db-backup") { + t.Fatalf("stale daily under a fresh manual: %+v", f) + } + stale := t.TempDir() + if err := os.WriteFile(filepath.Join(stale, dbbackup.BundleName(t0.Add(-30*time.Hour), "daily")), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if f := BackupFinding(stale, t0); f == nil || f.SummaryEN != "the newest daily control-plane database backup is 30h old (felis-db-20260923T060000Z-daily.tar)" || + f.Summary != "最新的每日控制面数据库备份已是 30h 前（felis-db-20260923T060000Z-daily.tar）" { + t.Fatalf("stale daily alone: %+v", f) + } + touch(t0.Add(-2*time.Hour), "daily") if f := BackupFinding(dir, t0); f != nil { t.Fatalf("fresh backup reported: %+v", f) } diff --git a/panel/dev/mockApi.ts b/panel/dev/mockApi.ts index eb3b9e0..46c68c6 100644 --- a/panel/dev/mockApi.ts +++ b/panel/dev/mockApi.ts @@ -1013,6 +1013,7 @@ async function handleSession(ctx: SessionContext): Promise { felis_version: "dev", schema_version: 31, dir: "/var/lib/felis/db-backups", + daily_at: at.toISOString(), }, stale: false, max_age_seconds: 26 * 3600, diff --git a/panel/src/i18n/resources/en-US/admin.json b/panel/src/i18n/resources/en-US/admin.json index b96d01e..77d0f21 100644 --- a/panel/src/i18n/resources/en-US/admin.json +++ b/panel/src/i18n/resources/en-US/admin.json @@ -131,9 +131,12 @@ "dbbackup_label_manual": "Manual", "dbbackup_never_title": "No database backup has been recorded yet", "dbbackup_stale_title": "The newest backup is more than {{hours}} hours old", + "dbbackup_daily_stale_title": "The daily backup last completed {{when}}, past the {{hours}}-hour limit", + "dbbackup_daily_never_title": "The daily timer has not completed a backup yet", + "dbbackup_daily_fix_hint": "The newer backup above was taken by hand or by another job and can be restored from, but it grows older every day the timer stays broken. Run the daily backup on the host now, then read its log to find out why it stopped:", "dbbackup_servers_title": "The newest backup lacks the server definitions", "dbbackup_servers_hint": "The cluster did not answer when the backup was taken. Restoring from it brings back accounts and the database, but no servers. Once the cluster answers, take a backup on the host again:", - "dbbackup_fix_hint": "If the host failed now, accounts, server ownership and the archive index could not be recovered. Take a backup on the host now, then read the timer's log to find out why it did not run:", + "dbbackup_fix_hint": "If the host failed now, accounts, server ownership and the archive index could not be recovered. Run the daily backup on the host now, then read its log to find out why the timer missed it:", "dbbackup_copy": "Copy command", "dbbackup_offsite_note": "Backups are kept on this host only and are lost with its disk. Copy the backup directory to another machine regularly; restore and disaster-recovery steps are in the troubleshooting guide, §16.", "versions_title": "Component versions", diff --git a/panel/src/i18n/resources/zh-CN/admin.json b/panel/src/i18n/resources/zh-CN/admin.json index 8726fc1..a7742fb 100644 --- a/panel/src/i18n/resources/zh-CN/admin.json +++ b/panel/src/i18n/resources/zh-CN/admin.json @@ -131,9 +131,12 @@ "dbbackup_label_manual": "手动", "dbbackup_never_title": "还没有记录到任何数据库备份", "dbbackup_stale_title": "最近一次备份已超过 {{hours}} 小时", + "dbbackup_daily_stale_title": "每日定时备份最近一次完成是{{when}}，已超过 {{hours}} 小时", + "dbbackup_daily_never_title": "每日定时任务还没有完成过一次备份", + "dbbackup_daily_fix_hint": "上面这份更新的备份来自手动或其他任务，可以用来恢复，但定时任务修好之前它会一天天变旧。在主机上立即运行一次每日备份，再查看日志找出它停下的原因：", "dbbackup_servers_title": "最近一次备份缺少服务器定义", "dbbackup_servers_hint": "备份时集群没有响应。用它恢复能找回账号和数据库，但一台服务器都不会有。等集群恢复响应后，在主机上重新备份一次：", - "dbbackup_fix_hint": "此时主机出故障，账号、服务器归属和存档索引都无法恢复。在主机上立即备份一次，再查看定时任务日志找出它没有运行的原因：", + "dbbackup_fix_hint": "此时主机出故障，账号、服务器归属和存档索引都无法恢复。在主机上立即运行一次每日备份，再查看它的日志找出定时任务错过的原因：", "dbbackup_copy": "复制命令", "dbbackup_offsite_note": "备份只保存在这台主机上，硬盘损坏或主机丢失时会一起丢失。请定期把备份目录复制到另一台机器；恢复与灾备步骤见故障排查文档 §16。", "versions_title": "组件版本", diff --git a/panel/src/lib/openapi.gen.ts b/panel/src/lib/openapi.gen.ts index be050e6..21c5912 100644 --- a/panel/src/lib/openapi.gen.ts +++ b/panel/src/lib/openapi.gen.ts @@ -1103,7 +1103,7 @@ export interface paths { }; /** * Freshness of the newest control-plane database backup (admin). - * @description What the host's felis-db-backup.timer (or a manual `felis db backup`) last recorded in platform_settings. last is null before the first backup; stale is true then, and whenever the newest backup is older than max_age_seconds. Read-only: backups run on the host, never through the API. + * @description What the host's felis-db-backup.timer (or a manual `felis db backup`) last recorded in platform_settings. last is null before the first backup; stale is true then, and whenever the newest daily backup (last.daily_at) is missing or older than max_age_seconds. Read-only: backups run on the host, never through the API. */ get: operations["getDBBackup"]; put?: never; @@ -2343,8 +2343,13 @@ export interface components { dir: string; /** @description Why the bundle lacks the MinecraftServer objects (the cluster did not answer the export), when it does. A restore from it brings back the database but no servers. */ servers_error?: string; + /** + * Format: date-time + * @description When the newest daily bundle (felis-db-backup.timer) in dir was written, as of this record; absent when dir held none. Equal to at when this record is a daily one. + */ + daily_at?: string; } | null; - /** @description True when there is no record or it is older than max_age_seconds. */ + /** @description True when there is no record, no daily bundle, or the newest daily bundle is older than max_age_seconds. A newer manual, pre-migrate or off-site bundle leaves it as it is: the daily timer has still stopped. */ stale: boolean; /** * Format: int64 diff --git a/panel/src/lib/types.ts b/panel/src/lib/types.ts index 4cb19b1..55b419e 100644 --- a/panel/src/lib/types.ts +++ b/panel/src/lib/types.ts @@ -469,12 +469,14 @@ export interface DBBackupRecord { dir: string; /** Why the bundle lacks the MinecraftServer objects, when it does: a restore from it brings back no servers. */ servers_error?: string; + /** When the newest daily bundle in dir was written, as of this record; absent when there was none. */ + daily_at?: string; } export interface DBBackupStatus { /** Null until the host has recorded its first backup. */ last: DBBackupRecord | null; - /** True when there is no record or it is older than max_age_seconds. */ + /** True when there is no record, no daily bundle, or the newest daily bundle is older than max_age_seconds. */ stale: boolean; max_age_seconds: number; } diff --git a/panel/src/pages/admin/DBBackupCard.test.tsx b/panel/src/pages/admin/DBBackupCard.test.tsx index 8d2eafa..c33450d 100644 --- a/panel/src/pages/admin/DBBackupCard.test.tsx +++ b/panel/src/pages/admin/DBBackupCard.test.tsx @@ -68,6 +68,38 @@ describe("DBBackupCard", () => { expect(screen.getByText("The newest backup lacks the server definitions")).toBeTruthy(); }); + it("turns an old daily backup red and hands over the daily unit", async () => { + const old = new Date(Date.now() - 30 * 3600 * 1000).toISOString(); + calls.getDBBackup.mockResolvedValue(status({ at: old, daily_at: old }, true)); + render(); + expect(await screen.findByText("Overdue")).toBeTruthy(); + expect(screen.getByText("The newest backup is more than 26 hours old")).toBeTruthy(); + expect(screen.getByText(/find out why the timer missed it:$/)).toBeTruthy(); + expect(screen.getByText("yesterday").className).toBe("text-destructive"); + expect(screen.getByTitle("sudo systemctl start felis-db-backup.service")).toBeTruthy(); + }); + + it("names the stopped daily timer under a fresh manual backup", async () => { + const daily = new Date(Date.now() - 50 * 3600 * 1000).toISOString(); + calls.getDBBackup.mockResolvedValue( + status({ label: "manual", name: "felis-db-20260926T101500Z-manual.tar", daily_at: daily }, true), + ); + render(); + expect(await screen.findByText("Overdue")).toBeTruthy(); + expect(screen.getByText("The daily backup last completed 2 days ago, past the 26-hour limit")).toBeTruthy(); + expect(screen.getByText(/^The newer backup above was taken by hand or by another job and can be restored from/)).toBeTruthy(); + // The manual bundle itself is fresh: its age stays out of the alarm. + expect(screen.getByText("3 hours ago").className).toBe(""); + expect(screen.getByTitle("sudo systemctl start felis-db-backup.service")).toBeTruthy(); + }); + + it("says the daily timer never completed when only other kinds exist", async () => { + calls.getDBBackup.mockResolvedValue(status({ label: "pre-migrate", name: "felis-db-20260926T101500Z-pre-migrate.tar" }, true)); + render(); + expect(await screen.findByText("The daily timer has not completed a backup yet")).toBeTruthy(); + expect(screen.getByText(/^The newer backup above was taken by hand/)).toBeTruthy(); + }); + it("names an off-site copy snapshot by its kind", async () => { calls.getDBBackup.mockResolvedValue(status({ label: "offsite", name: "felis-db-20260926T101500Z-offsite.tar" })); render(); diff --git a/panel/src/pages/admin/DBBackupCard.tsx b/panel/src/pages/admin/DBBackupCard.tsx index 7e872f0..920dd38 100644 --- a/panel/src/pages/admin/DBBackupCard.tsx +++ b/panel/src/pages/admin/DBBackupCard.tsx @@ -23,7 +23,9 @@ const LABEL_KEY: Record = { manual: "dbbackup_label_manual", }; -const FIX_COMMANDS = ["sudo felis db backup", "journalctl -u felis-db-backup -n 50 --no-pager"]; +// The daily unit itself, run now: its bundle is what clears the alarm, and a +// failure lands in the log the second command reads. +const FIX_COMMANDS = ["sudo systemctl start felis-db-backup.service", "journalctl -u felis-db-backup -n 50 --no-pager"]; // A bundle the cluster did not add its MinecraftServer objects to: see why the // cluster did not answer, then take a whole one. const SERVERS_COMMANDS = ["sudo k3s kubectl get minecraftservers -A", "sudo felis db backup"]; @@ -78,6 +80,10 @@ export function DBBackupCard() { const last = data?.last ?? null; const maxAgeHours = data ? Math.round(data.max_age_seconds / 3600) : 26; const serversError = last?.servers_error ?? ""; + // Stale goes by the daily timer's newest bundle. When the newest record is + // another kind, the alarm names the timer and leaves that bundle's own age + // alone: it can still be restored from. + const lastIsDaily = last?.label === "daily"; const state: "loading" | "error" | "never" | "stale" | "ok" = !data ? error @@ -88,6 +94,15 @@ export function DBBackupCard() { : data.stale ? "stale" : "ok"; + const dailyAlarm = state === "stale" && !lastIsDaily; + const staleTitle = + state === "never" + ? t("dbbackup_never_title") + : !dailyAlarm + ? t("dbbackup_stale_title", { hours: maxAgeHours }) + : last?.daily_at + ? t("dbbackup_daily_stale_title", { hours: maxAgeHours, when: formatRelative(last.daily_at, Date.now(), locale) }) + : t("dbbackup_daily_never_title"); const badge = (() => { switch (state) { @@ -176,7 +191,7 @@ export function DBBackupCard() { {last && (
- + {formatRelative(last.at, Date.now(), locale) || "—"} @@ -206,10 +221,10 @@ export function DBBackupCard() {
-

- {state === "never" ? t("dbbackup_never_title") : t("dbbackup_stale_title", { hours: maxAgeHours })} +

{staleTitle}

+

+ {dailyAlarm ? t("dbbackup_daily_fix_hint") : t("dbbackup_fix_hint")}

-

{t("dbbackup_fix_hint")}