From 149ab0be66c319df22ebae096e4017dccccc58c6 Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Sun, 27 Sep 2026 07:55:40 +0800 Subject: [PATCH] =?UTF-8?q?fix(offsite):=20=E6=A1=B6=E5=86=85=E8=AE=B0?= =?UTF-8?q?=E5=BD=95=E5=86=99=E5=85=A5=E4=B8=BB=E6=9C=BA=EF=BC=8C=E6=BC=94?= =?UTF-8?q?=E7=BB=83=E6=9C=BA=E5=8F=AA=E8=AF=BB=E4=B8=8D=E5=86=99=E4=B9=9F?= =?UTF-8?q?=E4=B8=8D=E7=BB=99=E7=94=9F=E4=BA=A7=20owner=20=E5=8F=91?= =?UTF-8?q?=E5=91=8A=E8=AD=A6=EF=BC=8Ctake-over=20=E6=98=BE=E5=BC=8F?= =?UTF-8?q?=E6=8E=A5=E7=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/felis/offsite.go | 178 ++++++++++++++++-- cmd/felis/offsite_test.go | 259 ++++++++++++++++++++++++- cmd/felis/watchdog.go | 26 ++- cmd/felis/watchdog_test.go | 49 +++++ deploy/bootstrap.sh | 50 ++++- deploy/bootstrap_test.sh | 56 +++++- docs/troubleshooting.md | 66 ++++++- internal/offsite/keymark.go | 20 +- internal/offsite/keymark_test.go | 19 +- internal/offsite/status.go | 40 ++++ internal/offsite/sync.go | 7 +- internal/offsite/writer.go | 263 ++++++++++++++++++++++++++ internal/offsite/writer_test.go | 313 +++++++++++++++++++++++++++++++ internal/watchdog/probes.go | 22 +++ internal/watchdog/probes_test.go | 15 ++ 15 files changed, 1327 insertions(+), 56 deletions(-) create mode 100644 internal/offsite/writer.go create mode 100644 internal/offsite/writer_test.go diff --git a/cmd/felis/offsite.go b/cmd/felis/offsite.go index 5863ad2..362b023 100644 --- a/cmd/felis/offsite.go +++ b/cmd/felis/offsite.go @@ -34,6 +34,7 @@ const offsiteUsage = `usage: felis offsite fetch-images [-config path] [-registry host:port] [-at version] felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version] felis offsite check-key [-config path] + felis offsite take-over [-config path] [-status-file path] [-yes] felis offsite keygen Every verb but keygen reads the bucket credentials and the encryption key from @@ -44,6 +45,13 @@ FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from check-key tells whether the key is the one the bucket's objects are sealed with, writing nothing; it exits 3 when they are sealed with another key, and sync then refuses to write or prune anything in the bucket. + +take-over names the host that writes the bucket, writing nothing; it exits 4 +when that is another host and this one never wrote it, and 5 when another +host took the bucket over from this one. A host built from another host's +backup (a rehearsal, or a rebuild) copies nothing into that host's bucket +until -yes makes it the writer; the host it replaces then stops copying and +says so. ` // defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the @@ -81,6 +89,8 @@ func cmdOffsite(args []string, stdout, stderr io.Writer) int { return offsiteFetchUploads(fs, rest, stdout, stderr) case "check-key": return offsiteCheckKey(fs, rest, stdout, stderr) + case "take-over": + return offsiteTakeOver(fs, rest, stdout, stderr) case "keygen": k, err := offsite.NewKey() if err != nil { @@ -213,28 +223,32 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int fmt.Fprintf(stderr, "felis offsite sync: %v\n", err) return 1 } - st := offsite.Status{ - LastAttempt: time.Now().UTC(), Endpoint: env.cfg.Endpoint, Bucket: env.cfg.Bucket, - Prefix: env.cfg.Prefix, KeyID: offsite.KeyID(env.key), - } - if prev, _ := offsite.ReadStatus(*statusFile); prev != nil { - st.LastSuccess = prev.LastSuccess - } + st, lease := startRun(env.cfg, env.key, *statusFile, time.Now()) res, err := runOffsiteSync(cfg, env, offsiteSources{ archiveDir: *archiveDir, backupPVC: *backupPVC, dbDir: *dbDir, registry: offsiteRegistryEndpoint(*registry, cfg.Registry), uploadsDir: *uploadsDir, uploadsPVC: *uploadsPVC, - }, stderr) - recordRun(&st, res, err) + }, &lease, stderr) + recordRun(&st, res, err, lease) if werr := offsite.WriteStatus(*statusFile, st); werr != nil { fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr) } - fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; uploads copied=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s), %d uploads (%s)\n", - res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired, - res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned, - res.UploadsUploaded, res.UploadObjectsPruned, - res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes), - res.Uploads, offsite.HumanBytes(res.RemoteUploadBytes)) + return reportRun(res, err, stdout, stderr) +} + +// reportRun prints one pass's outcome and its exit code. A run stopped before +// it copied anything (a bucket that did not answer, another key's objects, +// another host writing the bucket) prints no counts: its zeros would read as +// an empty bucket. +func reportRun(res offsite.Result, err error, stdout, stderr io.Writer) int { + if err == nil || len(res.Errors) > 0 { + fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; uploads copied=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s), %d uploads (%s)\n", + res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired, + res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned, + res.UploadsUploaded, res.UploadObjectsPruned, + res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes), + res.Uploads, offsite.HumanBytes(res.RemoteUploadBytes)) + } for _, m := range res.WorldsMissing { fmt.Fprintf(stderr, "felis offsite sync: recorded archive not on the volume, nothing to copy: %s\n", m) } @@ -248,15 +262,40 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int return 0 } -// recordRun puts one pass's outcome into its status record. -func recordRun(st *offsite.Status, res offsite.Result, err error) { +// startRun begins a pass: its status record, in this release's format and +// carrying the last success over, and this host's lease, read from the record +// the last pass left. +func startRun(cfg config.OffsiteConfig, key []byte, statusFile string, now time.Time) (offsite.Status, offsite.Lease) { + st := offsite.Status{ + LastAttempt: now.UTC(), Endpoint: cfg.Endpoint, Bucket: cfg.Bucket, + Prefix: cfg.Prefix, KeyID: offsite.KeyID(key), Format: offsite.StatusFormat, + } + if prev, _ := offsite.ReadStatus(statusFile); prev != nil { + st.LastSuccess = prev.LastSuccess + } + return st, offsite.HostLease(statusFile) +} + +// recordRun puts one pass's outcome into its status record. A host that +// inherited the bucket from an older release keeps that until it has an id. +func recordRun(st *offsite.Status, res offsite.Result, err error, lease offsite.Lease) { st.Result = res if err != nil { st.LastError = err.Error() st.KeyMismatch = errors.Is(err, offsite.ErrKeyMismatch) + var we *offsite.WriterError + if errors.As(err, &we) { + st.Standby = errors.Is(err, offsite.ErrStandby) + st.Displaced = errors.Is(err, offsite.ErrDisplaced) + st.Writer = we.Writer + } } else { st.LastSuccess = st.LastAttempt } + if lease.Inherited { + id, _ := lease.ID() + st.Inherited = id == "" + } } // offsiteSources is where one sync pass reads from: the world archive volume @@ -276,7 +315,7 @@ type offsiteSources struct { // TimeoutStartSec sits above this. const offsiteRunLimit = 23 * time.Hour -func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, log io.Writer) (offsite.Result, error) { +func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, lease *offsite.Lease, log io.Writer) (offsite.Result, error) { ctx, cancel := context.WithTimeout(context.Background(), offsiteRunLimit) defer cancel() checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second) @@ -309,7 +348,7 @@ func runOffsiteSync(cfg *config.Config, env *offsiteEnv, src offsiteSources, log defer drv.Close() s := &offsite.Syncer{ Bucket: env.bucket, Catalog: offsite.PGCatalog{DB: drv.DB()}, Key: env.key, - ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Log: log, + ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Lease: lease, Log: log, } if src.registry != "" { s.Images = newRegistryImages(src.registry) @@ -455,6 +494,23 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in fmt.Fprintf(stdout, "\nThe last run was refused: the bucket's objects are sealed with another key than this host's (key id %s). No sync copies or prunes anything there until FELIS_OFFSITE_KEY in %s is theirs (sudo felis offsite check-key).\n", st.KeyID, defaultOffsiteEnvFile) return 1 } + if st.Displaced && st.Writer != nil { + fmt.Fprintf(stdout, "\nThe last run was refused: %s took the bucket over (it last wrote it at %s), and this host copies nothing there any more. If that host is a rehearsal machine, take the bucket back: sudo felis offsite take-over -yes\n", + st.Writer, st.Writer.At.Local().Format(time.DateTime)) + return 1 + } + if st.Standby { + switch w := st.StandsBy(now); { + case w != nil: + fmt.Fprintf(stdout, "\nThis host stands by: %s writes the bucket (last at %s). This host was built from its backup, copies nothing into the bucket and, while that host keeps writing, mails no watchdog alert.", w, w.At.Local().Format(time.DateTime)) + case st.Writer != nil: + fmt.Fprintf(stdout, "\nThis host copies nothing into the bucket: %s wrote it, last at %s, and this host was built from its backup.", st.Writer, st.Writer.At.Local().Format(time.DateTime)) + default: + fmt.Fprint(stdout, "\nThis host copies nothing into the bucket: it holds copies this host did not write, and names no host writing it.") + } + fmt.Fprintln(stdout, " Once this host replaces that one for good: sudo felis offsite take-over -yes") + return 1 + } r := st.Result fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n", r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB)) @@ -612,6 +668,90 @@ func checkKey(ctx context.Context, b offsite.Bucket, key []byte, stdout, stderr return 0 } +func offsiteTakeOver(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int { + cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml") + envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set") + statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes; this host's id is kept next to it") + yes := fs.Bool("yes", false, "make this host the one that writes the bucket") + if err := fs.Parse(args); err != nil { + return 2 + } + _, env, err := loadOffsite(*cfgPath, *envFile) + if err != nil { + fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err) + return 1 + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := env.bucket.Check(ctx); err != nil { + fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err) + return 1 + } + return takeOver(ctx, env.bucket, env.key, offsite.HostLease(*statusFile), *statusFile, *yes, time.Now(), stdout, stderr) +} + +// takeOver is take-over once the bucket is open: without yes it says which +// host writes the bucket, 0 for this one (or none yet), 4 for another and 5 +// for one that took the bucket over from this host; with +// yes it records this host as the writer. A key the bucket's objects refuse +// is 3, as in check-key: taking over a bucket this host cannot copy into +// would only stop the host that can. +func takeOver(ctx context.Context, b offsite.Bucket, key []byte, lease offsite.Lease, statusFile string, yes bool, now time.Time, stdout, stderr io.Writer) int { + fit, err := offsite.CheckKey(ctx, b, key) + if err != nil { + fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err) + if errors.Is(err, offsite.ErrKeyMismatch) { + return 3 + } + return 1 + } + role, w, err := lease.Plan(ctx, b, fit == offsite.KeyUnused) + if err != nil { + fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err) + return 1 + } + switch role { + case offsite.RoleWrites: + id, _ := lease.ID() + fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket; nothing to take over\n", id) + return 0 + case offsite.RoleClaims: + fmt.Fprintln(stdout, "felis offsite take-over: the bucket names no host writing it; this host's next sync records itself") + return 0 + } + who := "another host" + if w != nil { + who = w.String() + fmt.Fprintf(stdout, "felis offsite take-over: %s writes the bucket, last at %s (%s ago)\n", w, w.At.Local().Format(time.DateTime), dbbackup.Age(now.Sub(w.At))) + } else { + fmt.Fprintln(stdout, "felis offsite take-over: the bucket holds copies this host did not write, and names no host writing it") + } + if !yes { + if role == offsite.RoleDisplaced { + fmt.Fprintf(stdout, "It took the bucket over from this host: this host copies nothing there any more, and its watchdog mails the owners about it. If that host is a rehearsal machine, take the bucket back:\n sudo felis offsite take-over -yes\n") + return 5 + } + fmt.Fprintf(stdout, "This host was built from its backup and copies nothing into the bucket.\n") + fmt.Fprintf(stdout, "Taking it over makes this host the one that copies into the bucket and prunes it; %s stops at its next copy and mails its owners. Do it once that host is gone for good, or is a rehearsal machine you are done with:\n sudo felis offsite take-over -yes\n", who) + return 4 + } + if _, err := lease.TakeOver(ctx, b, now); err != nil { + fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err) + return 1 + } + // The refusal the last sync recorded is over: the watchdog mails again + // from now on, and status shows the next run's outcome. + if st, err := offsite.ReadStatus(statusFile); err == nil && st != nil && (st.Standby || st.Displaced) { + st.Standby, st.Displaced, st.Writer, st.LastError, st.Inherited = false, false, nil, "", false + if err := offsite.WriteStatus(statusFile, *st); err != nil { + fmt.Fprintf(stderr, "felis offsite take-over: record status: %v\n", err) + } + } + id, _ := lease.ID() + fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket now; %s stops at its next copy.\nStart the first copy: sudo systemctl start felis-offsite.service\n", id, who) + return 0 +} + // keyHint explains an object the key cannot open when the bucket records // another key's id, "" otherwise. func keyHint(ctx context.Context, b offsite.Bucket, key []byte, err error) string { diff --git a/cmd/felis/offsite_test.go b/cmd/felis/offsite_test.go index 001b523..02c634c 100644 --- a/cmd/felis/offsite_test.go +++ b/cmd/felis/offsite_test.go @@ -356,23 +356,218 @@ func TestOffsiteCheckKey(t *testing.T) { } } -func TestRecordRunMarksAKeyMismatch(t *testing.T) { +func TestRecordRun(t *testing.T) { t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC) + w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0} for _, tc := range []struct { - err error - mismatch bool - success bool + err error + mismatch, standby, displaced, success bool }{ - {nil, false, true}, - {errors.New("list worlds/ in the bucket: connection reset"), false, false}, - {fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false}, + {nil, false, false, false, true}, + {errors.New("list worlds/ in the bucket: connection reset"), false, false, false, false}, + {fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false, false, false}, + {&offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, false, true, false, false}, + {&offsite.WriterError{Kind: offsite.ErrDisplaced, Writer: w}, false, false, true, false}, } { st := offsite.Status{LastAttempt: t0} - recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err) - if st.KeyMismatch != tc.mismatch || st.LastSuccess.Equal(t0) != tc.success || st.Result.RemoteDB != 2 { + recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err, offsite.Lease{}) + if st.KeyMismatch != tc.mismatch || st.Standby != tc.standby || st.Displaced != tc.displaced || + (st.Writer != nil) != (tc.standby || tc.displaced) || st.LastSuccess.Equal(t0) != tc.success || st.Result.RemoteDB != 2 { t.Errorf("err %v: status %+v", tc.err, st) } } + + // A host that copied before writers were recorded keeps that claim over + // failed runs until it has an id. + l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Inherited: true} + st := offsite.Status{} + recordRun(&st, offsite.Result{}, errors.New("cannot reach bucket"), l) + if !st.Inherited { + t.Error("a failed run on a host with no id dropped the older release's claim") + } + writeTestFile(t, l.IDFile, "aaaaaaaaaaaaaaaa\n", 0o600) + st = offsite.Status{Inherited: true} + recordRun(&st, offsite.Result{}, nil, l) + if st.Inherited { + t.Error("a host with an id still carries the older release's claim") + } +} + +// A refused run has copied nothing and listed nothing: its zero counts would +// tell the journal the bucket is empty. A pass that ran and failed a step +// shows what it did get to. +func TestReportRun(t *testing.T) { + w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)} + for _, tc := range []struct { + name string + res offsite.Result + err error + code int + counts bool + }{ + {"a pass", offsite.Result{DBUploaded: 1, RemoteDB: 3}, nil, 0, true}, + {"a pass with a failed step", offsite.Result{RemoteDB: 3, Errors: []string{"copy db/x: timeout"}}, errors.New("1 of this run's steps failed; first: copy db/x: timeout"), 1, true}, + {"standing by", offsite.Result{}, &offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, 1, false}, + {"another key", offsite.Result{}, fmt.Errorf("%w: the bucket records key id 1111111111111111", offsite.ErrKeyMismatch), 1, false}, + {"no bucket", offsite.Result{}, errors.New("bucket: access denied"), 1, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var out, errOut bytes.Buffer + code := reportRun(tc.res, tc.err, &out, &errOut) + if code != tc.code { + t.Errorf("exit %d, want %d", code, tc.code) + } + if got := strings.Contains(out.String(), "bucket holds 0 worlds (0 B), 3 bundles"); got != tc.counts { + t.Errorf("counts shown = %v, want %v: %q", got, tc.counts, out.String()) + } + if !tc.counts && out.Len() > 0 { + t.Errorf("a refused run printed %q", out.String()) + } + if tc.err != nil && !strings.Contains(errOut.String(), "felis offsite sync: "+tc.err.Error()) { + t.Errorf("stderr %q lacks the error", errOut.String()) + } + }) + } +} + +func TestOffsiteTakeOver(t *testing.T) { + newKey := func() []byte { + raw, _ := offsite.NewKey() + k, _ := offsite.ParseKey(raw) + return k + } + key, other := newKey(), newKey() + const mine, theirs = "aaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbb" + t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC) + sealed := func(k []byte, writer string) mapBucket { + b := mapBucket{} + putBundle(t, b, k, 0, nil) + b["felis-key-id"] = []byte(offsite.KeyID(k) + "\n") + if writer != "" { + raw, _ := json.Marshal(offsite.Writer{HostID: writer, Host: "prod-1", At: t0.Add(-20 * time.Minute)}) + b["felis-writer"] = raw + } + return b + } + lease := func(id string) offsite.Lease { + l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Host: "spare-1"} + if id != "" { + writeTestFile(t, l.IDFile, id+"\n", 0o600) + } + return l + } + run := func(b mapBucket, l offsite.Lease, statusFile string, yes bool) (int, string, string) { + t.Helper() + var out, errb bytes.Buffer + code := takeOver(context.Background(), b, key, l, statusFile, yes, t0, &out, &errb) + return code, out.String(), errb.String() + } + for _, tc := range []struct { + what string + bucket mapBucket + id string + code int + says []string + }{ + {"this host writes the bucket", sealed(key, mine), mine, 0, []string{"this host (id " + mine + ") writes the bucket; nothing to take over"}}, + {"an empty bucket", mapBucket{}, "", 0, []string{"names no host writing it; this host's next sync records itself"}}, + {"a host built from the writer's backup", sealed(key, theirs), "", 4, []string{"host prod-1 (id " + theirs + ") writes the bucket, last at", "(20m ago)", "built from its backup", "sudo felis offsite take-over -yes"}}, + {"another host's copies, no writer named", sealed(key, ""), "", 4, []string{"holds copies this host did not write, and names no host writing it", "take-over -yes"}}, + {"a host another one took over from", sealed(key, theirs), mine, 5, []string{"took the bucket over from this host"}}, + {"a key the bucket refuses", sealed(other, theirs), "", 3, []string{"sealed with another key"}}, + } { + t.Run(tc.what, func(t *testing.T) { + before := string(tc.bucket["felis-writer"]) + l := lease(tc.id) + code, out, errb := run(tc.bucket, l, filepath.Join(t.TempDir(), "status.json"), false) + if code != tc.code { + t.Fatalf("exit %d, want %d\n%s%s", code, tc.code, out, errb) + } + for _, s := range tc.says { + if !strings.Contains(out+errb, s) { + t.Errorf("output lacks %q:\n%s%s", s, out, errb) + } + } + if string(tc.bucket["felis-writer"]) != before { + t.Error("take-over without -yes wrote the bucket's writer") + } + if tc.id == "" { + if _, err := os.Stat(l.IDFile); !errors.Is(err, os.ErrNotExist) { + t.Errorf("take-over without -yes made this host an id: %v", err) + } + } + }) + } + + // -yes on a standby host: the bucket names it, and the refusal the last + // sync recorded is cleared, so the watchdog mails again at once. + b := sealed(key, theirs) + l := lease("") + statusFile := filepath.Join(t.TempDir(), "status.json") + lastSuccess := t0.Add(-48 * time.Hour) + if err := offsite.WriteStatus(statusFile, offsite.Status{ + LastAttempt: t0.Add(-time.Hour), LastSuccess: lastSuccess, LastError: "offsite: another host writes this bucket", Format: offsite.StatusFormat, + Standby: true, Writer: &offsite.Writer{HostID: theirs, Host: "prod-1", At: t0}, Inherited: true, + }); err != nil { + t.Fatal(err) + } + code, out, errb := run(b, l, statusFile, true) + id, _ := l.ID() + if code != 0 || id == "" || !strings.Contains(out, "this host (id "+id+") writes the bucket now; host prod-1 (id "+theirs+") stops at its next copy") || !strings.Contains(out, "systemctl start felis-offsite.service") { + t.Fatalf("take-over -yes: exit %d, id %q\n%s%s", code, id, out, errb) + } + if w, err := offsite.BucketWriter(context.Background(), b); err != nil || w.HostID != id || w.Host != "spare-1" || !w.At.Equal(t0) { + t.Errorf("writer after take-over -yes = %+v, %v", w, err) + } + st, err := offsite.ReadStatus(statusFile) + if err != nil || st.Standby || st.Writer != nil || st.LastError != "" || st.Inherited || !st.LastSuccess.Equal(lastSuccess) { + t.Errorf("status after take-over -yes = %+v, %v; want the refusal cleared and the last success kept", st, err) + } + + // -yes with a key the bucket refuses writes nothing. + b = sealed(other, theirs) + before := string(b["felis-writer"]) + if code, _, _ := run(b, lease(""), filepath.Join(t.TempDir(), "status.json"), true); code != 3 || string(b["felis-writer"]) != before { + t.Errorf("take-over -yes under another key: exit %d, writer %s", code, b["felis-writer"]) + } +} + +func TestOffsiteStatusSaysWhoWritesTheBucket(t *testing.T) { + dir := t.TempDir() + cfg := filepath.Join(dir, "felis.toml") + writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600) + statusFile := filepath.Join(dir, "status.json") + now := time.Now() + w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-30 * time.Minute)} + stale := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-offsite.WriterLive - time.Hour)} + for _, tc := range []struct { + what string + st offsite.Status + says []string + not string + }{ + {"standing by for a live writer", offsite.Status{Standby: true, Writer: w}, []string{"This host stands by: host prod-1 (id bbbbbbbbbbbbbbbb) writes the bucket", "mails no watchdog alert", "take-over -yes"}, "wrote it, last at"}, + {"standing by for a writer gone quiet", offsite.Status{Standby: true, Writer: stale}, []string{"copies nothing into the bucket: host prod-1 (id bbbbbbbbbbbbbbbb) wrote it, last at", "take-over -yes"}, "mails no watchdog alert"}, + {"standing by, no writer named", offsite.Status{Standby: true}, []string{"names no host writing it", "take-over -yes"}, "stands by:"}, + {"displaced", offsite.Status{Displaced: true, Writer: w}, []string{"host prod-1 (id bbbbbbbbbbbbbbbb) took the bucket over", "rehearsal machine", "take-over -yes"}, "stands by"}, + } { + t.Run(tc.what, func(t *testing.T) { + tc.st.LastAttempt, tc.st.LastSuccess, tc.st.LastError = now.Add(-time.Minute), now.Add(-time.Hour), "offsite: another host writes this bucket" + if err := offsite.WriteStatus(statusFile, tc.st); err != nil { + t.Fatal(err) + } + var out, errb bytes.Buffer + code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb) + if code != 1 || strings.Contains(out.String(), "bucket holds:") || strings.Contains(out.String(), tc.not) { + t.Errorf("exit %d\n%s%s", code, out.String(), errb.String()) + } + for _, s := range tc.says { + if !strings.Contains(out.String(), s) { + t.Errorf("output lacks %q:\n%s", s, out.String()) + } + } + }) + } } func TestOffsiteStatusSaysTheKeyWasRefused(t *testing.T) { @@ -432,3 +627,49 @@ func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) { } } } + +// TestRestoredHostKeepsStandingBy walks the status file across runs: a host +// restored from the writer's backup stands by on its first run and on every +// run after it, a host an older release left copying claims the bucket once, +// and a failed first run after the upgrade keeps that claim. +func TestRestoredHostKeepsStandingBy(t *testing.T) { + rawKey, _ := offsite.NewKey() + key, _ := offsite.ParseKey(rawKey) + cfg := config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-backups"} + t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC) + standby := &offsite.WriterError{Kind: offsite.ErrStandby, Writer: &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}} + pass := func(statusFile string, at time.Time, err error) offsite.Lease { + t.Helper() + st, lease := startRun(cfg, key, statusFile, at) + recordRun(&st, offsite.Result{}, err, lease) + if werr := offsite.WriteStatus(statusFile, st); werr != nil { + t.Fatal(werr) + } + return lease + } + + restored := filepath.Join(t.TempDir(), "status.json") + for i := range 3 { + if l := pass(restored, t0.Add(time.Duration(i)*time.Hour), standby); l.Inherited { + t.Fatalf("run %d of a restored host claims the bucket", i+1) + } + } + if st, _ := offsite.ReadStatus(restored); !st.Standby || st.Format != offsite.StatusFormat || st.KeyID != offsite.KeyID(key) || st.Bucket != "felis-backups" { + t.Errorf("restored host's status = %+v", st) + } + + upgraded := filepath.Join(t.TempDir(), "status.json") + if err := offsite.WriteStatus(upgraded, offsite.Status{LastAttempt: t0.Add(-time.Hour), LastSuccess: t0.Add(-time.Hour)}); err != nil { + t.Fatal(err) + } + if l := pass(upgraded, t0, errors.New("cannot reach bucket")); !l.Inherited { + t.Fatal("the first run after the upgrade does not claim the bucket") + } + l := pass(upgraded, t0.Add(time.Hour), nil) + if !l.Inherited { + t.Fatal("a failed first run after the upgrade lost the claim") + } + if st, _ := offsite.ReadStatus(upgraded); !st.LastSuccess.Equal(t0.Add(time.Hour)) { + t.Errorf("upgraded host's status = %+v", st) + } +} diff --git a/cmd/felis/watchdog.go b/cmd/felis/watchdog.go index bc7ede3..909516d 100644 --- a/cmd/felis/watchdog.go +++ b/cmd/felis/watchdog.go @@ -155,8 +155,8 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int { if plan.Empty() { return save() } - if until := watchdog.QuietUntil(*quietPath); now.Before(until) { - fmt.Fprintf(stdout, "felis watchdog: quiet until %s (installer running); holding this mail: %s\n", until.UTC().Format(time.RFC3339), subject) + if hold := mailHold(*quietPath, cfg.Offsite.Enabled(), *offsiteStatus, now); hold != "" { + fmt.Fprintf(stdout, "felis watchdog: %s; holding this mail: %s\n", hold, subject) return save() } switch { @@ -177,6 +177,28 @@ func cmdWatchdog(args []string, stdout, stderr io.Writer) int { return save() } +// mailHold is why this run's mail waits, "" when it goes out: the installer's +// quiet window, or this host standing by for another host's off-site bucket +// (offsite.Status.StandsBy). A standby host is a rehearsal, or a rebuild not +// yet taken over, and the owners its restored database names are that host's, +// which mails them itself. +func mailHold(quietPath string, offsiteOn bool, offsiteStatus string, now time.Time) string { + if until := watchdog.QuietUntil(quietPath); now.Before(until) { + return fmt.Sprintf("quiet until %s (installer running)", until.UTC().Format(time.RFC3339)) + } + if !offsiteOn { + return "" + } + st, err := offsite.ReadStatus(offsiteStatus) + if err != nil { + return "" + } + if w := st.StandsBy(now); w != nil { + return fmt.Sprintf("this host stands by for %s, which wrote the off-site bucket at %s and mails its owners itself", w, w.At.UTC().Format(time.RFC3339)) + } + return "" +} + // usesMirroredScanDB reports whether build scans read the vulnerability DB copy // felis mirror-build-tools keeps in the platform registry: the default, or an // explicit trivy_db_repository under the registry's mirror/. diff --git a/cmd/felis/watchdog_test.go b/cmd/felis/watchdog_test.go index 8ff042f..d3e0acf 100644 --- a/cmd/felis/watchdog_test.go +++ b/cmd/felis/watchdog_test.go @@ -2,9 +2,15 @@ package main import ( "context" + "fmt" "net" + "os" + "path/filepath" "strings" "testing" + "time" + + "felis.lolicon.best/internal/offsite" ) // TestProxyFinding: a listening proxy is healthy; a closed port is the critical @@ -40,3 +46,46 @@ func TestSplitList(t *testing.T) { t.Fatalf("splitList = %q", got) } } + +// TestMailHold: mail waits through the installer's quiet window, and on a host +// standing by for another host's off-site bucket while that host writes it. +func TestMailHold(t *testing.T) { + dir := t.TempDir() + quiet, status := filepath.Join(dir, "quiet"), filepath.Join(dir, "status.json") + now := time.Now() + if got := mailHold(quiet, true, status, now); got != "" { + t.Errorf("no quiet file, no status: %q", got) + } + writeTestFile(t, quiet, fmt.Sprintf("%d\n", now.Add(time.Hour).Unix()), 0o644) + if got := mailHold(quiet, false, status, now); !strings.Contains(got, "quiet until") { + t.Errorf("inside the quiet window: %q", got) + } + os.Remove(quiet) + + w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-40 * time.Minute)} + for _, tc := range []struct { + what string + st offsite.Status + offsiteOn bool + held bool + }{ + {"standing by for a live writer", offsite.Status{Standby: true, Writer: w}, true, true}, + {"standing by, [offsite] since removed", offsite.Status{Standby: true, Writer: w}, false, false}, + {"standing by for a writer gone quiet", offsite.Status{Standby: true, Writer: &offsite.Writer{HostID: w.HostID, Host: w.Host, At: now.Add(-offsite.WriterLive - time.Minute)}}, true, false}, + {"standing by, no writer named", offsite.Status{Standby: true}, true, false}, + {"displaced", offsite.Status{Displaced: true, Writer: w}, true, false}, + {"the writer itself", offsite.Status{LastSuccess: now}, true, false}, + } { + if err := offsite.WriteStatus(status, tc.st); err != nil { + t.Fatal(err) + } + got := mailHold(quiet, tc.offsiteOn, status, now) + if (got != "") != tc.held || (tc.held && !strings.Contains(got, "stands by for host prod-1 (id bbbbbbbbbbbbbbbb)")) { + t.Errorf("%s: hold = %q, want held %v", tc.what, got, tc.held) + } + } + writeTestFile(t, status, "{", 0o600) + if got := mailHold(quiet, true, status, now); got != "" { + t.Errorf("an unreadable status held the mail: %q", got) + } +} diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index b47a183..4a26b7b 100644 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -4541,9 +4541,9 @@ quiet_watchdog() { } # The hourly off-site copy. The bucket is checked now, in the install, so wrong -# credentials, an unreachable endpoint or a key other than the one its objects are sealed -# with show up here; the first copy itself runs in the background, since a host with many -# archives can take a long while to upload them. +# credentials, an unreachable endpoint, a key other than the one its objects are sealed +# with, or another host writing it show up here; the first copy itself runs in the +# background, since a host with many archives can take a long while to upload them. # With no bucket configured the timer is removed (the operator deleted [offsite]) and the # install says loudly that every backup is on this machine only. install_offsite_timer() { @@ -4590,8 +4590,40 @@ EOF "$HOST_BIN" offsite check-key -config "${STATE_DIR}/felis.host.toml" -env-file "$OFFSITE_ENV" >/dev/null || rc=$? case "$rc" in 0) + # A host built from another host's backup (a rehearsal on a spare machine, or a + # rebuild) finds that host named as the bucket's writer and stands by: its copy + # writes nothing there and its watchdog mails nobody while that host keeps writing. + # A host another one took the bucket over from (5) stops copying and mails its + # owners. The first copy still runs, to record either for the watchdog. + local who wrc=0 + who="$("$HOST_BIN" offsite take-over -config "${STATE_DIR}/felis.host.toml" -env-file "$OFFSITE_ENV")" || wrc=$? systemctl start --no-block felis-offsite.service - ok "off-site copy: hourly to the [offsite] bucket, first copy started (sudo felis offsite status; journalctl -u felis-offsite)" + case "$wrc" in + 0) ok "off-site copy: hourly to the [offsite] bucket, first copy started (sudo felis offsite status; journalctl -u felis-offsite)" ;; + 4) + OFFSITE_STANDBY=1 + warn "================================================================================" + warn "Another host writes the [offsite] bucket, and this host was built from its backup:" + warn " $(printf '%s\n' "$who" | head -n 1 | sed 's/^felis offsite take-over: //')" + warn "This host copies nothing into the bucket and, while that host keeps writing it," + warn "mails no watchdog alert: a rehearsal leaves that host and its owners alone. When" + warn "this host replaces it for good, run sudo felis offsite take-over -yes, then" + warn "sudo systemctl start felis-offsite.service (docs/troubleshooting.md §16)." + warn "================================================================================" + ;; + 5) + OFFSITE_DISPLACED=1 + warn "================================================================================" + warn "Another host took the [offsite] bucket over from this host:" + warn " $(printf '%s\n' "$who" | head -n 1 | sed 's/^felis offsite take-over: //')" + warn "This host copies nothing into the bucket any more, and its watchdog mails the" + warn "owners about it. If that host is a rehearsal machine, take the bucket back:" + warn "sudo felis offsite take-over -yes, then sudo systemctl start felis-offsite.service" + warn "(docs/troubleshooting.md §16)." + warn "================================================================================" + ;; + *) warn "could not tell which host writes the [offsite] bucket (error above); the first copy started and says what it found: journalctl -u felis-offsite" ;; + esac ;; 3) # The timer stays: every run is refused (and reported by the watchdog) until the @@ -4623,6 +4655,14 @@ summary_offsite() { warn "FELIS_OFFSITE_ACCESS_KEY and FELIS_OFFSITE_SECRET_KEY and re-run (docs/troubleshooting.md §16)." return 0 fi + if [ "${OFFSITE_DISPLACED:-0}" = 1 ]; then + warn "OFF-SITE COPY STOPPED: another host took the [offsite] bucket over (see above)." + warn "This host's backups stay on this machine until: sudo felis offsite take-over -yes" + fi + if [ "${OFFSITE_STANDBY:-0}" = 1 ]; then + warn "OFF-SITE COPY ON STANDBY: another host writes the [offsite] bucket (see above)." + warn "This host's backups stay on this machine until: sudo felis offsite take-over -yes" + fi if [ "${OFFSITE_KEY_MISMATCH:-0}" = 1 ]; then # A key the bucket refuses is no key to store; this run's is in OFFSITE_ENV if the # operator moves to an empty bucket instead. @@ -4824,6 +4864,8 @@ configure_offsite() { OFFSITE_ENABLED=0 OFFSITE_KEY_NEW=0 OFFSITE_KEY_MISMATCH=0 + OFFSITE_STANDBY=0 + OFFSITE_DISPLACED=0 offsite_enabled || return 0 OFFSITE_ENABLED=1 local env_ak="${FELIS_OFFSITE_ACCESS_KEY:-}" env_sk="${FELIS_OFFSITE_SECRET_KEY:-}" env_key="${FELIS_OFFSITE_KEY:-}" diff --git a/deploy/bootstrap_test.sh b/deploy/bootstrap_test.sh index b2802e8..6de6360 100644 --- a/deploy/bootstrap_test.sh +++ b/deploy/bootstrap_test.sh @@ -1983,7 +1983,7 @@ ofile="$(mktemp)" for fn in validate_offsite_settings persisted_offsite_block offsite_block offsite_enabled configure_offsite install_offsite_timer summary_offsite; do blk="$(awk "/^${fn}\\(\\) \\{/,/^}/" "$BS")" [ -n "$blk" ] || { echo "FAIL: no ${fn} found in $BS"; exit 1; } - [ "$(printf '%s\n' "$blk" | wc -l)" -lt 90 ] \ + [ "$(printf '%s\n' "$blk" | wc -l)" -lt 120 ] \ || { echo "FAIL: the extracted block is not ${fn} -- did its closing brace move?"; exit 1; } printf '%s\n' "$blk" >> "$ofile" done @@ -1998,7 +1998,12 @@ run_offsite() { # script; runs with the off-site functions sourced log() { printf "LOG: %s\n" "$*"; }; ok() { printf "OK: %s\n" "$*"; }; warn() { printf "WARN: %s\n" "$*"; } systemctl() { printf "SYSTEMCTL: %s\n" "$*" >&2; } fakefelis() { printf "RUN: %s\n" "$*" >&2; [ -z "${CHECK_FAILS:-}" ] || { echo "bucket: access denied" >&2; return 1; } - [ -z "${KEY_MISMATCH:-}" ] || { echo "the bucket records key id 1111111111111111, this key is 2222222222222222" >&2; return 3; }; } + [ -z "${KEY_MISMATCH:-}" ] || { echo "the bucket records key id 1111111111111111, this key is 2222222222222222" >&2; return 3; } + [ "$2" != take-over ] || [ -z "${STANDBY:-}" ] || { echo "felis offsite take-over: host prod-1 (id 3333333333333333) writes the bucket, last at 2026-09-27 07:00:00 (20m ago)" + echo "This host was built from its backup and copies nothing into the bucket."; return 4; } + [ "$2" != take-over ] || [ -z "${DISPLACED:-}" ] || { echo "felis offsite take-over: host spare-1 (id 4444444444444444) writes the bucket, last at 2026-09-27 07:10:00 (10m ago)" + echo "It took the bucket over from this host: this host copies nothing there any more, and its watchdog mails the owners about it."; return 5; } + [ "$2" != take-over ] || [ -z "${WRITER_FAILS:-}" ] || { echo "felis offsite take-over: offsite: felis-writer in the bucket is not a record Felis wrote" >&2; return 1; }; } FELIS_OFFSITE_ENDPOINT="${FELIS_OFFSITE_ENDPOINT:-}" FELIS_OFFSITE_BUCKET="${FELIS_OFFSITE_BUCKET:-}" FELIS_OFFSITE_REGION="${FELIS_OFFSITE_REGION:-}" FELIS_OFFSITE_PREFIX="${FELIS_OFFSITE_PREFIX:-}" FELIS_OFFSITE_DB_KEEP="${FELIS_OFFSITE_DB_KEEP:-}" @@ -2142,6 +2147,53 @@ esac out="$(OFFSITE_ENABLED=1 OFFSITE_KEY_NEW=1 FELIS_OFFSITE_KEY=NEWKEY run_offsite 'install_offsite_timer; summary_offsite')" expect "a key the bucket takes is shown once" "WARN: FELIS_OFFSITE_KEY=NEWKEY" "$out" +# A rehearsal on a spare machine restores the production host's [offsite] settings: the +# install must find the production host writing the bucket, say this host stands by, and +# still start the first copy so the watchdog learns it. +out="$(OFFSITE_ENABLED=1 run_offsite install_offsite_timer)" +expect "the install asks which host writes the bucket" "RUN: offsite take-over -config $odir/felis.host.toml -env-file $odir/offsite.env" "$out" +case "$out" in + *"take-over -yes"* | *"-config $odir/felis.host.toml -env-file $odir/offsite.env -yes"*) echo "FAIL the install took the bucket over: $out"; fails=$((fails + 1)) ;; + *) echo "PASS the install only asks, never takes the bucket over" ;; +esac +out="$(OFFSITE_ENABLED=1 STANDBY=1 run_offsite 'install_offsite_timer; summary_offsite')" +expect "a standby host names the writer" "WARN: host prod-1 (id 3333333333333333) writes the bucket, last at 2026-09-27 07:00:00 (20m ago)" "$out" +expect "a standby host is a loud warning" "WARN: Another host writes the [offsite] bucket, and this host was built from its backup:" "$out" +expect "a standby host says how to take over" "WARN: this host replaces it for good, run sudo felis offsite take-over -yes, then" "$out" +expect "a standby host still records itself through the first copy" "SYSTEMCTL: start --no-block felis-offsite.service" "$out" +expect "the summary says the copy stands by" "WARN: OFF-SITE COPY ON STANDBY: another host writes the [offsite] bucket (see above)." "$out" +expect "a standby host still says where its key is" "LOG: Off-site copy: the encryption key is in" "$out" +case "$out" in + *"OK: off-site copy: hourly"* | *"could not tell"* | *"OFF-SITE COPY STOPPED"*) echo "FAIL a standby host read as copying, as an error or as a key mismatch: $out"; fails=$((fails + 1)) ;; + *) echo "PASS a standby host reads as standing by only" ;; +esac +# A rehearsal machine that took the bucket over by mistake leaves the production host +# displaced: its re-run must say so, with how to take the bucket back, and never call it a +# standby, whose watchdog stays quiet. +out="$(OFFSITE_ENABLED=1 DISPLACED=1 run_offsite 'install_offsite_timer; summary_offsite')" +expect "a displaced host names the host that took over" "WARN: host spare-1 (id 4444444444444444) writes the bucket, last at 2026-09-27 07:10:00 (10m ago)" "$out" +expect "a displaced host is a loud warning" "WARN: Another host took the [offsite] bucket over from this host:" "$out" +expect "a displaced host says how to take the bucket back" "WARN: sudo felis offsite take-over -yes, then sudo systemctl start felis-offsite.service" "$out" +expect "the summary says the copy stopped" "WARN: OFF-SITE COPY STOPPED: another host took the [offsite] bucket over (see above)." "$out" +expect "a displaced host still says where its key is" "LOG: Off-site copy: the encryption key is in" "$out" +case "$out" in + *"OK: off-site copy: hourly"* | *"could not tell"* | *"ON STANDBY"* | *"built from its backup:"* | *"sealed with another key"*) echo "FAIL a displaced host read as copying, as an error, as a standby or as a key mismatch: $out"; fails=$((fails + 1)) ;; + *) echo "PASS a displaced host reads as taken over only" ;; +esac +out="$(OFFSITE_ENABLED=1 WRITER_FAILS=1 run_offsite 'install_offsite_timer; summary_offsite')" +expect "an unreadable writer record shows why" "felis-writer in the bucket is not a record Felis wrote" "$out" +expect "an unreadable writer record is a warning" "WARN: could not tell which host writes the [offsite] bucket (error above)" "$out" +case "$out" in + *"OK: off-site copy: hourly"* | *"ON STANDBY"*) echo "FAIL an unreadable writer record read as copying or standing by: $out"; fails=$((fails + 1)) ;; + *) echo "PASS an unreadable writer record reads as neither copying nor standing by" ;; +esac +out="$(OFFSITE_ENABLED=1 run_offsite 'install_offsite_timer; summary_offsite')" +expect "the host that writes the bucket copies" "OK: off-site copy: hourly to the [offsite] bucket, first copy started" "$out" +case "$out" in + *"ON STANDBY"* | *"Another host writes"*) echo "FAIL the writer was called a standby: $out"; fails=$((fails + 1)) ;; + *) echo "PASS the writer is not called a standby" ;; +esac + out="$(OFFSITE_ENABLED=0 run_offsite 'systemctl() { printf "SYSTEMCTL: %s\n" "$*" >> "$STATE_DIR/systemctl.log"; }; install_offsite_timer')" expect "removing [offsite] disables the timer" "SYSTEMCTL: disable --now felis-offsite.timer" "$(cat "$odir/systemctl.log")" expect "removing [offsite] says so" "WARN: no [offsite] bucket is configured any more" "$out" diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index cf9e34b..66c6690 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1548,6 +1548,13 @@ How it mails: delay is never mailed; one that turns critical is mailed again at once. - **During an install** nothing is mailed. `bootstrap.sh` writes `/run/felis/watchdog-quiet-until` and removes it when it exits. +- **On a host built from another host's backup** (a rehearsal on a spare + machine, a rebuild before `felis offsite take-over`) nothing is mailed while + the host the off-site bucket names ran in the last 3 hours: the owners in + the restored database are that host's, and it mails them itself. The run + logs `this host stands by for host ...; holding this mail`. Once that host + stops running, the standby is mailed like any other finding (§16). + [GO-TESTED: `TestMailHold`] - **Caching:** the relay password comes from `/etc/felis/smtp-password`, which the watchdog reads even while the API server is down (an install without that file reads the `felis-smtp` Secret instead). It and the recipient list are @@ -2120,7 +2127,11 @@ bucket holding (images, uploads, world archives) at the bucket's bandwidth, and each skips what is already in place, so an interrupted one resumes. Write those sizes down with the bucket's download rate and you have the recovery time for your install. A whole-host rehearsal on a spare machine, once per -release, is the way to know it for sure. +release, is the way to know it for sure. The spare follows the steps below +without step 8: it finds the production host named in the bucket and stands +by, so it copies nothing into the bucket, prunes nothing there and, while +production keeps writing, mails none of the owners its restored database +holds. The order below matters: the state goes in before the installer so it reuses the old secrets and bucket; the images go back before the database so the @@ -2154,8 +2165,9 @@ host yourself, plus the off-site encryption key if the copy is in the bucket. `latest` is the newest bundle, unless that one holds no servers and at most one account while an older one holds more. That is the database a - rebuilt host backs up and copies off-site within its first hour, before - anyone restores onto it, so `fetch-db latest` refuses it and names up to + rebuilt host copies off-site when it takes the bucket over before anyone + restores onto it (with an older release, within its first hour), so + `fetch-db latest` refuses it and names up to three older bundles with their counts; fetch the one you want by name in place of `latest` (`felis offsite list` shows them all, each with its counts). A bundle fetched by name that looks like a new install's is @@ -2174,7 +2186,10 @@ host yourself, plus the off-site encryption key if the copy is in the bucket. 3. Run the installer as for a first install. `bootstrap.done` is not in the bundle, so it takes the fresh-install path, creates the empty database with the restored password and migrates it. It finds `[offsite]` in the restored - `felis.host.toml` and turns the hourly copy back on. + `felis.host.toml`, turns the hourly copy back on and reports that another + host writes the bucket: this host was built from its backup, so it stands + by and copies nothing there until step 8, and ends with `OFF-SITE COPY ON + STANDBY`. 4. Push the user images back into the new registry: ``` @@ -2187,8 +2202,8 @@ host yourself, plus the off-site encryption key if the copy is in the bucket. its digest, and pushes only what the registry lacks. The pruner counts a restored image as freshly pushed and keeps it for 24 hours; finish the database step within that window so the restored servers and whitelist - entries keep naming it. When the new host's hourly copy has already recorded - its still-empty registry, `fetch-images` refuses that newest list and names + entries keep naming it. When the new host has already taken the bucket over + and recorded its still-empty registry, `fetch-images` refuses that newest list and names the version to pass with `-at`; `fetch-uploads` does the same. 5. Put the submission uploads back: @@ -2221,6 +2236,18 @@ host yourself, plus the off-site encryption key if the copy is in the bucket. nothing has used it yet (a short-lived `felis-bind-felis-backups-*` pod). It lists any it could not find in the bucket. Restore a world from its archive as usual (§10, §13). +8. Make this host the one that writes the bucket, and send its first copy: + + ``` + sudo felis offsite take-over # names the host the bucket names now + sudo felis offsite take-over -yes + sudo systemctl start felis-offsite.service + ``` + + Without `-yes` it names the host the bucket records and when that host last + wrote it, and exits 4. With `-yes` it records this host; the old host, if it + ever runs again, copies nothing more and says it was taken over. Skip this + step on a rehearsal machine. Check the rebuild before letting players in: @@ -2282,6 +2309,27 @@ empty bucket or prefix and re-run the installer. [GO-TESTED: `TestSyncRefusesAnotherKeysBucket`, `TestCheckKey`] [SH-TESTED] [VM-TESTED: MinIO, the other key's sync refused with the bucket unchanged, check-key 0/3, fetch-db naming both ids] +The bucket also names the host that writes it: `felis-writer`, rewritten by +every sync, holds that host's id (kept in `/var/lib/felis/offsite/host-id`, +which no bundle carries), its name and the time of its last run. A host +restored from a bundle has the bucket's key and credentials but no id, so it +finds another host named there and stands by: its syncs copy and prune +nothing, `felis offsite status` names the host that writes the bucket and exits +1, and while that host ran in the last 3 hours the watchdog holds this host's +mail (§14). Once it has not run for 3 hours, the watchdog mails this host's +owners that it copies nothing into the bucket. A bucket that holds sealed +objects and names no writer puts a host on standby too, except a host that +copied to it with a release before writers were recorded, which claims it on +its next sync. `sudo felis offsite take-over` names the host that writes the +bucket; with `-yes` it records this host instead (step 8 of a rebuild). The +host it replaced copies nothing from its next sync on: its `status` exits 1, +and its watchdog mails its owners at once that `the off-site copy has +stopped`. When that happened by mistake (a rehearsal machine took the bucket +over), run `sudo felis offsite take-over -yes` on the production host to take +it back. [GO-TESTED: `TestLeasePlan`, `TestSyncStandsBy`, `TestOffsiteTakeOver`, +`TestRestoredHostKeepsStandingBy`] [SH-TESTED] +[VM-TESTED: MinIO, a restored host standing by with the bucket unchanged, take-over, the old host displaced and taking it back, an older release's host claiming its prefix] + What runs: - **`felis-offsite.timer`** runs `felis offsite sync` hourly (plus up to @@ -2322,7 +2370,8 @@ What runs: `registry/index/.json.fenc`, `uploads/blobs/.fenc` and `uploads/index/.json.fenc`: AES-256-GCM in 64 KiB segments, so truncation, reordering and a wrong key are all refused on the way back. - `felis-key-id` next to them holds the key's id in the clear. + `felis-key-id` and `felis-writer` next to them hold the key's id and the + host writing the bucket in the clear. - A pass sends the database bundles first, then world archives, images and uploads. Each object has its own time limit: 10 minutes plus its size at 512 KiB/s (about 6 hours for 10 GiB). An archive the uplink cannot send in @@ -2333,7 +2382,8 @@ What runs: - The reaper deletes an idle world only after its archive is in the bucket (§10). - The watchdog mails the owners when no sync has completed for 12 hours - (`the off-site copy last completed ... ago`). + (`the off-site copy last completed ... ago`), and at once when the bucket + refuses this host's key or another host took the bucket over. Checking it: diff --git a/internal/offsite/keymark.go b/internal/offsite/keymark.go index 6c4fa16..6de4445 100644 --- a/internal/offsite/keymark.go +++ b/internal/offsite/keymark.go @@ -8,6 +8,7 @@ import ( "regexp" "sort" "strings" + "time" ) // keyMark is the one object the bucket holds in the clear: the KeyID of the @@ -137,13 +138,24 @@ func CheckKey(ctx context.Context, b Bucket, key []byte) (KeyFit, error) { return KeyUnused, nil } -// ClaimKey is CheckKey, then records key's id in a bucket that has none, so -// that every later check reads the id. -func ClaimKey(ctx context.Context, b Bucket, key []byte) error { +// claim is the check before a run writes anything: CheckKey, then the lease +// (nil checks none), then key's id recorded in a bucket that has none, so +// that every later check reads the id. The key goes first, so a host with the +// wrong key never records itself as the writer, and the lease before the key +// id, so a standby host writes nothing at all. +func claim(ctx context.Context, b Bucket, key []byte, lease *Lease, now time.Time) error { fit, err := CheckKey(ctx, b, key) - if err != nil || fit == KeyRecorded { + if err != nil { return err } + if lease != nil { + if err := lease.Acquire(ctx, b, fit == KeyUnused, now); err != nil { + return err + } + } + if fit == KeyRecorded { + return nil + } id := KeyID(key) + "\n" if err := b.Put(ctx, keyMark, strings.NewReader(id), int64(len(id))); err != nil { return fmt.Errorf("record the key id in %s: %w", keyMark, err) diff --git a/internal/offsite/keymark_test.go b/internal/offsite/keymark_test.go index b831b5e..a948baf 100644 --- a/internal/offsite/keymark_test.go +++ b/internal/offsite/keymark_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "os" "strings" "testing" "time" @@ -160,36 +161,36 @@ func TestCheckKey(t *testing.T) { } } -func TestClaimKey(t *testing.T) { +func TestClaim(t *testing.T) { key, other := testKey(t), testKey(t) marker := func(b *memBucket) string { return string(b.objs[keyMark]) } b := newMemBucket() - if err := ClaimKey(context.Background(), b, key); err != nil { + if err := claim(context.Background(), b, key, nil, time.Time{}); err != nil { t.Fatal(err) } if marker(b) != KeyID(key)+"\n" { t.Fatalf("empty bucket: marker = %q, want %s", marker(b), KeyID(key)) } - if err := ClaimKey(context.Background(), b, key); err != nil || b.puts != 1 { + if err := claim(context.Background(), b, key, nil, time.Time{}); err != nil || b.puts != 1 { t.Errorf("second claim: err %v, puts %d; want the marker written once", err, b.puts) } if fit, err := CheckKey(context.Background(), b, key); fit != KeyRecorded || err != nil { t.Errorf("after the claim: CheckKey = %v, %v", fit, err) } - if err := ClaimKey(context.Background(), b, other); !errors.Is(err, ErrKeyMismatch) || marker(b) != KeyID(key)+"\n" { + if err := claim(context.Background(), b, other, nil, time.Time{}); !errors.Is(err, ErrKeyMismatch) || marker(b) != KeyID(key)+"\n" { t.Errorf("another key: err %v, marker %q; want a refusal that leaves the marker", err, marker(b)) } b = newMemBucket() putAt(b, "worlds/1.fenc", seal(t, []byte("1"), key), 0) - if err := ClaimKey(context.Background(), b, key); err != nil || marker(b) != KeyID(key)+"\n" { + if err := claim(context.Background(), b, key, nil, time.Time{}); err != nil || marker(b) != KeyID(key)+"\n" { t.Errorf("unmarked bucket the key opens: err %v, marker %q", err, marker(b)) } b = newMemBucket() putAt(b, "worlds/1.fenc", seal(t, []byte("1"), other), 0) - if err := ClaimKey(context.Background(), b, key); !errors.Is(err, ErrKeyMismatch) || b.puts != 0 { + if err := claim(context.Background(), b, key, nil, time.Time{}); !errors.Is(err, ErrKeyMismatch) || b.puts != 0 { t.Errorf("unmarked bucket under another key: err %v, puts %d; want a refusal that writes nothing", err, b.puts) } } @@ -214,6 +215,9 @@ func TestSyncRefusesAnotherKeysBucket(t *testing.T) { for i, name := range []string{"felis-db-20260901T030000Z-daily.tar", "felis-db-20260902T030000Z-daily.tar", "felis-db-20260903T030000Z-daily.tar"} { putAt(b, DBKey(name), seal(t, []byte(name), other), i) } + // A new host: the wrong key must not record it as the writer either. + l := testLease(t, "") + s.Lease = &l before := len(b.objs) _, err := s.Run(context.Background()) @@ -226,6 +230,9 @@ func TestSyncRefusesAnotherKeysBucket(t *testing.T) { if !cat.rows[0].offsite.IsZero() { t.Error("the refused run recorded alpha as copied") } + if _, err := os.Stat(l.IDFile); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the refused run made this host an id: %v", err) + } }) } } diff --git a/internal/offsite/status.go b/internal/offsite/status.go index 3a6b174..fd614cd 100644 --- a/internal/offsite/status.go +++ b/internal/offsite/status.go @@ -32,6 +32,46 @@ type Status struct { // with another key (ErrKeyMismatch): no later run copies anything until // the key is fixed, so the watchdog reports it at once. KeyMismatch bool `json:"key_mismatch,omitempty"` + // Standby and Displaced are a run refused because another host, Writer, + // writes the bucket (ErrStandby, ErrDisplaced). + Standby bool `json:"standby,omitempty"` + Displaced bool `json:"displaced,omitempty"` + Writer *Writer `json:"writer,omitempty"` + // Format is StatusFormat in every record this release writes; 0 is a + // record from before writers were recorded, whose host had been copying + // to the bucket (Lease.Inherited). + Format int `json:"format,omitempty"` + // Inherited carries Lease.Inherited over runs that ended before the host + // recorded itself (an unreachable bucket on the first run after the + // upgrade), until it has an id. + Inherited bool `json:"inherited,omitempty"` +} + +// StatusFormat marks a status record that knows about felis-writer. +const StatusFormat = 2 + +// StandsBy is the host this one stands by for: the last run was refused +// because that host writes the bucket, and it wrote it within WriterLive of +// now. While it keeps writing, this host is a rehearsal (or a rebuild not yet +// taken over), and the owners in its restored database are that host's: the +// watchdog here mails them nothing. +func (st *Status) StandsBy(now time.Time) *Writer { + if st == nil || !st.Standby || st.Writer == nil || now.Sub(st.Writer.At) > WriterLive { + return nil + } + return st.Writer +} + +// HostLease is the Lease of the host whose status file is statusFile: its id +// next to it, and Inherited when that file was written by an older release +// (or carries Inherited from one). +func HostLease(statusFile string) Lease { + host, _ := os.Hostname() + l := Lease{IDFile: filepath.Join(filepath.Dir(statusFile), HostIDFile), Host: host} + if prev, err := ReadStatus(statusFile); err == nil && prev != nil && (prev.Format == 0 || prev.Inherited) { + l.Inherited = true + } + return l } // ReadStatus reads the status file. A missing file is (nil, nil): no sync has diff --git a/internal/offsite/sync.go b/internal/offsite/sync.go index d468d62..915f921 100644 --- a/internal/offsite/sync.go +++ b/internal/offsite/sync.go @@ -104,6 +104,8 @@ type Syncer struct { UploadsDir string // UploadGrace and MinRate bound one object's upload: UploadGrace plus the // time the object takes at MinRate bytes a second. Zero takes the defaults. + // Lease names this host in felis-writer (writer.go); nil checks no writer. + Lease *Lease UploadGrace time.Duration MinRate int64 Now func() time.Time @@ -201,8 +203,9 @@ func (s *Syncer) Run(ctx context.Context) (Result, error) { } // Before anything is written or pruned: objects sealed with another key - // are copies only that key opens, and DBKeep would prune them. - if err := ClaimKey(ctx, s.Bucket, s.Key); err != nil { + // are copies only that key opens, and DBKeep would prune them; a bucket + // another host writes is that host's to prune. + if err := claim(ctx, s.Bucket, s.Key, s.Lease, s.now()); err != nil { return res, err } remoteWorlds, err := s.listSizes(ctx, worldsDir) diff --git a/internal/offsite/writer.go b/internal/offsite/writer.go new file mode 100644 index 0000000..d350387 --- /dev/null +++ b/internal/offsite/writer.go @@ -0,0 +1,263 @@ +package offsite + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + "unicode" +) + +// writerMark names the host that writes the bucket. A rehearsal on a spare +// machine restores the production host's /etc/felis, [offsite] and its key +// included: without the record the spare would copy its bundles into the +// production prefix and prune the production host's by DBKeep. The writer +// rewrites it on every run; a host restored from its backup finds another +// host named there and stands by, writing nothing, until `felis offsite +// take-over`. +const writerMark = "felis-writer" + +// WriterLive is how recently the writer must have run for a standby host to +// count it as alive. Both run hourly, so a writer seen within it is one that +// ran in the last two hours. +const WriterLive = 3 * time.Hour + +// HostIDFile is the file, next to the status file, holding this host's id. It +// is written when the host first writes the bucket and never leaves the host +// (a bundle carries /etc/felis only), so a host restored from a bundle has +// none. +const HostIDFile = "host-id" + +var ( + // ErrStandby is a run refused because another host writes the bucket and + // this one never has. + ErrStandby = errors.New("offsite: another host writes this bucket") + // ErrDisplaced is a run refused because another host took the bucket + // over from this one. + ErrDisplaced = errors.New("offsite: another host took this bucket over") +) + +const takeOverHint = "sudo felis offsite take-over -yes (docs/troubleshooting.md §16)" + +// Writer is the felis-writer record. +type Writer struct { + HostID string `json:"host_id"` + Host string `json:"host"` + At time.Time `json:"at"` +} + +func (w *Writer) String() string { + return fmt.Sprintf("host %s (id %s)", w.Host, w.HostID) +} + +// WriterError is a run refused because another host writes the bucket. +type WriterError struct { + // Kind is ErrStandby or ErrDisplaced. + Kind error + // Writer is the host named in the bucket; nil for a bucket that holds + // another host's copies and names no writer. + Writer *Writer +} + +func (e *WriterError) Error() string { + switch { + case e.Kind == ErrDisplaced: + return fmt.Sprintf("%v: %s writes it now (last at %s), and this host copies nothing there any more; if that host is a rehearsal machine, take the bucket back here: %s", + e.Kind, e.Writer, e.Writer.At.UTC().Format(time.RFC3339), takeOverHint) + case e.Writer != nil: + return fmt.Sprintf("%v: %s writes it (last at %s); this host was built from its backup and copies nothing there, until it replaces that host for good: %s", + e.Kind, e.Writer, e.Writer.At.UTC().Format(time.RFC3339), takeOverHint) + default: + return fmt.Sprintf("%v: the bucket holds copies this host did not write and names no host writing it; this host copies nothing there, until it replaces that host for good: %s", + e.Kind, takeOverHint) + } +} + +func (e *WriterError) Unwrap() error { return e.Kind } + +// Role is what a host's next run does with the bucket. +type Role int + +const ( + // RoleWrites: the bucket names this host. + RoleWrites Role = iota + 1 + // RoleClaims: the bucket names no host, and this one records itself. + RoleClaims + // RoleStandby: another host writes the bucket, and this one never has. + RoleStandby + // RoleDisplaced: another host took the bucket over from this one. + RoleDisplaced +) + +// Lease is this host's side of felis-writer. +type Lease struct { + // IDFile is this host's id (HostIDFile next to the status file). + IDFile string + // Host is this host's name, shown to the others. + Host string + // Inherited is a host that copied to the bucket before writers were + // recorded: a status file an older release wrote. It claims a bucket + // that names no writer. + Inherited bool +} + +// BucketWriter reads the felis-writer record, nil when there is none. +func BucketWriter(ctx context.Context, b Bucket) (*Writer, error) { + rc, err := b.Get(ctx, writerMark) + if errors.Is(err, ErrNotFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read %s: %w", writerMark, err) + } + defer rc.Close() + raw, err := io.ReadAll(io.LimitReader(rc, 1024)) + if err != nil { + return nil, fmt.Errorf("read %s: %w", writerMark, err) + } + var w Writer + if json.Unmarshal(raw, &w) != nil || !keyIDPattern.MatchString(w.HostID) { + return nil, fmt.Errorf("offsite: %s in the bucket is not a record Felis wrote", writerMark) + } + w.Host = printable(w.Host) + return &w, nil +} + +// Plan says what this host's next run does with the bucket, and the host the +// bucket names (nil for none). empty is a bucket holding no sealed object +// (CheckKey's KeyUnused), which any host may claim. +func (l Lease) Plan(ctx context.Context, b Bucket, empty bool) (Role, *Writer, error) { + w, err := BucketWriter(ctx, b) + if err != nil { + return 0, nil, err + } + mine, err := l.ID() + if err != nil { + return 0, nil, err + } + switch { + case w != nil && w.HostID == mine: + return RoleWrites, w, nil + case w != nil && mine != "": + return RoleDisplaced, w, nil + case w != nil: + return RoleStandby, w, nil + case mine != "" || l.Inherited || empty: + return RoleClaims, nil, nil + default: + return RoleStandby, nil, nil + } +} + +// Acquire is Plan before a run writes anything: a host that writes or claims +// the bucket records itself there at now; one that stands by or was displaced +// gets a *WriterError and writes nothing. +func (l Lease) Acquire(ctx context.Context, b Bucket, empty bool, now time.Time) error { + role, w, err := l.Plan(ctx, b, empty) + if err != nil { + return err + } + switch role { + case RoleStandby: + return &WriterError{Kind: ErrStandby, Writer: w} + case RoleDisplaced: + return &WriterError{Kind: ErrDisplaced, Writer: w} + } + return l.record(ctx, b, now) +} + +// TakeOver records this host as the bucket's writer whatever the bucket named, +// and returns the writer it replaced (nil for none). That host's next run is +// refused with ErrDisplaced. +func (l Lease) TakeOver(ctx context.Context, b Bucket, now time.Time) (*Writer, error) { + prev, err := BucketWriter(ctx, b) + if err != nil { + return nil, err + } + return prev, l.record(ctx, b, now) +} + +// record writes this host into felis-writer, creating its id first: a host +// whose id is on disk but not in the bucket claims the bucket on its next run, +// where one named in the bucket without an id on disk would stand by for +// itself. +func (l Lease) record(ctx context.Context, b Bucket, now time.Time) error { + id, err := l.ID() + if err != nil { + return err + } + if id == "" { + if id, err = l.newID(); err != nil { + return err + } + } + raw, err := json.Marshal(Writer{HostID: id, Host: printable(l.Host), At: now.UTC()}) + if err != nil { + return err + } + raw = append(raw, '\n') + if err := b.Put(ctx, writerMark, strings.NewReader(string(raw)), int64(len(raw))); err != nil { + return fmt.Errorf("record this host in %s: %w", writerMark, err) + } + return nil +} + +// ID is this host's id, "" when it has never written a bucket. +func (l Lease) ID() (string, error) { + raw, err := os.ReadFile(l.IDFile) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("read this host's id: %w", err) + } + id := strings.TrimSpace(string(raw)) + if !keyIDPattern.MatchString(id) { + return "", fmt.Errorf("offsite: %s is not a host id Felis wrote; remove it and run sudo felis offsite take-over -yes", l.IDFile) + } + return id, nil +} + +func (l Lease) newID() (string, error) { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + id := hex.EncodeToString(b[:]) + if err := os.MkdirAll(filepath.Dir(l.IDFile), 0o700); err != nil { + return "", fmt.Errorf("record this host's id: %w", err) + } + tmp := l.IDFile + ".tmp" + if err := os.WriteFile(tmp, []byte(id+"\n"), 0o600); err != nil { + return "", fmt.Errorf("record this host's id: %w", err) + } + if err := os.Rename(tmp, l.IDFile); err != nil { + return "", fmt.Errorf("record this host's id: %w", err) + } + return id, nil +} + +// printable keeps a host name fit for a message: at most 64 printable runes, +// "unknown" for none. +func printable(s string) string { + s = strings.Map(func(r rune) rune { + if unicode.IsPrint(r) { + return r + } + return -1 + }, s) + if r := []rune(s); len(r) > 64 { + s = string(r[:64]) + } + if strings.TrimSpace(s) == "" { + return "unknown" + } + return s +} diff --git a/internal/offsite/writer_test.go b/internal/offsite/writer_test.go new file mode 100644 index 0000000..7193db5 --- /dev/null +++ b/internal/offsite/writer_test.go @@ -0,0 +1,313 @@ +package offsite + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +const ( + myID = "aaaaaaaaaaaaaaaa" + otherID = "bbbbbbbbbbbbbbbb" +) + +func putWriter(t *testing.T, b *memBucket, id, host string, at time.Time) { + t.Helper() + raw, err := json.Marshal(Writer{HostID: id, Host: host, At: at}) + if err != nil { + t.Fatal(err) + } + b.objs[writerMark] = raw +} + +// testLease is a lease whose id file is in a temp dir, holding id unless it +// is "". +func testLease(t *testing.T, id string) Lease { + t.Helper() + l := Lease{IDFile: filepath.Join(t.TempDir(), HostIDFile), Host: "spare-1"} + if id != "" { + if err := os.WriteFile(l.IDFile, []byte(id+"\n"), 0o600); err != nil { + t.Fatal(err) + } + } + return l +} + +func TestLeasePlan(t *testing.T) { + at := now.Add(-20 * time.Minute) + for _, tc := range []struct { + what string + mine string + inherited bool + writer string // the id felis-writer names, "" for none + empty bool + want Role + }{ + {"an empty bucket, a new host", "", false, "", true, RoleClaims}, + {"another host's copies, no writer named, a new host", "", false, "", false, RoleStandby}, + {"another host's copies, no writer named, a host that copied before writers were recorded", "", true, "", false, RoleClaims}, + {"no writer named, a host that wrote before", myID, false, "", false, RoleClaims}, + {"this host named", myID, false, myID, false, RoleWrites}, + {"another host named, a host that wrote before", myID, false, otherID, false, RoleDisplaced}, + {"another host named, a new host", "", false, otherID, false, RoleStandby}, + {"another host named, a host that copied before writers were recorded", "", true, otherID, false, RoleStandby}, + {"another host named over an empty bucket", "", false, otherID, true, RoleStandby}, + } { + t.Run(tc.what, func(t *testing.T) { + b := newMemBucket() + if tc.writer != "" { + putWriter(t, b, tc.writer, "prod-1", at) + } + l := testLease(t, tc.mine) + l.Inherited = tc.inherited + role, w, err := l.Plan(context.Background(), b, tc.empty) + if err != nil || role != tc.want { + t.Fatalf("Plan = %v, %v; want %v", role, err, tc.want) + } + if (w != nil) != (tc.writer != "") || (w != nil && (w.HostID != tc.writer || w.Host != "prod-1" || !w.At.Equal(at))) { + t.Errorf("Plan named %+v, want the bucket's writer %q", w, tc.writer) + } + if b.puts != 0 { + t.Errorf("Plan wrote %d objects", b.puts) + } + }) + } + + b := newMemBucket() + b.objs[writerMark] = []byte("hello") + if _, _, err := testLease(t, "").Plan(context.Background(), b, true); err == nil || !strings.Contains(err.Error(), "not a record Felis wrote") { + t.Errorf("a record Felis did not write: err = %v", err) + } + putWriter(t, b, "../../etc", "prod-1", at) + if _, _, err := testLease(t, "").Plan(context.Background(), b, true); err == nil { + t.Error("a record with an id Felis does not make was taken") + } + b = newMemBucket() + b.getErr = map[string]error{writerMark: errors.New("connection reset")} + if _, _, err := testLease(t, "").Plan(context.Background(), b, true); err == nil || !strings.Contains(err.Error(), "connection reset") { + t.Errorf("an unreadable record: err = %v, want the read error", err) + } + b = newMemBucket() + if _, _, err := testLease(t, "not-an-id").Plan(context.Background(), b, true); err == nil || !strings.Contains(err.Error(), "not a host id Felis wrote") { + t.Errorf("a damaged id file: err = %v", err) + } +} + +func TestLeaseAcquire(t *testing.T) { + ctx := context.Background() + + // A new host claims an empty bucket: its id is created and recorded. + b := newMemBucket() + l := testLease(t, "") + if err := l.Acquire(ctx, b, true, now); err != nil { + t.Fatal(err) + } + id, err := l.ID() + if err != nil || !keyIDPattern.MatchString(id) { + t.Fatalf("id after the claim = %q, %v", id, err) + } + if fi, err := os.Stat(l.IDFile); err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("id file: %v, %v", fi, err) + } + w, err := BucketWriter(ctx, b) + if err != nil || w == nil || w.HostID != id || w.Host != "spare-1" || !w.At.Equal(now) { + t.Fatalf("record after the claim = %+v, %v", w, err) + } + + // Its next run keeps the id and moves the time on. + later := now.Add(time.Hour) + if err := l.Acquire(ctx, b, false, later); err != nil { + t.Fatal(err) + } + if w, _ := BucketWriter(ctx, b); w.HostID != id || !w.At.Equal(later) { + t.Errorf("record after the next run = %+v, want %s at %s", w, id, later) + } + + // A host restored from its backup stands by: nothing written, no id made. + spare := testLease(t, "") + puts := b.puts + err = spare.Acquire(ctx, b, false, later) + var we *WriterError + if !errors.Is(err, ErrStandby) || !errors.As(err, &we) || we.Writer == nil || we.Writer.HostID != id { + t.Fatalf("spare: err = %v, want ErrStandby naming %s", err, id) + } + if b.puts != puts { + t.Error("the standby host wrote to the bucket") + } + if _, err := os.Stat(spare.IDFile); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the standby host made an id: %v", err) + } + + // The spare takes over; the first host is displaced. + prev, err := spare.TakeOver(ctx, b, later) + if err != nil || prev == nil || prev.HostID != id { + t.Fatalf("TakeOver = %+v, %v; want the first host replaced", prev, err) + } + spareID, _ := spare.ID() + if spareID == "" || spareID == id { + t.Fatalf("spare id after the take-over = %q", spareID) + } + puts = b.puts + err = l.Acquire(ctx, b, false, later) + if !errors.Is(err, ErrDisplaced) || !errors.As(err, &we) || we.Writer.HostID != spareID { + t.Fatalf("first host after the take-over: err = %v, want ErrDisplaced naming %s", err, spareID) + } + if b.puts != puts { + t.Error("the displaced host wrote to the bucket") + } + + // A host name is kept printable and short for the messages that show it. + b = newMemBucket() + l = testLease(t, "") + l.Host = "evil\x1b[2J\n" + strings.Repeat("x", 80) + if err := l.Acquire(ctx, b, true, now); err != nil { + t.Fatal(err) + } + if w, _ := BucketWriter(ctx, b); w.Host != "evil[2J"+strings.Repeat("x", 57) { + t.Errorf("recorded host = %q", w.Host) + } + + // A record that cannot be written fails the run. + b = newMemBucket() + b.putErr[writerMark] = errors.New("access denied") + if err := testLease(t, "").Acquire(ctx, b, true, now); err == nil || !strings.Contains(err.Error(), "access denied") { + t.Errorf("unwritable record: err = %v", err) + } +} + +func TestWriterErrorSays(t *testing.T) { + w := &Writer{HostID: otherID, Host: "prod-1", At: now} + for _, tc := range []struct { + err *WriterError + kind error + says []string + }{ + {&WriterError{Kind: ErrStandby, Writer: w}, ErrStandby, []string{"host prod-1 (id " + otherID + ")", "2026-09-24T12:00:00Z", "built from its backup", "take-over -yes"}}, + {&WriterError{Kind: ErrStandby}, ErrStandby, []string{"names no host writing it", "take-over -yes"}}, + {&WriterError{Kind: ErrDisplaced, Writer: w}, ErrDisplaced, []string{"host prod-1 (id " + otherID + ") writes it now", "rehearsal machine", "take-over -yes"}}, + } { + msg := tc.err.Error() + if !errors.Is(tc.err, tc.kind) { + t.Errorf("%q is not %v", msg, tc.kind) + } + for _, s := range tc.says { + if !strings.Contains(msg, s) { + t.Errorf("%q lacks %q", msg, s) + } + } + } +} + +func TestStandsBy(t *testing.T) { + w := &Writer{HostID: otherID, Host: "prod-1", At: now.Add(-2 * time.Hour)} + for _, tc := range []struct { + what string + st *Status + at time.Time + want bool + }{ + {"a standby run, the writer seen two hours ago", &Status{Standby: true, Writer: w}, now, true}, + {"a standby run, the writer gone quiet", &Status{Standby: true, Writer: w}, now.Add(WriterLive), false}, + {"a standby run, no writer named", &Status{Standby: true}, now, false}, + {"a displaced run", &Status{Displaced: true, Writer: w}, now, false}, + {"a run that copied", &Status{Writer: w}, now, false}, + {"no run yet", nil, now, false}, + } { + if got := tc.st.StandsBy(tc.at); (got != nil) != tc.want { + t.Errorf("%s: StandsBy = %+v, want %v", tc.what, got, tc.want) + } + } +} + +func TestHostLease(t *testing.T) { + dir := t.TempDir() + status := filepath.Join(dir, "status.json") + if l := HostLease(status); l.IDFile != filepath.Join(dir, HostIDFile) || l.Inherited || l.Host == "" { + t.Errorf("no status yet: %+v", l) + } + for _, tc := range []struct { + what string + st Status + want bool + }{ + {"a status an older release wrote", Status{LastAttempt: now}, true}, + {"a status this release wrote", Status{LastAttempt: now, Format: StatusFormat}, false}, + {"a status that carries the older release's claim", Status{LastAttempt: now, Format: StatusFormat, Inherited: true}, true}, + } { + if err := WriteStatus(status, tc.st); err != nil { + t.Fatal(err) + } + if got := HostLease(status).Inherited; got != tc.want { + t.Errorf("%s: Inherited = %v, want %v", tc.what, got, tc.want) + } + } + if err := os.WriteFile(status, []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + if HostLease(status).Inherited { + t.Error("an unreadable status counted as an older release's") + } +} + +// TestSyncStandsBy: a host restored from the writer's backup copies nothing +// into the bucket, prunes nothing, and records nothing as copied, whether the +// bucket names that host or holds its copies without naming anyone. +func TestSyncStandsBy(t *testing.T) { + for _, named := range []bool{true, false} { + t.Run(fmt.Sprintf("named=%v", named), func(t *testing.T) { + cat := &fakeCatalog{rows: []*row{ + {WorldBackup: WorldBackup{ID: "b1", Server: "alpha", Ref: "/a/alpha-1.tar.gz"}, status: "present"}, + }} + s, b := newSyncer(t, cat) + delete(b.objs, keyMark) + if named { + putWriter(t, b, otherID, "prod-1", now.Add(-30*time.Minute)) + } + writeFile(t, s.ArchiveDir, "alpha-1.tar.gz", 100) + writeFile(t, s.DBDir, "felis-db-20260924T030000Z-daily.tar", 50) + for i, name := range []string{"felis-db-20260901T030000Z-daily.tar", "felis-db-20260902T030000Z-daily.tar", "felis-db-20260903T030000Z-daily.tar"} { + putAt(b, DBKey(name), seal(t, []byte(name), s.Key), i) + } + l := testLease(t, "") + s.Lease = &l + before := len(b.objs) + + _, err := s.Run(context.Background()) + if !errors.Is(err, ErrStandby) { + t.Fatalf("Run error = %v, want ErrStandby", err) + } + if len(b.started) != 0 || len(b.removed) != 0 || len(b.objs) != before { + t.Errorf("the standby run began puts %v and removed %v", b.started, b.removed) + } + if !cat.rows[0].offsite.IsZero() { + t.Error("the standby run recorded alpha as copied") + } + }) + } +} + +// TestSyncRecordsTheWriter: the first run records this host before the key +// id and the first upload. +func TestSyncRecordsTheWriter(t *testing.T) { + s, b := newSyncer(t, &fakeCatalog{}) + delete(b.objs, keyMark) + writeFile(t, s.DBDir, "felis-db-20260924T030000Z-daily.tar", 50) + l := testLease(t, "") + s.Lease = &l + if _, err := s.Run(context.Background()); err != nil { + t.Fatal(err) + } + id, _ := l.ID() + if w, err := BucketWriter(context.Background(), b); err != nil || w == nil || w.HostID != id { + t.Fatalf("record = %+v, %v; want %s", w, err, id) + } + if len(b.started) != 3 || b.started[0] != writerMark || b.started[1] != keyMark { + t.Errorf("puts began in the order %v, want the writer, the key id, then the bundle", b.started) + } +} diff --git a/internal/watchdog/probes.go b/internal/watchdog/probes.go index a2dc404..919057d 100644 --- a/internal/watchdog/probes.go +++ b/internal/watchdog/probes.go @@ -368,6 +368,28 @@ func OffsiteFinding(statusFile string, now time.Time) *Finding { Hint: "`sudo felis offsite check-key`; set FELIS_OFFSITE_KEY in /etc/felis/offsite.env to the key the bucket was written with (docs/troubleshooting.md §16)", } } + if st != nil && st.Displaced && st.Writer != nil { + return &Finding{ + Key: "offsite", Severity: Warning, For: backupFor, + Summary: fmt.Sprintf("异地备份已停止：主机 %s（id %s）接管了异地备份桶，本机不再往桶里写入", st.Writer.Host, st.Writer.HostID), + SummaryEN: fmt.Sprintf("the off-site copy has stopped: %s took the bucket over, and this host copies nothing there any more", st.Writer), + Hint: "if that host is a rehearsal machine, take the bucket back with `sudo felis offsite take-over -yes`; `sudo felis offsite status` (docs/troubleshooting.md §16)", + } + } + if st != nil && st.Standby { + who, whoEN := "桶里有别的主机写入的副本，但没有记录是哪台主机", "the bucket holds another host's copies and names no host writing it" + if w := st.Writer; w != nil { + age := roundHours(max(now.Sub(w.At), 0)) + who = fmt.Sprintf("主机 %s（id %s）在 %s 前写入过这个桶", w.Host, w.HostID, age) + whoEN = fmt.Sprintf("%s wrote it %s ago", w, age) + } + return &Finding{ + Key: "offsite", Severity: Warning, For: backupFor, + Summary: "本机是从另一台主机的备份建起来的，不往异地备份桶写入：" + who, + SummaryEN: "this host was built from another host's backup and copies nothing into the off-site bucket: " + whoEN, + Hint: "once this host replaces that one for good: `sudo felis offsite take-over -yes` (docs/troubleshooting.md §16)", + } + } if st != nil && !st.LastSuccess.IsZero() && now.Sub(st.LastSuccess) <= offsite.StaleAfter { return nil } diff --git a/internal/watchdog/probes_test.go b/internal/watchdog/probes_test.go index 26f9fa8..aa0c732 100644 --- a/internal/watchdog/probes_test.go +++ b/internal/watchdog/probes_test.go @@ -188,6 +188,21 @@ func TestOffsiteFinding(t *testing.T) { if f := OffsiteFinding(path, t0); f == nil || !strings.Contains(f.SummaryEN, "has stopped") || !strings.Contains(f.SummaryEN, "(sealed with another key)") || !strings.Contains(f.Hint, "check-key") { t.Fatalf("key mismatch: %+v", f) } + // Another host writing the bucket stops every later run too: reported at + // once, a recent success notwithstanding. + w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0.Add(-5 * time.Hour)} + write(offsite.Status{LastAttempt: t0.Add(-time.Hour), LastSuccess: t0.Add(-2 * time.Hour), Displaced: true, Writer: w}) + if f := OffsiteFinding(path, t0); f == nil || !strings.Contains(f.SummaryEN, "has stopped: host prod-1 (id bbbbbbbbbbbbbbbb) took the bucket over") || !strings.Contains(f.Summary, "prod-1") || !strings.Contains(f.Hint, "take-over -yes") { + t.Fatalf("displaced: %+v", f) + } + write(offsite.Status{LastAttempt: t0.Add(-time.Hour), Standby: true, Writer: w}) + if f := OffsiteFinding(path, t0); f == nil || !strings.Contains(f.SummaryEN, "built from another host's backup") || !strings.Contains(f.SummaryEN, "host prod-1 (id bbbbbbbbbbbbbbbb) wrote it 5h ago") || !strings.Contains(f.Summary, "5h") || !strings.Contains(f.Hint, "take-over -yes") { + t.Fatalf("standing by: %+v", f) + } + write(offsite.Status{LastAttempt: t0.Add(-time.Hour), Standby: true}) + if f := OffsiteFinding(path, t0); f == nil || !strings.Contains(f.SummaryEN, "names no host writing it") { + t.Fatalf("standing by, no writer named: %+v", f) + } } func TestMemoryFinding(t *testing.T) {