fix(offsite): 桶内记录写入主机，演练机只读不写也不给生产 owner 发告警，take-over 显式接管 (149ab0be) · Commits · FelisMC / Felis · GitLab

cmd/felis/offsite.go

+153 −13

Changes for cmd/felis/offsite.go: 153 added lines, 13 removed lines.
Original line number	Diff line number	Diff line
		@@ -34,6 +34,7 @@ const offsiteUsage = `usage:
		felis offsite fetch-images [-config path] [-registry host:port] [-at version]
		felis offsite fetch-uploads [-config path] [-uploads-dir dir] [-at version]
		felis offsite check-key [-config path]
		felis offsite take-over [-config path] [-status-file path] [-yes]
		felis offsite keygen

		Every verb but keygen reads the bucket credentials and the encryption key from
		@@ -44,6 +45,13 @@ FELIS_OFFSITE_SECRET_KEY, FELIS_OFFSITE_KEY), taking any that are unset from
		check-key tells whether the key is the one the bucket's objects are sealed
		with, writing nothing; it exits 3 when they are sealed with another key, and
		sync then refuses to write or prune anything in the bucket.

		take-over names the host that writes the bucket, writing nothing; it exits 4
		when that is another host and this one never wrote it, and 5 when another
		host took the bucket over from this one. A host built from another host's
		backup (a rehearsal, or a rebuild) copies nothing into that host's bucket
		until -yes makes it the writer; the host it replaces then stops copying and
		says so.
		`

		// defaultOffsiteEnvFile is where bootstrap keeps the [offsite] secrets; the
		@@ -81,6 +89,8 @@ func cmdOffsite(args []string, stdout, stderr io.Writer) int {
		return offsiteFetchUploads(fs, rest, stdout, stderr)
		case "check-key":
		return offsiteCheckKey(fs, rest, stdout, stderr)
		case "take-over":
		return offsiteTakeOver(fs, rest, stdout, stderr)
		case "keygen":
		k, err := offsite.NewKey()
		if err != nil {
		@@ -213,28 +223,32 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
		fmt.Fprintf(stderr, "felis offsite sync: %v\n", err)
		return 1
		}
		st := offsite.Status{
		LastAttempt: time.Now().UTC(), Endpoint: env.cfg.Endpoint, Bucket: env.cfg.Bucket,
		Prefix: env.cfg.Prefix, KeyID: offsite.KeyID(env.key),
		}
		if prev, _ := offsite.ReadStatus(*statusFile); prev != nil {
		st.LastSuccess = prev.LastSuccess
		}
		st, lease := startRun(env.cfg, env.key, *statusFile, time.Now())
		res, err := runOffsiteSync(cfg, env, offsiteSources{
		archiveDir: archiveDir, backupPVC: backupPVC, dbDir: *dbDir,
		registry: offsiteRegistryEndpoint(*registry, cfg.Registry),
		uploadsDir: uploadsDir, uploadsPVC: uploadsPVC,
		}, stderr)
		recordRun(&st, res, err)
		}, &lease, stderr)
		recordRun(&st, res, err, lease)
		if werr := offsite.WriteStatus(*statusFile, st); werr != nil {
		fmt.Fprintf(stderr, "felis offsite sync: record status: %v\n", werr)
		}
		return reportRun(res, err, stdout, stderr)
		}

		// reportRun prints one pass's outcome and its exit code. A run stopped before
		// it copied anything (a bucket that did not answer, another key's objects,
		// another host writing the bucket) prints no counts: its zeros would read as
		// an empty bucket.
		func reportRun(res offsite.Result, err error, stdout, stderr io.Writer) int {
		if err == nil \|\| len(res.Errors) > 0 {
		fmt.Fprintf(stdout, "felis offsite sync: worlds copied=%d pending=%d missing=%d expired=%d; bundles copied=%d pruned=%d; images copied=%d blobs=%d pruned=%d; uploads copied=%d pruned=%d; bucket holds %d worlds (%s), %d bundles, %d images in %d repositories (%s), %d uploads (%s)\n",
		res.WorldsUploaded, res.WorldsPending, len(res.WorldsMissing), res.WorldsExpired,
		res.DBUploaded, res.DBPruned, res.ImagesUploaded, res.ImageBlobsUploaded, res.ImageObjectsPruned,
		res.UploadsUploaded, res.UploadObjectsPruned,
		res.RemoteWorlds, offsite.HumanBytes(res.RemoteBytes), res.RemoteDB, res.Images, res.ImageRepos, offsite.HumanBytes(res.RemoteImageBytes),
		res.Uploads, offsite.HumanBytes(res.RemoteUploadBytes))
		}
		for _, m := range res.WorldsMissing {
		fmt.Fprintf(stderr, "felis offsite sync: recorded archive not on the volume, nothing to copy: %s\n", m)
		}
		@@ -248,15 +262,40 @@ func offsiteSync(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int
		return 0
		}

		// recordRun puts one pass's outcome into its status record.
		func recordRun(st *offsite.Status, res offsite.Result, err error) {
		// startRun begins a pass: its status record, in this release's format and
		// carrying the last success over, and this host's lease, read from the record
		// the last pass left.
		func startRun(cfg config.OffsiteConfig, key []byte, statusFile string, now time.Time) (offsite.Status, offsite.Lease) {
		st := offsite.Status{
		LastAttempt: now.UTC(), Endpoint: cfg.Endpoint, Bucket: cfg.Bucket,
		Prefix: cfg.Prefix, KeyID: offsite.KeyID(key), Format: offsite.StatusFormat,
		}
		if prev, _ := offsite.ReadStatus(statusFile); prev != nil {
		st.LastSuccess = prev.LastSuccess
		}
		return st, offsite.HostLease(statusFile)
		}

		// recordRun puts one pass's outcome into its status record. A host that
		// inherited the bucket from an older release keeps that until it has an id.
		func recordRun(st *offsite.Status, res offsite.Result, err error, lease offsite.Lease) {
		st.Result = res
		if err != nil {
		st.LastError = err.Error()
		st.KeyMismatch = errors.Is(err, offsite.ErrKeyMismatch)
		var we *offsite.WriterError
		if errors.As(err, &we) {
		st.Standby = errors.Is(err, offsite.ErrStandby)
		st.Displaced = errors.Is(err, offsite.ErrDisplaced)
		st.Writer = we.Writer
		}
		} else {
		st.LastSuccess = st.LastAttempt
		}
		if lease.Inherited {
		id, _ := lease.ID()
		st.Inherited = id == ""
		}
		}

		// offsiteSources is where one sync pass reads from: the world archive volume
		@@ -276,7 +315,7 @@ type offsiteSources struct {
		// TimeoutStartSec sits above this.
		const offsiteRunLimit = 23 * time.Hour

		func runOffsiteSync(cfg config.Config, env offsiteEnv, src offsiteSources, log io.Writer) (offsite.Result, error) {
		func runOffsiteSync(cfg config.Config, env offsiteEnv, src offsiteSources, lease *offsite.Lease, log io.Writer) (offsite.Result, error) {
		ctx, cancel := context.WithTimeout(context.Background(), offsiteRunLimit)
		defer cancel()
		checkCtx, checkCancel := context.WithTimeout(ctx, 30*time.Second)
		@@ -309,7 +348,7 @@ func runOffsiteSync(cfg config.Config, env offsiteEnv, src offsiteSources, log
		defer drv.Close()
		s := &offsite.Syncer{
		Bucket: env.bucket, Catalog: offsite.PGCatalog{DB: drv.DB()}, Key: env.key,
		ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Log: log,
		ArchiveDir: archiveDir, DBDir: src.dbDir, DBKeep: env.cfg.DBKeep, UploadsDir: uploadsDir, Lease: lease, Log: log,
		}
		if src.registry != "" {
		s.Images = newRegistryImages(src.registry)
		@@ -455,6 +494,23 @@ func offsiteStatus(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) in
		fmt.Fprintf(stdout, "\nThe last run was refused: the bucket's objects are sealed with another key than this host's (key id %s). No sync copies or prunes anything there until FELIS_OFFSITE_KEY in %s is theirs (sudo felis offsite check-key).\n", st.KeyID, defaultOffsiteEnvFile)
		return 1
		}
		if st.Displaced && st.Writer != nil {
		fmt.Fprintf(stdout, "\nThe last run was refused: %s took the bucket over (it last wrote it at %s), and this host copies nothing there any more. If that host is a rehearsal machine, take the bucket back: sudo felis offsite take-over -yes\n",
		st.Writer, st.Writer.At.Local().Format(time.DateTime))
		return 1
		}
		if st.Standby {
		switch w := st.StandsBy(now); {
		case w != nil:
		fmt.Fprintf(stdout, "\nThis host stands by: %s writes the bucket (last at %s). This host was built from its backup, copies nothing into the bucket and, while that host keeps writing, mails no watchdog alert.", w, w.At.Local().Format(time.DateTime))
		case st.Writer != nil:
		fmt.Fprintf(stdout, "\nThis host copies nothing into the bucket: %s wrote it, last at %s, and this host was built from its backup.", st.Writer, st.Writer.At.Local().Format(time.DateTime))
		default:
		fmt.Fprint(stdout, "\nThis host copies nothing into the bucket: it holds copies this host did not write, and names no host writing it.")
		}
		fmt.Fprintln(stdout, " Once this host replaces that one for good: sudo felis offsite take-over -yes")
		return 1
		}
		r := st.Result
		fmt.Fprintf(stdout, "bucket holds: %d world archives (%s), %d database bundles, newest %s\n",
		r.RemoteWorlds, offsite.HumanBytes(r.RemoteBytes), r.RemoteDB, orNone(r.NewestDB))
		@@ -612,6 +668,90 @@ func checkKey(ctx context.Context, b offsite.Bucket, key []byte, stdout, stderr
		return 0
		}

		func offsiteTakeOver(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
		cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
		envFile := fs.String("env-file", defaultOffsiteEnvFile, "file with the [offsite] secrets, for variables not already set")
		statusFile := fs.String("status-file", offsite.DefaultStatusFile, "the record `sync` writes; this host's id is kept next to it")
		yes := fs.Bool("yes", false, "make this host the one that writes the bucket")
		if err := fs.Parse(args); err != nil {
		return 2
		}
		_, env, err := loadOffsite(cfgPath, envFile)
		if err != nil {
		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
		return 1
		}
		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
		defer cancel()
		if err := env.bucket.Check(ctx); err != nil {
		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
		return 1
		}
		return takeOver(ctx, env.bucket, env.key, offsite.HostLease(statusFile), statusFile, *yes, time.Now(), stdout, stderr)
		}

		// takeOver is take-over once the bucket is open: without yes it says which
		// host writes the bucket, 0 for this one (or none yet), 4 for another and 5
		// for one that took the bucket over from this host; with
		// yes it records this host as the writer. A key the bucket's objects refuse
		// is 3, as in check-key: taking over a bucket this host cannot copy into
		// would only stop the host that can.
		func takeOver(ctx context.Context, b offsite.Bucket, key []byte, lease offsite.Lease, statusFile string, yes bool, now time.Time, stdout, stderr io.Writer) int {
		fit, err := offsite.CheckKey(ctx, b, key)
		if err != nil {
		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
		if errors.Is(err, offsite.ErrKeyMismatch) {
		return 3
		}
		return 1
		}
		role, w, err := lease.Plan(ctx, b, fit == offsite.KeyUnused)
		if err != nil {
		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
		return 1
		}
		switch role {
		case offsite.RoleWrites:
		id, _ := lease.ID()
		fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket; nothing to take over\n", id)
		return 0
		case offsite.RoleClaims:
		fmt.Fprintln(stdout, "felis offsite take-over: the bucket names no host writing it; this host's next sync records itself")
		return 0
		}
		who := "another host"
		if w != nil {
		who = w.String()
		fmt.Fprintf(stdout, "felis offsite take-over: %s writes the bucket, last at %s (%s ago)\n", w, w.At.Local().Format(time.DateTime), dbbackup.Age(now.Sub(w.At)))
		} else {
		fmt.Fprintln(stdout, "felis offsite take-over: the bucket holds copies this host did not write, and names no host writing it")
		}
		if !yes {
		if role == offsite.RoleDisplaced {
		fmt.Fprintf(stdout, "It took the bucket over from this host: this host copies nothing there any more, and its watchdog mails the owners about it. If that host is a rehearsal machine, take the bucket back:\n sudo felis offsite take-over -yes\n")
		return 5
		}
		fmt.Fprintf(stdout, "This host was built from its backup and copies nothing into the bucket.\n")
		fmt.Fprintf(stdout, "Taking it over makes this host the one that copies into the bucket and prunes it; %s stops at its next copy and mails its owners. Do it once that host is gone for good, or is a rehearsal machine you are done with:\n sudo felis offsite take-over -yes\n", who)
		return 4
		}
		if _, err := lease.TakeOver(ctx, b, now); err != nil {
		fmt.Fprintf(stderr, "felis offsite take-over: %v\n", err)
		return 1
		}
		// The refusal the last sync recorded is over: the watchdog mails again
		// from now on, and status shows the next run's outcome.
		if st, err := offsite.ReadStatus(statusFile); err == nil && st != nil && (st.Standby \|\| st.Displaced) {
		st.Standby, st.Displaced, st.Writer, st.LastError, st.Inherited = false, false, nil, "", false
		if err := offsite.WriteStatus(statusFile, *st); err != nil {
		fmt.Fprintf(stderr, "felis offsite take-over: record status: %v\n", err)
		}
		}
		id, _ := lease.ID()
		fmt.Fprintf(stdout, "felis offsite take-over: this host (id %s) writes the bucket now; %s stops at its next copy.\nStart the first copy: sudo systemctl start felis-offsite.service\n", id, who)
		return 0
		}

		// keyHint explains an object the key cannot open when the bucket records
		// another key's id, "" otherwise.
		func keyHint(ctx context.Context, b offsite.Bucket, key []byte, err error) string {

cmd/felis/offsite_test.go

+249 −8

Changes for cmd/felis/offsite_test.go: 249 added lines, 8 removed lines.
Original line number	Diff line number	Diff line
		@@ -356,23 +356,218 @@ func TestOffsiteCheckKey(t *testing.T) {
		}
		}

		func TestRecordRunMarksAKeyMismatch(t *testing.T) {
		func TestRecordRun(t *testing.T) {
		t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
		w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}
		for _, tc := range []struct {
		err error
		mismatch bool
		success bool
		mismatch, standby, displaced, success bool
		}{
		{nil, false, true},
		{errors.New("list worlds/ in the bucket: connection reset"), false, false},
		{fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false},
		{nil, false, false, false, true},
		{errors.New("list worlds/ in the bucket: connection reset"), false, false, false, false},
		{fmt.Errorf("%w: the bucket records key id 0123456789abcdef", offsite.ErrKeyMismatch), true, false, false, false},
		{&offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, false, true, false, false},
		{&offsite.WriterError{Kind: offsite.ErrDisplaced, Writer: w}, false, false, true, false},
		} {
		st := offsite.Status{LastAttempt: t0}
		recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err)
		if st.KeyMismatch != tc.mismatch \|\| st.LastSuccess.Equal(t0) != tc.success \|\| st.Result.RemoteDB != 2 {
		recordRun(&st, offsite.Result{RemoteDB: 2}, tc.err, offsite.Lease{})
		if st.KeyMismatch != tc.mismatch \|\| st.Standby != tc.standby \|\| st.Displaced != tc.displaced \|\|
		(st.Writer != nil) != (tc.standby \|\| tc.displaced) \|\| st.LastSuccess.Equal(t0) != tc.success \|\| st.Result.RemoteDB != 2 {
		t.Errorf("err %v: status %+v", tc.err, st)
		}
		}

		// A host that copied before writers were recorded keeps that claim over
		// failed runs until it has an id.
		l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Inherited: true}
		st := offsite.Status{}
		recordRun(&st, offsite.Result{}, errors.New("cannot reach bucket"), l)
		if !st.Inherited {
		t.Error("a failed run on a host with no id dropped the older release's claim")
		}
		writeTestFile(t, l.IDFile, "aaaaaaaaaaaaaaaa\n", 0o600)
		st = offsite.Status{Inherited: true}
		recordRun(&st, offsite.Result{}, nil, l)
		if st.Inherited {
		t.Error("a host with an id still carries the older release's claim")
		}
		}

		// A refused run has copied nothing and listed nothing: its zero counts would
		// tell the journal the bucket is empty. A pass that ran and failed a step
		// shows what it did get to.
		func TestReportRun(t *testing.T) {
		w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)}
		for _, tc := range []struct {
		name string
		res offsite.Result
		err error
		code int
		counts bool
		}{
		{"a pass", offsite.Result{DBUploaded: 1, RemoteDB: 3}, nil, 0, true},
		{"a pass with a failed step", offsite.Result{RemoteDB: 3, Errors: []string{"copy db/x: timeout"}}, errors.New("1 of this run's steps failed; first: copy db/x: timeout"), 1, true},
		{"standing by", offsite.Result{}, &offsite.WriterError{Kind: offsite.ErrStandby, Writer: w}, 1, false},
		{"another key", offsite.Result{}, fmt.Errorf("%w: the bucket records key id 1111111111111111", offsite.ErrKeyMismatch), 1, false},
		{"no bucket", offsite.Result{}, errors.New("bucket: access denied"), 1, false},
		} {
		t.Run(tc.name, func(t *testing.T) {
		var out, errOut bytes.Buffer
		code := reportRun(tc.res, tc.err, &out, &errOut)
		if code != tc.code {
		t.Errorf("exit %d, want %d", code, tc.code)
		}
		if got := strings.Contains(out.String(), "bucket holds 0 worlds (0 B), 3 bundles"); got != tc.counts {
		t.Errorf("counts shown = %v, want %v: %q", got, tc.counts, out.String())
		}
		if !tc.counts && out.Len() > 0 {
		t.Errorf("a refused run printed %q", out.String())
		}
		if tc.err != nil && !strings.Contains(errOut.String(), "felis offsite sync: "+tc.err.Error()) {
		t.Errorf("stderr %q lacks the error", errOut.String())
		}
		})
		}
		}

		func TestOffsiteTakeOver(t *testing.T) {
		newKey := func() []byte {
		raw, _ := offsite.NewKey()
		k, _ := offsite.ParseKey(raw)
		return k
		}
		key, other := newKey(), newKey()
		const mine, theirs = "aaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbb"
		t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
		sealed := func(k []byte, writer string) mapBucket {
		b := mapBucket{}
		putBundle(t, b, k, 0, nil)
		b["felis-key-id"] = []byte(offsite.KeyID(k) + "\n")
		if writer != "" {
		raw, _ := json.Marshal(offsite.Writer{HostID: writer, Host: "prod-1", At: t0.Add(-20 * time.Minute)})
		b["felis-writer"] = raw
		}
		return b
		}
		lease := func(id string) offsite.Lease {
		l := offsite.Lease{IDFile: filepath.Join(t.TempDir(), offsite.HostIDFile), Host: "spare-1"}
		if id != "" {
		writeTestFile(t, l.IDFile, id+"\n", 0o600)
		}
		return l
		}
		run := func(b mapBucket, l offsite.Lease, statusFile string, yes bool) (int, string, string) {
		t.Helper()
		var out, errb bytes.Buffer
		code := takeOver(context.Background(), b, key, l, statusFile, yes, t0, &out, &errb)
		return code, out.String(), errb.String()
		}
		for _, tc := range []struct {
		what string
		bucket mapBucket
		id string
		code int
		says []string
		}{
		{"this host writes the bucket", sealed(key, mine), mine, 0, []string{"this host (id " + mine + ") writes the bucket; nothing to take over"}},
		{"an empty bucket", mapBucket{}, "", 0, []string{"names no host writing it; this host's next sync records itself"}},
		{"a host built from the writer's backup", sealed(key, theirs), "", 4, []string{"host prod-1 (id " + theirs + ") writes the bucket, last at", "(20m ago)", "built from its backup", "sudo felis offsite take-over -yes"}},
		{"another host's copies, no writer named", sealed(key, ""), "", 4, []string{"holds copies this host did not write, and names no host writing it", "take-over -yes"}},
		{"a host another one took over from", sealed(key, theirs), mine, 5, []string{"took the bucket over from this host"}},
		{"a key the bucket refuses", sealed(other, theirs), "", 3, []string{"sealed with another key"}},
		} {
		t.Run(tc.what, func(t *testing.T) {
		before := string(tc.bucket["felis-writer"])
		l := lease(tc.id)
		code, out, errb := run(tc.bucket, l, filepath.Join(t.TempDir(), "status.json"), false)
		if code != tc.code {
		t.Fatalf("exit %d, want %d\n%s%s", code, tc.code, out, errb)
		}
		for _, s := range tc.says {
		if !strings.Contains(out+errb, s) {
		t.Errorf("output lacks %q:\n%s%s", s, out, errb)
		}
		}
		if string(tc.bucket["felis-writer"]) != before {
		t.Error("take-over without -yes wrote the bucket's writer")
		}
		if tc.id == "" {
		if _, err := os.Stat(l.IDFile); !errors.Is(err, os.ErrNotExist) {
		t.Errorf("take-over without -yes made this host an id: %v", err)
		}
		}
		})
		}

		// -yes on a standby host: the bucket names it, and the refusal the last
		// sync recorded is cleared, so the watchdog mails again at once.
		b := sealed(key, theirs)
		l := lease("")
		statusFile := filepath.Join(t.TempDir(), "status.json")
		lastSuccess := t0.Add(-48 * time.Hour)
		if err := offsite.WriteStatus(statusFile, offsite.Status{
		LastAttempt: t0.Add(-time.Hour), LastSuccess: lastSuccess, LastError: "offsite: another host writes this bucket", Format: offsite.StatusFormat,
		Standby: true, Writer: &offsite.Writer{HostID: theirs, Host: "prod-1", At: t0}, Inherited: true,
		}); err != nil {
		t.Fatal(err)
		}
		code, out, errb := run(b, l, statusFile, true)
		id, _ := l.ID()
		if code != 0 \|\| id == "" \|\| !strings.Contains(out, "this host (id "+id+") writes the bucket now; host prod-1 (id "+theirs+") stops at its next copy") \|\| !strings.Contains(out, "systemctl start felis-offsite.service") {
		t.Fatalf("take-over -yes: exit %d, id %q\n%s%s", code, id, out, errb)
		}
		if w, err := offsite.BucketWriter(context.Background(), b); err != nil \|\| w.HostID != id \|\| w.Host != "spare-1" \|\| !w.At.Equal(t0) {
		t.Errorf("writer after take-over -yes = %+v, %v", w, err)
		}
		st, err := offsite.ReadStatus(statusFile)
		if err != nil \|\| st.Standby \|\| st.Writer != nil \|\| st.LastError != "" \|\| st.Inherited \|\| !st.LastSuccess.Equal(lastSuccess) {
		t.Errorf("status after take-over -yes = %+v, %v; want the refusal cleared and the last success kept", st, err)
		}

		// -yes with a key the bucket refuses writes nothing.
		b = sealed(other, theirs)
		before := string(b["felis-writer"])
		if code, _, _ := run(b, lease(""), filepath.Join(t.TempDir(), "status.json"), true); code != 3 \|\| string(b["felis-writer"]) != before {
		t.Errorf("take-over -yes under another key: exit %d, writer %s", code, b["felis-writer"])
		}
		}

		func TestOffsiteStatusSaysWhoWritesTheBucket(t *testing.T) {
		dir := t.TempDir()
		cfg := filepath.Join(dir, "felis.toml")
		writeTestFile(t, cfg, installerTOML("example.com", "127.0.0.1")+"\n[offsite]\nendpoint = \"https://s3.example.com\"\nbucket = \"felis-backups\"\n", 0o600)
		statusFile := filepath.Join(dir, "status.json")
		now := time.Now()
		w := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-30 * time.Minute)}
		stale := &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: now.Add(-offsite.WriterLive - time.Hour)}
		for _, tc := range []struct {
		what string
		st offsite.Status
		says []string
		not string
		}{
		{"standing by for a live writer", offsite.Status{Standby: true, Writer: w}, []string{"This host stands by: host prod-1 (id bbbbbbbbbbbbbbbb) writes the bucket", "mails no watchdog alert", "take-over -yes"}, "wrote it, last at"},
		{"standing by for a writer gone quiet", offsite.Status{Standby: true, Writer: stale}, []string{"copies nothing into the bucket: host prod-1 (id bbbbbbbbbbbbbbbb) wrote it, last at", "take-over -yes"}, "mails no watchdog alert"},
		{"standing by, no writer named", offsite.Status{Standby: true}, []string{"names no host writing it", "take-over -yes"}, "stands by:"},
		{"displaced", offsite.Status{Displaced: true, Writer: w}, []string{"host prod-1 (id bbbbbbbbbbbbbbbb) took the bucket over", "rehearsal machine", "take-over -yes"}, "stands by"},
		} {
		t.Run(tc.what, func(t *testing.T) {
		tc.st.LastAttempt, tc.st.LastSuccess, tc.st.LastError = now.Add(-time.Minute), now.Add(-time.Hour), "offsite: another host writes this bucket"
		if err := offsite.WriteStatus(statusFile, tc.st); err != nil {
		t.Fatal(err)
		}
		var out, errb bytes.Buffer
		code := cmdOffsite([]string{"status", "-config", cfg, "-status-file", statusFile}, &out, &errb)
		if code != 1 \|\| strings.Contains(out.String(), "bucket holds:") \|\| strings.Contains(out.String(), tc.not) {
		t.Errorf("exit %d\n%s%s", code, out.String(), errb.String())
		}
		for _, s := range tc.says {
		if !strings.Contains(out.String(), s) {
		t.Errorf("output lacks %q:\n%s", s, out.String())
		}
		}
		})
		}
		}

		func TestOffsiteStatusSaysTheKeyWasRefused(t *testing.T) {
		@@ -432,3 +627,49 @@ func TestPrintDBBundlesSaysWhatEachHolds(t *testing.T) {
		}
		}
		}

		// TestRestoredHostKeepsStandingBy walks the status file across runs: a host
		// restored from the writer's backup stands by on its first run and on every
		// run after it, a host an older release left copying claims the bucket once,
		// and a failed first run after the upgrade keeps that claim.
		func TestRestoredHostKeepsStandingBy(t *testing.T) {
		rawKey, _ := offsite.NewKey()
		key, _ := offsite.ParseKey(rawKey)
		cfg := config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-backups"}
		t0 := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
		standby := &offsite.WriterError{Kind: offsite.ErrStandby, Writer: &offsite.Writer{HostID: "bbbbbbbbbbbbbbbb", Host: "prod-1", At: t0}}
		pass := func(statusFile string, at time.Time, err error) offsite.Lease {
		t.Helper()
		st, lease := startRun(cfg, key, statusFile, at)
		recordRun(&st, offsite.Result{}, err, lease)
		if werr := offsite.WriteStatus(statusFile, st); werr != nil {
		t.Fatal(werr)
		}
		return lease
		}

		restored := filepath.Join(t.TempDir(), "status.json")
		for i := range 3 {
		if l := pass(restored, t0.Add(time.Duration(i)*time.Hour), standby); l.Inherited {
		t.Fatalf("run %d of a restored host claims the bucket", i+1)
		}
		}
		if st, _ := offsite.ReadStatus(restored); !st.Standby \|\| st.Format != offsite.StatusFormat \|\| st.KeyID != offsite.KeyID(key) \|\| st.Bucket != "felis-backups" {
		t.Errorf("restored host's status = %+v", st)
		}

		upgraded := filepath.Join(t.TempDir(), "status.json")
		if err := offsite.WriteStatus(upgraded, offsite.Status{LastAttempt: t0.Add(-time.Hour), LastSuccess: t0.Add(-time.Hour)}); err != nil {
		t.Fatal(err)
		}
		if l := pass(upgraded, t0, errors.New("cannot reach bucket")); !l.Inherited {
		t.Fatal("the first run after the upgrade does not claim the bucket")
		}
		l := pass(upgraded, t0.Add(time.Hour), nil)
		if !l.Inherited {
		t.Fatal("a failed first run after the upgrade lost the claim")
		}
		if st, _ := offsite.ReadStatus(upgraded); !st.LastSuccess.Equal(t0.Add(time.Hour)) {
		t.Errorf("upgraded host's status = %+v", st)
		}
		}

cmd/felis/watchdog.go

+24 −2

File changed.

Preview size limit exceeded, changes collapsed.

cmd/felis/watchdog_test.go

+49 −0

File changed.

Preview size limit exceeded, changes collapsed.

deploy/bootstrap.sh

+46 −4

File changed.

Preview size limit exceeded, changes collapsed.