fix(offsite): 桶内记录写入主机，演练机只读不写也不给生产 owner 发告警，take-over 显式接管

internal/offsite/keymark.go

@@ -8,6 +8,7 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"time"
 )
 
 // keyMark is the one object the bucket holds in the clear: the KeyID of the
@@ -137,13 +138,24 @@ func CheckKey(ctx context.Context, b Bucket, key []byte) (KeyFit, error) {
 	return KeyUnused, nil
 }
 
-// ClaimKey is CheckKey, then records key's id in a bucket that has none, so
-// that every later check reads the id.
-func ClaimKey(ctx context.Context, b Bucket, key []byte) error {
+// claim is the check before a run writes anything: CheckKey, then the lease
+// (nil checks none), then key's id recorded in a bucket that has none, so
+// that every later check reads the id. The key goes first, so a host with the
+// wrong key never records itself as the writer, and the lease before the key
+// id, so a standby host writes nothing at all.
+func claim(ctx context.Context, b Bucket, key []byte, lease *Lease, now time.Time) error {
 	fit, err := CheckKey(ctx, b, key)
-	if err != nil || fit == KeyRecorded {
+	if err != nil {
 		return err
 	}
+	if lease != nil {
+		if err := lease.Acquire(ctx, b, fit == KeyUnused, now); err != nil {
+			return err
+		}
+	}
+	if fit == KeyRecorded {
+		return nil
+	}
 	id := KeyID(key) + "\n"
 	if err := b.Put(ctx, keyMark, strings.NewReader(id), int64(len(id))); err != nil {
 		return fmt.Errorf("record the key id in %s: %w", keyMark, err)

internal/offsite/keymark_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -160,36 +161,36 @@ func TestCheckKey(t *testing.T) {
 	}
 }
 
-func TestClaimKey(t *testing.T) {
+func TestClaim(t *testing.T) {
 	key, other := testKey(t), testKey(t)
 	marker := func(b *memBucket) string { return string(b.objs[keyMark]) }
 
 	b := newMemBucket()
-	if err := ClaimKey(context.Background(), b, key); err != nil {
+	if err := claim(context.Background(), b, key, nil, time.Time{}); err != nil {
 		t.Fatal(err)
 	}
 	if marker(b) != KeyID(key)+"\n" {
 		t.Fatalf("empty bucket: marker = %q, want %s", marker(b), KeyID(key))
 	}
-	if err := ClaimKey(context.Background(), b, key); err != nil || b.puts != 1 {
+	if err := claim(context.Background(), b, key, nil, time.Time{}); err != nil || b.puts != 1 {
 		t.Errorf("second claim: err %v, puts %d; want the marker written once", err, b.puts)
 	}
 	if fit, err := CheckKey(context.Background(), b, key); fit != KeyRecorded || err != nil {
 		t.Errorf("after the claim: CheckKey = %v, %v", fit, err)
 	}
-	if err := ClaimKey(context.Background(), b, other); !errors.Is(err, ErrKeyMismatch) || marker(b) != KeyID(key)+"\n" {
+	if err := claim(context.Background(), b, other, nil, time.Time{}); !errors.Is(err, ErrKeyMismatch) || marker(b) != KeyID(key)+"\n" {
 		t.Errorf("another key: err %v, marker %q; want a refusal that leaves the marker", err, marker(b))
 	}
 
 	b = newMemBucket()
 	putAt(b, "worlds/1.fenc", seal(t, []byte("1"), key), 0)
-	if err := ClaimKey(context.Background(), b, key); err != nil || marker(b) != KeyID(key)+"\n" {
+	if err := claim(context.Background(), b, key, nil, time.Time{}); err != nil || marker(b) != KeyID(key)+"\n" {
 		t.Errorf("unmarked bucket the key opens: err %v, marker %q", err, marker(b))
 	}
 
 	b = newMemBucket()
 	putAt(b, "worlds/1.fenc", seal(t, []byte("1"), other), 0)
-	if err := ClaimKey(context.Background(), b, key); !errors.Is(err, ErrKeyMismatch) || b.puts != 0 {
+	if err := claim(context.Background(), b, key, nil, time.Time{}); !errors.Is(err, ErrKeyMismatch) || b.puts != 0 {
 		t.Errorf("unmarked bucket under another key: err %v, puts %d; want a refusal that writes nothing", err, b.puts)
 	}
 }
@@ -214,6 +215,9 @@ func TestSyncRefusesAnotherKeysBucket(t *testing.T) {
 			for i, name := range []string{"felis-db-20260901T030000Z-daily.tar", "felis-db-20260902T030000Z-daily.tar", "felis-db-20260903T030000Z-daily.tar"} {
 				putAt(b, DBKey(name), seal(t, []byte(name), other), i)
 			}
+			// A new host: the wrong key must not record it as the writer either.
+			l := testLease(t, "")
+			s.Lease = &l
 			before := len(b.objs)
 
 			_, err := s.Run(context.Background())
@@ -226,6 +230,9 @@ func TestSyncRefusesAnotherKeysBucket(t *testing.T) {
 			if !cat.rows[0].offsite.IsZero() {
 				t.Error("the refused run recorded alpha as copied")
 			}
+			if _, err := os.Stat(l.IDFile); !errors.Is(err, os.ErrNotExist) {
+				t.Errorf("the refused run made this host an id: %v", err)
+			}
 		})
 	}
 }

internal/offsite/status.go

@@ -32,6 +32,46 @@ type Status struct {
 	// with another key (ErrKeyMismatch): no later run copies anything until
 	// the key is fixed, so the watchdog reports it at once.
 	KeyMismatch bool `json:"key_mismatch,omitempty"`
+	// Standby and Displaced are a run refused because another host, Writer,
+	// writes the bucket (ErrStandby, ErrDisplaced).
+	Standby   bool    `json:"standby,omitempty"`
+	Displaced bool    `json:"displaced,omitempty"`
+	Writer    *Writer `json:"writer,omitempty"`
+	// Format is StatusFormat in every record this release writes; 0 is a
+	// record from before writers were recorded, whose host had been copying
+	// to the bucket (Lease.Inherited).
+	Format int `json:"format,omitempty"`
+	// Inherited carries Lease.Inherited over runs that ended before the host
+	// recorded itself (an unreachable bucket on the first run after the
+	// upgrade), until it has an id.
+	Inherited bool `json:"inherited,omitempty"`
+}
+
+// StatusFormat marks a status record that knows about felis-writer.
+const StatusFormat = 2
+
+// StandsBy is the host this one stands by for: the last run was refused
+// because that host writes the bucket, and it wrote it within WriterLive of
+// now. While it keeps writing, this host is a rehearsal (or a rebuild not yet
+// taken over), and the owners in its restored database are that host's: the
+// watchdog here mails them nothing.
+func (st *Status) StandsBy(now time.Time) *Writer {
+	if st == nil || !st.Standby || st.Writer == nil || now.Sub(st.Writer.At) > WriterLive {
+		return nil
+	}
+	return st.Writer
+}
+
+// HostLease is the Lease of the host whose status file is statusFile: its id
+// next to it, and Inherited when that file was written by an older release
+// (or carries Inherited from one).
+func HostLease(statusFile string) Lease {
+	host, _ := os.Hostname()
+	l := Lease{IDFile: filepath.Join(filepath.Dir(statusFile), HostIDFile), Host: host}
+	if prev, err := ReadStatus(statusFile); err == nil && prev != nil && (prev.Format == 0 || prev.Inherited) {
+		l.Inherited = true
+	}
+	return l
 }
 
 // ReadStatus reads the status file. A missing file is (nil, nil): no sync has

internal/offsite/sync.go

@@ -104,6 +104,8 @@ type Syncer struct {
 	UploadsDir string
 	// UploadGrace and MinRate bound one object's upload: UploadGrace plus the
 	// time the object takes at MinRate bytes a second. Zero takes the defaults.
+	// Lease names this host in felis-writer (writer.go); nil checks no writer.
+	Lease       *Lease
 	UploadGrace time.Duration
 	MinRate     int64
 	Now         func() time.Time
@@ -201,8 +203,9 @@ func (s *Syncer) Run(ctx context.Context) (Result, error) {
 	}
 
 	// Before anything is written or pruned: objects sealed with another key
-	// are copies only that key opens, and DBKeep would prune them.
-	if err := ClaimKey(ctx, s.Bucket, s.Key); err != nil {
+	// are copies only that key opens, and DBKeep would prune them; a bucket
+	// another host writes is that host's to prune.
+	if err := claim(ctx, s.Bucket, s.Key, s.Lease, s.now()); err != nil {
 		return res, err
 	}
 	remoteWorlds, err := s.listSizes(ctx, worldsDir)

internal/offsite/writer.go

@@ -0,0 +1,263 @@
+package offsite
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+	"unicode"
+)
+
+// writerMark names the host that writes the bucket. A rehearsal on a spare
+// machine restores the production host's /etc/felis, [offsite] and its key
+// included: without the record the spare would copy its bundles into the
+// production prefix and prune the production host's by DBKeep. The writer
+// rewrites it on every run; a host restored from its backup finds another
+// host named there and stands by, writing nothing, until `felis offsite
+// take-over`.
+const writerMark = "felis-writer"
+
+// WriterLive is how recently the writer must have run for a standby host to
+// count it as alive. Both run hourly, so a writer seen within it is one that
+// ran in the last two hours.
+const WriterLive = 3 * time.Hour
+
+// HostIDFile is the file, next to the status file, holding this host's id. It
+// is written when the host first writes the bucket and never leaves the host
+// (a bundle carries /etc/felis only), so a host restored from a bundle has
+// none.
+const HostIDFile = "host-id"
+
+var (
+	// ErrStandby is a run refused because another host writes the bucket and
+	// this one never has.
+	ErrStandby = errors.New("offsite: another host writes this bucket")
+	// ErrDisplaced is a run refused because another host took the bucket
+	// over from this one.
+	ErrDisplaced = errors.New("offsite: another host took this bucket over")
+)
+
+const takeOverHint = "sudo felis offsite take-over -yes (docs/troubleshooting.md §16)"
+
+// Writer is the felis-writer record.
+type Writer struct {
+	HostID string    `json:"host_id"`
+	Host   string    `json:"host"`
+	At     time.Time `json:"at"`
+}
+
+func (w *Writer) String() string {
+	return fmt.Sprintf("host %s (id %s)", w.Host, w.HostID)
+}
+
+// WriterError is a run refused because another host writes the bucket.
+type WriterError struct {
+	// Kind is ErrStandby or ErrDisplaced.
+	Kind error
+	// Writer is the host named in the bucket; nil for a bucket that holds
+	// another host's copies and names no writer.
+	Writer *Writer
+}
+
+func (e *WriterError) Error() string {
+	switch {
+	case e.Kind == ErrDisplaced:
+		return fmt.Sprintf("%v: %s writes it now (last at %s), and this host copies nothing there any more; if that host is a rehearsal machine, take the bucket back here: %s",
+			e.Kind, e.Writer, e.Writer.At.UTC().Format(time.RFC3339), takeOverHint)
+	case e.Writer != nil:
+		return fmt.Sprintf("%v: %s writes it (last at %s); this host was built from its backup and copies nothing there, until it replaces that host for good: %s",
+			e.Kind, e.Writer, e.Writer.At.UTC().Format(time.RFC3339), takeOverHint)
+	default:
+		return fmt.Sprintf("%v: the bucket holds copies this host did not write and names no host writing it; this host copies nothing there, until it replaces that host for good: %s",
+			e.Kind, takeOverHint)
+	}
+}
+
+func (e *WriterError) Unwrap() error { return e.Kind }
+
+// Role is what a host's next run does with the bucket.
+type Role int
+
+const (
+	// RoleWrites: the bucket names this host.
+	RoleWrites Role = iota + 1
+	// RoleClaims: the bucket names no host, and this one records itself.
+	RoleClaims
+	// RoleStandby: another host writes the bucket, and this one never has.
+	RoleStandby
+	// RoleDisplaced: another host took the bucket over from this one.
+	RoleDisplaced
+)
+
+// Lease is this host's side of felis-writer.
+type Lease struct {
+	// IDFile is this host's id (HostIDFile next to the status file).
+	IDFile string
+	// Host is this host's name, shown to the others.
+	Host string
+	// Inherited is a host that copied to the bucket before writers were
+	// recorded: a status file an older release wrote. It claims a bucket
+	// that names no writer.
+	Inherited bool
+}
+
+// BucketWriter reads the felis-writer record, nil when there is none.
+func BucketWriter(ctx context.Context, b Bucket) (*Writer, error) {
+	rc, err := b.Get(ctx, writerMark)
+	if errors.Is(err, ErrNotFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", writerMark, err)
+	}
+	defer rc.Close()
+	raw, err := io.ReadAll(io.LimitReader(rc, 1024))
+	if err != nil {
+		return nil, fmt.Errorf("read %s: %w", writerMark, err)
+	}
+	var w Writer
+	if json.Unmarshal(raw, &w) != nil || !keyIDPattern.MatchString(w.HostID) {
+		return nil, fmt.Errorf("offsite: %s in the bucket is not a record Felis wrote", writerMark)
+	}
+	w.Host = printable(w.Host)
+	return &w, nil
+}
+
+// Plan says what this host's next run does with the bucket, and the host the
+// bucket names (nil for none). empty is a bucket holding no sealed object
+// (CheckKey's KeyUnused), which any host may claim.
+func (l Lease) Plan(ctx context.Context, b Bucket, empty bool) (Role, *Writer, error) {
+	w, err := BucketWriter(ctx, b)
+	if err != nil {
+		return 0, nil, err
+	}
+	mine, err := l.ID()
+	if err != nil {
+		return 0, nil, err
+	}
+	switch {
+	case w != nil && w.HostID == mine:
+		return RoleWrites, w, nil
+	case w != nil && mine != "":
+		return RoleDisplaced, w, nil
+	case w != nil:
+		return RoleStandby, w, nil
+	case mine != "" || l.Inherited || empty:
+		return RoleClaims, nil, nil
+	default:
+		return RoleStandby, nil, nil
+	}
+}
+
+// Acquire is Plan before a run writes anything: a host that writes or claims
+// the bucket records itself there at now; one that stands by or was displaced
+// gets a *WriterError and writes nothing.
+func (l Lease) Acquire(ctx context.Context, b Bucket, empty bool, now time.Time) error {
+	role, w, err := l.Plan(ctx, b, empty)
+	if err != nil {
+		return err
+	}
+	switch role {
+	case RoleStandby:
+		return &WriterError{Kind: ErrStandby, Writer: w}
+	case RoleDisplaced:
+		return &WriterError{Kind: ErrDisplaced, Writer: w}
+	}
+	return l.record(ctx, b, now)
+}
+
+// TakeOver records this host as the bucket's writer whatever the bucket named,
+// and returns the writer it replaced (nil for none). That host's next run is
+// refused with ErrDisplaced.
+func (l Lease) TakeOver(ctx context.Context, b Bucket, now time.Time) (*Writer, error) {
+	prev, err := BucketWriter(ctx, b)
+	if err != nil {
+		return nil, err
+	}
+	return prev, l.record(ctx, b, now)
+}
+
+// record writes this host into felis-writer, creating its id first: a host
+// whose id is on disk but not in the bucket claims the bucket on its next run,
+// where one named in the bucket without an id on disk would stand by for
+// itself.
+func (l Lease) record(ctx context.Context, b Bucket, now time.Time) error {
+	id, err := l.ID()
+	if err != nil {
+		return err
+	}
+	if id == "" {
+		if id, err = l.newID(); err != nil {
+			return err
+		}
+	}
+	raw, err := json.Marshal(Writer{HostID: id, Host: printable(l.Host), At: now.UTC()})
+	if err != nil {
+		return err
+	}
+	raw = append(raw, '\n')
+	if err := b.Put(ctx, writerMark, strings.NewReader(string(raw)), int64(len(raw))); err != nil {
+		return fmt.Errorf("record this host in %s: %w", writerMark, err)
+	}
+	return nil
+}
+
+// ID is this host's id, "" when it has never written a bucket.
+func (l Lease) ID() (string, error) {
+	raw, err := os.ReadFile(l.IDFile)
+	if errors.Is(err, os.ErrNotExist) {
+		return "", nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("read this host's id: %w", err)
+	}
+	id := strings.TrimSpace(string(raw))
+	if !keyIDPattern.MatchString(id) {
+		return "", fmt.Errorf("offsite: %s is not a host id Felis wrote; remove it and run sudo felis offsite take-over -yes", l.IDFile)
+	}
+	return id, nil
+}
+
+func (l Lease) newID() (string, error) {
+	var b [8]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", err
+	}
+	id := hex.EncodeToString(b[:])
+	if err := os.MkdirAll(filepath.Dir(l.IDFile), 0o700); err != nil {
+		return "", fmt.Errorf("record this host's id: %w", err)
+	}
+	tmp := l.IDFile + ".tmp"
+	if err := os.WriteFile(tmp, []byte(id+"\n"), 0o600); err != nil {
+		return "", fmt.Errorf("record this host's id: %w", err)
+	}
+	if err := os.Rename(tmp, l.IDFile); err != nil {
+		return "", fmt.Errorf("record this host's id: %w", err)
+	}
+	return id, nil
+}
+
+// printable keeps a host name fit for a message: at most 64 printable runes,
+// "unknown" for none.
+func printable(s string) string {
+	s = strings.Map(func(r rune) rune {
+		if unicode.IsPrint(r) {
+			return r
+		}
+		return -1
+	}, s)
+	if r := []rune(s); len(r) > 64 {
+		s = string(r[:64])
+	}
+	if strings.TrimSpace(s) == "" {
+		return "unknown"
+	}
+	return s
+}

internal/offsite/writer_test.go

@@ -0,0 +1,313 @@
+package offsite
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+const (
+	myID    = "aaaaaaaaaaaaaaaa"
+	otherID = "bbbbbbbbbbbbbbbb"
+)
+
+func putWriter(t *testing.T, b *memBucket, id, host string, at time.Time) {
+	t.Helper()
+	raw, err := json.Marshal(Writer{HostID: id, Host: host, At: at})
+	if err != nil {
+		t.Fatal(err)
+	}
+	b.objs[writerMark] = raw
+}
+
+// testLease is a lease whose id file is in a temp dir, holding id unless it
+// is "".
+func testLease(t *testing.T, id string) Lease {
+	t.Helper()
+	l := Lease{IDFile: filepath.Join(t.TempDir(), HostIDFile), Host: "spare-1"}
+	if id != "" {
+		if err := os.WriteFile(l.IDFile, []byte(id+"\n"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return l
+}
+
+func TestLeasePlan(t *testing.T) {
+	at := now.Add(-20 * time.Minute)
+	for _, tc := range []struct {
+		what      string
+		mine      string
+		inherited bool
+		writer    string // the id felis-writer names, "" for none
+		empty     bool
+		want      Role
+	}{
+		{"an empty bucket, a new host", "", false, "", true, RoleClaims},
+		{"another host's copies, no writer named, a new host", "", false, "", false, RoleStandby},
+		{"another host's copies, no writer named, a host that copied before writers were recorded", "", true, "", false, RoleClaims},
+		{"no writer named, a host that wrote before", myID, false, "", false, RoleClaims},
+		{"this host named", myID, false, myID, false, RoleWrites},
+		{"another host named, a host that wrote before", myID, false, otherID, false, RoleDisplaced},
+		{"another host named, a new host", "", false, otherID, false, RoleStandby},
+		{"another host named, a host that copied before writers were recorded", "", true, otherID, false, RoleStandby},
+		{"another host named over an empty bucket", "", false, otherID, true, RoleStandby},
+	} {
+		t.Run(tc.what, func(t *testing.T) {
+			b := newMemBucket()
+			if tc.writer != "" {
+				putWriter(t, b, tc.writer, "prod-1", at)
+			}
+			l := testLease(t, tc.mine)
+			l.Inherited = tc.inherited
+			role, w, err := l.Plan(context.Background(), b, tc.empty)
+			if err != nil || role != tc.want {
+				t.Fatalf("Plan = %v, %v; want %v", role, err, tc.want)
+			}
+			if (w != nil) != (tc.writer != "") || (w != nil && (w.HostID != tc.writer || w.Host != "prod-1" || !w.At.Equal(at))) {
+				t.Errorf("Plan named %+v, want the bucket's writer %q", w, tc.writer)
+			}
+			if b.puts != 0 {
+				t.Errorf("Plan wrote %d objects", b.puts)
+			}
+		})
+	}
+
+	b := newMemBucket()
+	b.objs[writerMark] = []byte("hello")
+	if _, _, err := testLease(t, "").Plan(context.Background(), b, true); err == nil || !strings.Contains(err.Error(), "not a record Felis wrote") {
+		t.Errorf("a record Felis did not write: err = %v", err)
+	}
+	putWriter(t, b, "../../etc", "prod-1", at)
+	if _, _, err := testLease(t, "").Plan(context.Background(), b, true); err == nil {
+		t.Error("a record with an id Felis does not make was taken")
+	}
+	b = newMemBucket()
+	b.getErr = map[string]error{writerMark: errors.New("connection reset")}
+	if _, _, err := testLease(t, "").Plan(context.Background(), b, true); err == nil || !strings.Contains(err.Error(), "connection reset") {
+		t.Errorf("an unreadable record: err = %v, want the read error", err)
+	}
+	b = newMemBucket()
+	if _, _, err := testLease(t, "not-an-id").Plan(context.Background(), b, true); err == nil || !strings.Contains(err.Error(), "not a host id Felis wrote") {
+		t.Errorf("a damaged id file: err = %v", err)
+	}
+}
+
+func TestLeaseAcquire(t *testing.T) {
+	ctx := context.Background()
+
+	// A new host claims an empty bucket: its id is created and recorded.
+	b := newMemBucket()
+	l := testLease(t, "")
+	if err := l.Acquire(ctx, b, true, now); err != nil {
+		t.Fatal(err)
+	}
+	id, err := l.ID()
+	if err != nil || !keyIDPattern.MatchString(id) {
+		t.Fatalf("id after the claim = %q, %v", id, err)
+	}
+	if fi, err := os.Stat(l.IDFile); err != nil || fi.Mode().Perm() != 0o600 {
+		t.Errorf("id file: %v, %v", fi, err)
+	}
+	w, err := BucketWriter(ctx, b)
+	if err != nil || w == nil || w.HostID != id || w.Host != "spare-1" || !w.At.Equal(now) {
+		t.Fatalf("record after the claim = %+v, %v", w, err)
+	}
+
+	// Its next run keeps the id and moves the time on.
+	later := now.Add(time.Hour)
+	if err := l.Acquire(ctx, b, false, later); err != nil {
+		t.Fatal(err)
+	}
+	if w, _ := BucketWriter(ctx, b); w.HostID != id || !w.At.Equal(later) {
+		t.Errorf("record after the next run = %+v, want %s at %s", w, id, later)
+	}
+
+	// A host restored from its backup stands by: nothing written, no id made.
+	spare := testLease(t, "")
+	puts := b.puts
+	err = spare.Acquire(ctx, b, false, later)
+	var we *WriterError
+	if !errors.Is(err, ErrStandby) || !errors.As(err, &we) || we.Writer == nil || we.Writer.HostID != id {
+		t.Fatalf("spare: err = %v, want ErrStandby naming %s", err, id)
+	}
+	if b.puts != puts {
+		t.Error("the standby host wrote to the bucket")
+	}
+	if _, err := os.Stat(spare.IDFile); !errors.Is(err, os.ErrNotExist) {
+		t.Errorf("the standby host made an id: %v", err)
+	}
+
+	// The spare takes over; the first host is displaced.
+	prev, err := spare.TakeOver(ctx, b, later)
+	if err != nil || prev == nil || prev.HostID != id {
+		t.Fatalf("TakeOver = %+v, %v; want the first host replaced", prev, err)
+	}
+	spareID, _ := spare.ID()
+	if spareID == "" || spareID == id {
+		t.Fatalf("spare id after the take-over = %q", spareID)
+	}
+	puts = b.puts
+	err = l.Acquire(ctx, b, false, later)
+	if !errors.Is(err, ErrDisplaced) || !errors.As(err, &we) || we.Writer.HostID != spareID {
+		t.Fatalf("first host after the take-over: err = %v, want ErrDisplaced naming %s", err, spareID)
+	}
+	if b.puts != puts {
+		t.Error("the displaced host wrote to the bucket")
+	}
+
+	// A host name is kept printable and short for the messages that show it.
+	b = newMemBucket()
+	l = testLease(t, "")
+	l.Host = "evil\x1b[2J\n" + strings.Repeat("x", 80)
+	if err := l.Acquire(ctx, b, true, now); err != nil {
+		t.Fatal(err)
+	}
+	if w, _ := BucketWriter(ctx, b); w.Host != "evil[2J"+strings.Repeat("x", 57) {
+		t.Errorf("recorded host = %q", w.Host)
+	}
+
+	// A record that cannot be written fails the run.
+	b = newMemBucket()
+	b.putErr[writerMark] = errors.New("access denied")
+	if err := testLease(t, "").Acquire(ctx, b, true, now); err == nil || !strings.Contains(err.Error(), "access denied") {
+		t.Errorf("unwritable record: err = %v", err)
+	}
+}
+
+func TestWriterErrorSays(t *testing.T) {
+	w := &Writer{HostID: otherID, Host: "prod-1", At: now}
+	for _, tc := range []struct {
+		err  *WriterError
+		kind error
+		says []string
+	}{
+		{&WriterError{Kind: ErrStandby, Writer: w}, ErrStandby, []string{"host prod-1 (id " + otherID + ")", "2026-09-24T12:00:00Z", "built from its backup", "take-over -yes"}},
+		{&WriterError{Kind: ErrStandby}, ErrStandby, []string{"names no host writing it", "take-over -yes"}},
+		{&WriterError{Kind: ErrDisplaced, Writer: w}, ErrDisplaced, []string{"host prod-1 (id " + otherID + ") writes it now", "rehearsal machine", "take-over -yes"}},
+	} {
+		msg := tc.err.Error()
+		if !errors.Is(tc.err, tc.kind) {
+			t.Errorf("%q is not %v", msg, tc.kind)
+		}
+		for _, s := range tc.says {
+			if !strings.Contains(msg, s) {
+				t.Errorf("%q lacks %q", msg, s)
+			}
+		}
+	}
+}
+
+func TestStandsBy(t *testing.T) {
+	w := &Writer{HostID: otherID, Host: "prod-1", At: now.Add(-2 * time.Hour)}
+	for _, tc := range []struct {
+		what string
+		st   *Status
+		at   time.Time
+		want bool
+	}{
+		{"a standby run, the writer seen two hours ago", &Status{Standby: true, Writer: w}, now, true},
+		{"a standby run, the writer gone quiet", &Status{Standby: true, Writer: w}, now.Add(WriterLive), false},
+		{"a standby run, no writer named", &Status{Standby: true}, now, false},
+		{"a displaced run", &Status{Displaced: true, Writer: w}, now, false},
+		{"a run that copied", &Status{Writer: w}, now, false},
+		{"no run yet", nil, now, false},
+	} {
+		if got := tc.st.StandsBy(tc.at); (got != nil) != tc.want {
+			t.Errorf("%s: StandsBy = %+v, want %v", tc.what, got, tc.want)
+		}
+	}
+}
+
+func TestHostLease(t *testing.T) {
+	dir := t.TempDir()
+	status := filepath.Join(dir, "status.json")
+	if l := HostLease(status); l.IDFile != filepath.Join(dir, HostIDFile) || l.Inherited || l.Host == "" {
+		t.Errorf("no status yet: %+v", l)
+	}
+	for _, tc := range []struct {
+		what string
+		st   Status
+		want bool
+	}{
+		{"a status an older release wrote", Status{LastAttempt: now}, true},
+		{"a status this release wrote", Status{LastAttempt: now, Format: StatusFormat}, false},
+		{"a status that carries the older release's claim", Status{LastAttempt: now, Format: StatusFormat, Inherited: true}, true},
+	} {
+		if err := WriteStatus(status, tc.st); err != nil {
+			t.Fatal(err)
+		}
+		if got := HostLease(status).Inherited; got != tc.want {
+			t.Errorf("%s: Inherited = %v, want %v", tc.what, got, tc.want)
+		}
+	}
+	if err := os.WriteFile(status, []byte("{"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if HostLease(status).Inherited {
+		t.Error("an unreadable status counted as an older release's")
+	}
+}
+
+// TestSyncStandsBy: a host restored from the writer's backup copies nothing
+// into the bucket, prunes nothing, and records nothing as copied, whether the
+// bucket names that host or holds its copies without naming anyone.
+func TestSyncStandsBy(t *testing.T) {
+	for _, named := range []bool{true, false} {
+		t.Run(fmt.Sprintf("named=%v", named), func(t *testing.T) {
+			cat := &fakeCatalog{rows: []*row{
+				{WorldBackup: WorldBackup{ID: "b1", Server: "alpha", Ref: "/a/alpha-1.tar.gz"}, status: "present"},
+			}}
+			s, b := newSyncer(t, cat)
+			delete(b.objs, keyMark)
+			if named {
+				putWriter(t, b, otherID, "prod-1", now.Add(-30*time.Minute))
+			}
+			writeFile(t, s.ArchiveDir, "alpha-1.tar.gz", 100)
+			writeFile(t, s.DBDir, "felis-db-20260924T030000Z-daily.tar", 50)
+			for i, name := range []string{"felis-db-20260901T030000Z-daily.tar", "felis-db-20260902T030000Z-daily.tar", "felis-db-20260903T030000Z-daily.tar"} {
+				putAt(b, DBKey(name), seal(t, []byte(name), s.Key), i)
+			}
+			l := testLease(t, "")
+			s.Lease = &l
+			before := len(b.objs)
+
+			_, err := s.Run(context.Background())
+			if !errors.Is(err, ErrStandby) {
+				t.Fatalf("Run error = %v, want ErrStandby", err)
+			}
+			if len(b.started) != 0 || len(b.removed) != 0 || len(b.objs) != before {
+				t.Errorf("the standby run began puts %v and removed %v", b.started, b.removed)
+			}
+			if !cat.rows[0].offsite.IsZero() {
+				t.Error("the standby run recorded alpha as copied")
+			}
+		})
+	}
+}
+
+// TestSyncRecordsTheWriter: the first run records this host before the key
+// id and the first upload.
+func TestSyncRecordsTheWriter(t *testing.T) {
+	s, b := newSyncer(t, &fakeCatalog{})
+	delete(b.objs, keyMark)
+	writeFile(t, s.DBDir, "felis-db-20260924T030000Z-daily.tar", 50)
+	l := testLease(t, "")
+	s.Lease = &l
+	if _, err := s.Run(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	id, _ := l.ID()
+	if w, err := BucketWriter(context.Background(), b); err != nil || w == nil || w.HostID != id {
+		t.Fatalf("record = %+v, %v; want %s", w, err, id)
+	}
+	if len(b.started) != 3 || b.started[0] != writerMark || b.started[1] != keyMark {
+		t.Errorf("puts began in the order %v, want the writer, the key id, then the bundle", b.started)
+	}
+}