fix(reaper): deliver pre-reap warnings for real — and never fake a delivery

The §18 warning path had no delivery channel at all: no Warner implementation
existed, `felis reaper` passed nil, and maybeWarn still stamped warned_3d_at/
warned_1d_at and counted `warned=N`. So every owned server was silently reaped
15 days after its last join with no notice, and the operator's only feedback
said warnings were sent. Two changes close that:

- Honest stamps: warned_* now records a DELIVERED notice. A nil Warner logs
  `warning suppressed — no warner wired` and does NOT stamp; a delivery error
  logs and retries on the next daily run (bounded by the warning window). The
  stamps are no longer burned by notices nobody received.

- A real channel: mail.SendNotice (the second and last message shape the mail
  package sends) plus a mailWarner that resolves the owner's VERIFIED email
  and mails the notice through the configured [smtp] relay. `felis reaper`
  wires it when [smtp] is set (same password_ref convention as felis-api) and
  prints exactly what happens when it is not.

Plumbing so the in-cluster CronJob can actually reach the relay: the reaper
pod gets the optional FELIS_SMTP_PASSWORD env (same Secret as felis-api), and
the "configure email" screen now refreshes the minecraft-namespace mirrors of
felis-smtp AND felis-config (a secretKeyRef is namespace-local, and the config
mirror is what carries [smtp] into the reaper's own config). `felis setup`'s
replica list gains felis-smtp for fresh installs.

Tests: the delivered/retried/suppressed matrix in internal/reaper (the old
"stamp advances on failure" contract is deliberately replaced), the notice
message shape, the warner's resolve/send/failure paths, and the CronJob's
optional-secret env. docs/troubleshooting.md §10 now states the real semantics.

internal/mail/mail.go

@@ -1,8 +1,9 @@
 // Package mail is the SMTP implementation of the api.OTPMailer seam: it
 // delivers the email one-time codes the passwordless doors mint (onboarding,
 // email login, op-login) through the relay configured in felis.toml [smtp].
-// It is deliberately tiny — one message shape, stdlib net/smtp — because the
-// only mail Felis ever sends is a six-digit code.
+// It is deliberately tiny — two message shapes, stdlib net/smtp — because the
+// only mail Felis ever sends is a six-digit code plus the reaper's pre-deletion
+// notice (SendNotice).
 //
 // TLS posture: port 465 dials implicit TLS; any other port dials plaintext and
 // upgrades via STARTTLS when the relay advertises it. AUTH is attempted only
@@ -53,6 +54,23 @@ func (s *SMTP) SendOTP(ctx context.Context, email, code string) error {
 	return c.Quit()
 }
 
+// SendNotice mails one operator-composed notice to email — the reaper's
+// pre-deletion warning is its only caller. Subject and body are the caller's;
+// the body is CRLF-normalized so a multi-line string renders as one text/plain
+// message. Delivery errors surface exactly like SendOTP's, so the caller can
+// retry on its own cadence.
+func (s *SMTP) SendNotice(ctx context.Context, email, subject, body string) error {
+	c, err := s.connect(ctx)
+	if err != nil {
+		return err
+	}
+	defer c.Close()
+	if err := s.deliver(c, email, notice(s.From, email, subject, body, time.Now())); err != nil {
+		return err
+	}
+	return c.Quit()
+}
+
 // Ping proves the configured relay will actually ACCEPT mail from this sender,
 // by running a complete transaction — connect, (STARTTLS,) AUTH, MAIL FROM,
 // RCPT TO, DATA — and delivering a short self-test message to From itself. The
@@ -208,3 +226,14 @@ func selfTest(from string, now time.Time) []byte {
 	b.WriteString("Sent by `felis setup` when the SMTP relay was configured. / 由 `felis setup` 配置 SMTP 时发出。\r\n")
 	return []byte(b.String())
 }
+
+// notice renders an operator notice: the shared header block plus the caller's
+// body, CRLF-normalized so every line obeys RFC 5322 regardless of which line
+// endings the caller's format string produced.
+func notice(from, to, subject, body string, now time.Time) []byte {
+	body = strings.ReplaceAll(strings.ReplaceAll(body, "\r\n", "\n"), "\n", "\r\n")
+	if !strings.HasSuffix(body, "\r\n") {
+		body += "\r\n"
+	}
+	return []byte(headers(from, to, subject, now) + body)
+}

internal/mail/mail_test.go

@@ -58,6 +58,33 @@ func TestSelfTestCarriesNoCode(t *testing.T) {
 	}
 }
 
+// TestNoticeShape pins the second message shape — the reaper's pre-deletion
+// warning: CRLF throughout even when the caller's body used bare LFs, a
+// Q-encoded subject when it carries non-ASCII, and the caller's text rendered
+// verbatim between the header block and the wire.
+func TestNoticeShape(t *testing.T) {
+	now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC)
+	msg := string(notice("[email protected]", "[email protected]",
+		"Felis: 服务器 survival 将回收", "line one\nline two\n", now))
+
+	if strings.Contains(strings.ReplaceAll(msg, "\r\n", ""), "\n") {
+		t.Error("notice contains a bare LF; every line must end CRLF")
+	}
+	headers, body, ok := strings.Cut(msg, "\r\n\r\n")
+	if !ok {
+		t.Fatal("notice has no blank line between headers and body")
+	}
+	if !strings.Contains(headers, "To: [email protected]") {
+		t.Errorf("headers missing To:\n%s", headers)
+	}
+	if !strings.Contains(headers, "Subject: =?utf-8?") {
+		t.Errorf("non-ASCII subject must be Q-encoded:\n%s", headers)
+	}
+	if !strings.Contains(body, "line one\r\nline two\r\n") {
+		t.Errorf("body must be CRLF-normalized verbatim text:\n%q", body)
+	}
+}
+
 // fakeRelay speaks just enough SMTP for net/smtp, answering 250 to MAIL FROM
 // and RCPT TO but dataVerdict at end-of-DATA. That split is the entire point:
 // relays which validate sender identity (Fastmail among them) accept MAIL FROM

internal/platform/workloads.go

@@ -536,6 +536,20 @@ func reaperCronJob(p Params) *batchv1.CronJob {
 			"--config", configFilePath,
 			"--worlds-root", worldsMountPath,
 		},
+		// The [smtp] relay password for pre-reap warning emails — same optional
+		// Secret felis-api reads. Namespace caveat: a secretKeyRef is
+		// namespace-local, so this resolves against the minecraft-ns felis-smtp
+		// mirror that the "configure email" screen refreshes (the felis-config
+		// mirror it also refreshes is what puts [smtp] in this pod's config).
+		// Absent Secret ⇒ empty env ⇒ the reaper logs suppressed warnings
+		// instead of stamping them (never a failed pod).
+		Env: []corev1.EnvVar{
+			{Name: SMTPPasswordEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
+				LocalObjectReference: corev1.LocalObjectReference{Name: SMTPSecretName},
+				Key:                  SMTPSecretPasswordKey,
+				Optional:             boolPtr(true),
+			}}},
+		},
 		VolumeMounts: []corev1.VolumeMount{
 			{Name: configVolume, MountPath: configMountPath, ReadOnly: true},
 			{Name: worldsVolume, MountPath: worldsMountPath, ReadOnly: true},

internal/platform/workloads_test.go

@@ -729,6 +729,19 @@ func TestReaperCronJob_Shape(t *testing.T) {
 		t.Errorf("reaper image = %q, want FelisImage %q", c.Image, p.FelisImage)
 	}
 
+	// The relay password for pre-reap warning emails: same optional Secret as
+	// felis-api, resolved against the minecraft-ns mirror. Optional so an install
+	// without SMTP still starts (the reaper then logs suppressed warnings).
+	smtpEnv := envVar(c.Env, SMTPPasswordEnv)
+	if smtpEnv == nil || smtpEnv.ValueFrom == nil || smtpEnv.ValueFrom.SecretKeyRef == nil {
+		t.Fatalf("reaper must wire %s from a secretKeyRef", SMTPPasswordEnv)
+	}
+	if ref := smtpEnv.ValueFrom.SecretKeyRef; ref.Name != SMTPSecretName || ref.Key != SMTPSecretPasswordKey {
+		t.Errorf("reaper %s ref = %s/%s, want %s/%s", SMTPPasswordEnv, ref.Name, ref.Key, SMTPSecretName, SMTPSecretPasswordKey)
+	} else if ref.Optional == nil || !*ref.Optional {
+		t.Errorf("reaper %s secretKeyRef must be optional", SMTPPasswordEnv)
+	}
+
 	// config: Secret, mounted read-only (it carries the DB URL).
 	cfgVol := volumeByName(ps.Volumes, configVolume)
 	if cfgVol == nil || cfgVol.Secret == nil || cfgVol.Secret.SecretName != configSecretName {

internal/reaper/reaper.go

@@ -203,7 +203,10 @@ type Cluster interface {
 }
 
 // Warner delivers an impending-reap notice. It is optional and best-effort: a
-// nil Warner or a delivery error never blocks a reap (red line ⑤).
+// nil Warner or a delivery error never blocks a reap (red line ⑤). Warn returns
+// nil only when the notice was handed to the delivery channel; an error (or a
+// nil Warner) leaves warned_* unstamped, so the next daily run retries instead
+// of silently burning the owner's only warning.
 type Warner interface {
 	Warn(ctx context.Context, ownerID, server, remaining string) error
 }
@@ -433,9 +436,11 @@ func (r *Reaper) ensureCapacity(ctx context.Context, now time.Time, sum *Summary
 
 // maybeWarn sends at most one impending-reap notice per run, honoring §18's
 // elif precedence (earliest unsent warning first). Unowned servers are never
-// warned but are still reaped at the deadline (red line ⑤). A warner delivery
-// failure is logged but the warned_* stamp still advances so the notice is not
-// retried forever; a real join (RecordJoin) is what clears the stamps.
+// warned but are still reaped at the deadline (red line ⑤). The warned_* stamp
+// records a DELIVERED notice: a nil Warner or a delivery error is logged and
+// leaves the stamp untouched, so the next run retries — bounded by the warning
+// window, since the reap itself removes the candidate. A real join (RecordJoin)
+// clears the stamps when a player renews.
 func (r *Reaper) maybeWarn(ctx context.Context, now time.Time, idle time.Duration, offs []time.Duration, c Candidate, sum *Summary) {
 	if c.OwnerID == "" {
 		return
@@ -449,10 +454,21 @@ func (r *Reaper) maybeWarn(ctx context.Context, now time.Time, idle time.Duratio
 		if !c.warnedAt(tier).IsZero() {
 			continue // already sent this tier
 		}
-		if r.Warner != nil {
-			if err := r.Warner.Warn(ctx, c.OwnerID, c.Name, formatRemaining(offs[i])); err != nil {
-				r.log().Warn("reaper: warn delivery failed (best-effort)", "server", c.Name, "err", err)
-			}
+		if r.Warner == nil {
+			// No delivery channel is wired at all. Do not stamp: an operator who
+			// wires one later must still be able to warn, and a stamp here would
+			// have recorded a notice nobody received. Logged every run so silence
+			// is never mistaken for delivery.
+			r.log().Warn("reaper: warning suppressed — no warner wired",
+				"server", c.Name, "owner", c.OwnerID, "remaining", formatRemaining(offs[i]))
+			return
+		}
+		if err := r.Warner.Warn(ctx, c.OwnerID, c.Name, formatRemaining(offs[i])); err != nil {
+			// Best-effort: the reap still proceeds on schedule, but the stamp
+			// stays empty so the next daily run retries the delivery instead of
+			// permanently suppressing the owner's only notice.
+			r.log().Warn("reaper: warn delivery failed; will retry next run", "server", c.Name, "err", err)
+			return
 		}
 		if err := r.Store.MarkWarned(ctx, c.Name, tier, now); err != nil {
 			r.log().Error("reaper: mark warned failed", "server", c.Name, "err", err)

internal/reaper/reaper_test.go

@@ -472,6 +472,8 @@ func TestWarningsDerivedFromNonDefaultDeadline(t *testing.T) {
 		// past 7d but unowned -> never warned (red line ⑤)
 		Candidate{Name: "e", OwnerID: "", LastActiveAt: idleBy(8 * Day)},
 	)
+	rw := &recordingWarner{}
+	r.Warner = rw
 
 	sum := mustRun(t, r)
 	if sum.WorldsReaped != 0 {
@@ -480,6 +482,9 @@ func TestWarningsDerivedFromNonDefaultDeadline(t *testing.T) {
 	if sum.Warned != 2 {
 		t.Fatalf("Warned = %d, want 2 (a:3d, b:1d)", sum.Warned)
 	}
+	if len(rw.sent) != 2 {
+		t.Fatalf("deliveries = %d, want 2 (a:3d, b:1d)", len(rw.sent))
+	}
 	if cl.deletePVCCalls != 0 {
 		t.Fatalf("a warning path deleted a PVC")
 	}
@@ -494,20 +499,48 @@ func TestWarningsDerivedFromNonDefaultDeadline(t *testing.T) {
 	}
 }
 
-// Red line ⑤ (best-effort): a Warner delivery error does not abort the run, and
-// the warned_* stamp still advances (a real join, not a failed warn, is what
-// resets the clock).
-func TestWarningBestEffortOnDeliveryFailure(t *testing.T) {
+// Red line ⑤ (best-effort) with delivery honesty: a Warner failure does not
+// abort the run, and it does NOT stamp — the stamp records a DELIVERED notice,
+// so the next daily run retries (the warning window bounds the retries, and the
+// reap clears the candidate either way).
+func TestWarningDeliveryRetriedAfterFailure(t *testing.T) {
 	r, st, _, _ := newReaper(DefaultConfig(),
 		Candidate{Name: "h", OwnerID: "u-h", LastActiveAt: idleBy(13 * Day)})
 	r.Warner = failWarner{}
 
 	sum := mustRun(t, r)
-	if sum.Warned != 1 {
-		t.Fatalf("Warned = %d, want 1 despite delivery failure", sum.Warned)
+	if sum.Warned != 0 {
+		t.Fatalf("Warned = %d, want 0 (nothing was delivered)", sum.Warned)
+	}
+	if !st.byName["h"].Warned3dAt.IsZero() {
+		t.Fatal("a failed delivery must not stamp warned_3d_at")
+	}
+
+	// Next run with a working channel: the SAME warning goes out and stamps.
+	rw := &recordingWarner{}
+	r.Warner = rw
+	sum = mustRun(t, r)
+	if sum.Warned != 1 || len(rw.sent) != 1 {
+		t.Fatalf("retry: Warned=%d sent=%d, want 1/1", sum.Warned, len(rw.sent))
 	}
 	if st.byName["h"].Warned3dAt.IsZero() {
-		t.Fatalf("warned_3d_at not stamped after best-effort warn")
+		t.Fatal("a delivered warning must stamp warned_3d_at")
+	}
+}
+
+// A nil Warner suppresses the warning WITHOUT stamping it: nothing was sent, so
+// nothing is recorded as sent — and the day a channel is wired, the owner can
+// still be warned.
+func TestWarningSuppressedWithoutWarner(t *testing.T) {
+	r, st, _, _ := newReaper(DefaultConfig(),
+		Candidate{Name: "n", OwnerID: "u-n", LastActiveAt: idleBy(13 * Day)})
+
+	sum := mustRun(t, r)
+	if sum.Warned != 0 {
+		t.Fatalf("Warned = %d, want 0 with no warner wired", sum.Warned)
+	}
+	if !st.byName["n"].Warned3dAt.IsZero() || !st.byName["n"].Warned1dAt.IsZero() {
+		t.Fatal("a suppressed warning must not stamp either tier")
 	}
 }
 
@@ -517,6 +550,15 @@ func (failWarner) Warn(context.Context, string, string, string) error {
 	return errors.New("smtp unavailable")
 }
 
+// recordingWarner captures deliveries so the threshold tests exercise the real
+// deliver-then-stamp path.
+type recordingWarner struct{ sent []string }
+
+func (w *recordingWarner) Warn(_ context.Context, ownerID, server, remaining string) error {
+	w.sent = append(w.sent, ownerID+"/"+server+"/"+remaining)
+	return nil
+}
+
 // §26 capacity: when the store is over its cap, the oldest backup is evicted
 // early (destructive — audited) to make room, then the reap proceeds.
 func TestCapacityEvictsOldestThenReaps(t *testing.T) {