fix(reaper): deliver pre-reap warnings for real — and never fake a delivery

The §18 warning path had no delivery channel at all: no Warner implementation
existed, `felis reaper` passed nil, and maybeWarn still stamped warned_3d_at/
warned_1d_at and counted `warned=N`. So every owned server was silently reaped
15 days after its last join with no notice, and the operator's only feedback
said warnings were sent. Two changes close that:

- Honest stamps: warned_* now records a DELIVERED notice. A nil Warner logs
  `warning suppressed — no warner wired` and does NOT stamp; a delivery error
  logs and retries on the next daily run (bounded by the warning window). The
  stamps are no longer burned by notices nobody received.

- A real channel: mail.SendNotice (the second and last message shape the mail
  package sends) plus a mailWarner that resolves the owner's VERIFIED email
  and mails the notice through the configured [smtp] relay. `felis reaper`
  wires it when [smtp] is set (same password_ref convention as felis-api) and
  prints exactly what happens when it is not.

Plumbing so the in-cluster CronJob can actually reach the relay: the reaper
pod gets the optional FELIS_SMTP_PASSWORD env (same Secret as felis-api), and
the "configure email" screen now refreshes the minecraft-namespace mirrors of
felis-smtp AND felis-config (a secretKeyRef is namespace-local, and the config
mirror is what carries [smtp] into the reaper's own config). `felis setup`'s
replica list gains felis-smtp for fresh installs.

Tests: the delivered/retried/suppressed matrix in internal/reaper (the old
"stamp advances on failure" contract is deliberately replaced), the notice
message shape, the warner's resolve/send/failure paths, and the CronJob's
optional-secret env. docs/troubleshooting.md §10 now states the real semantics.
This commit is contained in:
Lemon-miaow committed 2026-09-23 03:47:19 +08:00
1 parent 1d0ec61c9d
commit 8e7c7bbf24
12 files changed
+375 -27

No files matched your search

+77
View File
@@ -2,6 +2,8 @@ package main
import ( import (
"context" "context"
"database/sql"
"errors"
"flag" "flag"
"fmt" "fmt"
"io" "io"
@@ -14,6 +16,8 @@ import (
"felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/backup" "felis.lolicon.best/internal/backup"
"felis.lolicon.best/internal/config" "felis.lolicon.best/internal/config"
"felis.lolicon.best/internal/mail"
"felis.lolicon.best/internal/platform"
"felis.lolicon.best/internal/reaper" "felis.lolicon.best/internal/reaper"
"felis.lolicon.best/internal/store" "felis.lolicon.best/internal/store"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
@@ -81,6 +85,47 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
Archiver: archiver, Archiver: archiver,
} }
// Pre-reap warnings go out by email when [smtp] is configured (the same
// relay and password_ref convention felis-api uses); without it the channel
// stays nil and the reaper logs each suppressed warning instead of stamping
// it, so a later SMTP setup still gets to warn. The owner must have a
// VERIFIED address — that flag is what proves the mailbox.
if cfg.SMTP.Host != "" {
passRef := cfg.SMTP.PasswordRef
if passRef == "" {
passRef = platform.SMTPPasswordEnv
}
password := os.Getenv(passRef)
if cfg.SMTP.Username != "" && password == "" {
fmt.Fprintf(stderr, "felis reaper: warning: [smtp] username is set but credentials env %s is empty — warning emails will fail AUTH\n", passRef)
}
db := drv.DB()
r.Warner = &mailWarner{
lookupEmail: func(ctx context.Context, ownerID string) (string, error) {
var email string
switch err := db.QueryRowContext(ctx,
`SELECT email FROM users
WHERE id = $1 AND email_verified = true AND COALESCE(email, '') <> ''`,
ownerID).Scan(&email); {
case errors.Is(err, sql.ErrNoRows):
return "", fmt.Errorf("owner %s has no verified email", ownerID)
case err != nil:
return "", err
}
return email, nil
},
notifier: &mail.SMTP{
Host: cfg.SMTP.Host,
Port: cfg.SMTP.Port,
From: cfg.SMTP.From,
Username: cfg.SMTP.Username,
Password: password,
},
}
} else {
fmt.Fprintln(stderr, "felis reaper: [smtp] not configured — pre-reap warnings are logged and NOT marked sent")
}
sum, err := r.RunOnce(ctx) sum, err := r.RunOnce(ctx)
if err != nil { if err != nil {
fmt.Fprintf(stderr, "felis reaper: %v\n", err) fmt.Fprintf(stderr, "felis reaper: %v\n", err)
@@ -91,6 +136,38 @@ func cmdReaper(args []string, stdout, stderr io.Writer) int {
return 0 return 0
} }
// mailWarner delivers a pre-reap notice to the owner's verified email — the
// only channel this build can reach. Unowned owners and owners who never proved
// a mailbox yield an error; the reaper retries such notices on its next run and
// never lets them block the reap (red line ⑤).
type mailWarner struct {
lookupEmail func(ctx context.Context, ownerID string) (string, error)
notifier noticeNotifier
}
// noticeNotifier is the slice of mail.SMTP the warner needs (injected in tests).
type noticeNotifier interface {
SendNotice(ctx context.Context, email, subject, body string) error
}
func (w *mailWarner) Warn(ctx context.Context, ownerID, server, remaining string) error {
email, err := w.lookupEmail(ctx, ownerID)
if err != nil {
return fmt.Errorf("resolve owner email: %w", err)
}
subject := fmt.Sprintf("Felis: 服务器 %s 将在 %s 后回收 · server reaped in %s", server, remaining, remaining)
body := fmt.Sprintf(
"Felis 世界回收提醒 / world-reaper notice\r\n"+
"\r\n"+
"服务器 / Server: %s\r\n"+
"距回收 / Time left: %s\r\n"+
"\r\n"+
"闲置的服务器会先自动备份,再释放世界;有人加入游戏即可重置倒计时。\r\n"+
"Idle servers are backed up and then released; any join resets the countdown.\r\n",
server, remaining)
return w.notifier.SendNotice(ctx, email, subject, body)
}
// reaperConfig derives the reaper's retention windows from felis.toml. The 15d // reaperConfig derives the reaper's retention windows from felis.toml. The 15d
// idle deadline is fixed by §18; only the warning offsets, retention, and the // idle deadline is fixed by §18; only the warning offsets, retention, and the
// store soft-cap are configurable (§24). // store soft-cap are configurable (§24).
+46
View File
@@ -2,6 +2,7 @@ package main
import ( import (
"context" "context"
"errors"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@@ -75,3 +76,48 @@ func TestResolveWorldDir(t *testing.T) {
} }
}) })
} }
// The pre-reap warner resolves the owner's VERIFIED email and hands the notice
// to the mailer. Every failure (no verified address, relay refusal) returns an
// error so the reaper retries on its next run instead of stamping a notice
// nobody received.
func TestMailWarner(t *testing.T) {
lookup := func(email string, err error) func(context.Context, string) (string, error) {
return func(context.Context, string) (string, error) { return email, err }
}
n := &captureNotifier{}
w := &mailWarner{lookupEmail: lookup("[email protected]", nil), notifier: n}
if err := w.Warn(context.Background(), "u1", "survival", "3d"); err != nil {
t.Fatalf("Warn: %v", err)
}
if n.email != "[email protected]" || !strings.Contains(n.subject, "survival") || !strings.Contains(n.subject, "3d") {
t.Fatalf("notice envelope = (%q, %q)", n.email, n.subject)
}
if !strings.Contains(n.body, "survival") || !strings.Contains(n.body, "3d") {
t.Fatalf("body missing server/remaining:\n%s", n.body)
}
w = &mailWarner{lookupEmail: lookup("", errors.New("owner u2 has no verified email")), notifier: n}
if err := w.Warn(context.Background(), "u2", "survival", "3d"); err == nil || !strings.Contains(err.Error(), "verified email") {
t.Fatalf("unverified owner = %v, want the lookup error surfaced", err)
}
w = &mailWarner{lookupEmail: lookup("[email protected]", nil), notifier: &captureNotifier{err: errors.New("relay down")}}
if err := w.Warn(context.Background(), "u1", "survival", "3d"); err == nil || !strings.Contains(err.Error(), "relay down") {
t.Fatalf("relay failure = %v, want it surfaced", err)
}
}
type captureNotifier struct {
email, subject, body string
err error
}
func (n *captureNotifier) SendNotice(_ context.Context, email, subject, body string) error {
if n.err != nil {
return n.err
}
n.email, n.subject, n.body = email, subject, body
return nil
}
+7
View File
@@ -246,6 +246,13 @@ func provisionSystemServers(ctx context.Context, cfg *config.Config, out io.Writ
naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns"), naming.ForwardingSecretName, naming.ForwardingSecretKey, "forwarding-secret", "minecraft ns"),
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace, ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
"felis-config", "felis.toml", "config", "minecraft ns"), "felis-config", "felis.toml", "config", "minecraft ns"),
// The reaper's pre-reap warning emails authenticate with the same relay
// password felis-api uses; the reaper pod runs in the minecraft namespace,
// where a secretKeyRef resolves only against a local mirror. Skipped while
// the relay is not configured yet — the "configure email" screen refreshes
// both mirrors when it applies.
ensureSecretReplica(ctx, cl, controlNS, cfg.K8s.Namespace,
"felis-smtp", "password", "smtp", "minecraft ns"),
// The build namespace needs the same token: the build Job's fetch // The build namespace needs the same token: the build Job's fetch
// initContainer reads the submission context from the internal face. Best // initContainer reads the submission context from the internal face. Best
// effort — a deployment that only installs the control plane simply never // effort — a deployment that only installs the control plane simply never
+5 -4
View File
@@ -478,11 +478,12 @@ func phaseOrPending(p v1alpha1.Phase) string {
// beside the control plane — so without this replica the secretKeyRef would dangle and // beside the control plane — so without this replica the secretKeyRef would dangle and
// wedge the pod in CreateContainerConfigError. // wedge the pod in CreateContainerConfigError.
// //
// Two Secrets need it, for different reasons: the service token (the login limbo and // Three Secrets need it, for different reasons: the service token (the login limbo and
// the build Pod's context fetch — both authenticate to the felis-api internal face) // the build Pod's context fetch — both authenticate to the felis-api internal face),
// and the Velocity modern-forwarding secret (every backend — it is how a backend knows // the Velocity modern-forwarding secret (every backend — it is how a backend knows
// a login really came from the proxy, and so that the player's UUID is Mojang-verified // a login really came from the proxy, and so that the player's UUID is Mojang-verified
// rather than offline-derived). // rather than offline-derived), and the SMTP relay password (the reaper's pre-reap
// warning emails; the felis-config mirror is what carries [smtp] into its pod).
// //
// It is create-if-absent: an existing replica is left untouched so a hand-rotated // It is create-if-absent: an existing replica is left untouched so a hand-rotated
// value in the workload namespace is never clobbered (to rotate, delete the replica // value in the workload namespace is never clobbered (to rotate, delete the replica
+59 -6
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"os"
"strconv" "strconv"
"strings" "strings"
@@ -338,17 +339,23 @@ func applySMTPConfig(ctx context.Context, in smtpInputs) error {
if err := applyFelisConfigSecret(ctx); err != nil { if err := applyFelisConfigSecret(ctx); err != nil {
return err return err
} }
// Refresh the workload-namespace copies too (the reaper's warning path): the
// OTP path is already live in the control namespace, so a replica miss is
// reported but not fatal.
if err := replicateSMTPToWorkloadNamespace(ctx, in.password); err != nil {
fmt.Fprintf(os.Stderr, "felis setup: warning: email is configured, but refreshing the workload copies failed (pre-reap warning emails may stay suppressed): %v\n", err)
}
if err := kubectl(ctx, "-n", "felis", "rollout", "restart", "deployment/felis-api"); err != nil { if err := kubectl(ctx, "-n", "felis", "rollout", "restart", "deployment/felis-api"); err != nil {
return err return err
} }
return kubectl(ctx, "-n", "felis", "rollout", "status", "deployment/felis-api", "--timeout=180s") return kubectl(ctx, "-n", "felis", "rollout", "status", "deployment/felis-api", "--timeout=180s")
} }
// applySMTPSecret creates (or replaces) the felis-smtp Secret the felis-api // smtpSecretManifest renders the felis-smtp Secret (in the control namespace,
// Deployment injects the relay password from. Rendered in-process and piped to // via the caller's apply) the felis-api Deployment injects the relay password
// `kubectl apply` — the password is never a command-line arg, so it never // from. Rendered in-process and piped to `kubectl apply` — the password is
// appears in the host process table. // never a command-line arg, so it never appears in the host process table.
func applySMTPSecret(ctx context.Context, password string) error { func smtpSecretManifest(password string) ([]byte, error) {
secret := &corev1.Secret{ secret := &corev1.Secret{
TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"},
ObjectMeta: metav1.ObjectMeta{Name: platform.SMTPSecretName, Namespace: "felis"}, ObjectMeta: metav1.ObjectMeta{Name: platform.SMTPSecretName, Namespace: "felis"},
@@ -359,7 +366,53 @@ func applySMTPSecret(ctx context.Context, password string) error {
} }
manifest, err := yaml.Marshal(secret) manifest, err := yaml.Marshal(secret)
if err != nil { if err != nil {
return fmt.Errorf("render smtp secret: %w", err) return nil, fmt.Errorf("render smtp secret: %w", err)
}
return manifest, nil
}
func applySMTPSecret(ctx context.Context, password string) error {
manifest, err := smtpSecretManifest(password)
if err != nil {
return err
} }
return kubectlWithInput(ctx, manifest, "apply", "-f", "-") return kubectlWithInput(ctx, manifest, "apply", "-f", "-")
} }
// replicateSMTPToWorkloadNamespace refreshes the workload-namespace (minecraft)
// copies of felis-smtp and felis-config after email is reconfigured. The
// reaper's CronJob runs there and resolves both by local reference — a
// secretKeyRef is namespace-local, and `felis setup` creates the felis-config
// replica create-if-absent, so without this refresh a later SMTP change would
// never reach the pre-reap warning emails. Deliberately OVERWRITES both: these
// are mirrors of the control-namespace sources, and a stale mirror is exactly
// the failure this closes.
func replicateSMTPToWorkloadNamespace(ctx context.Context, password string) error {
cfg, err := config.Load(hostSetupConfigPath)
if err != nil {
return err
}
ns := cfg.K8s.Namespace
if ns == "" || ns == "felis" {
return nil
}
smtpManifest, err := smtpSecretManifest(password)
if err != nil {
return err
}
if err := kubectlWithInput(ctx, smtpManifest, "-n", ns, "apply", "-f", "-"); err != nil {
return fmt.Errorf("replicate %s to %s: %w", platform.SMTPSecretName, ns, err)
}
manifest, err := kubectlOutput(ctx,
"-n", ns, "create", "secret", "generic", "felis-config",
"--from-file=felis.toml="+podSetupConfigPath,
"--dry-run=client", "-o", "yaml",
)
if err != nil {
return fmt.Errorf("render felis-config for %s: %w", ns, err)
}
if err := kubectlWithInput(ctx, manifest, "-n", ns, "apply", "-f", "-"); err != nil {
return fmt.Errorf("replicate felis-config to %s: %w", ns, err)
}
return nil
}
+23
View File
@@ -504,6 +504,29 @@ So a missing backup never results in a deleted world. [GO-TESTED:
- CRD missing → logs `reaper: CRD missing, skipping`, skipped. - CRD missing → logs `reaper: CRD missing, skipping`, skipped.
- Idle `≤ 15d` → not yet eligible. - Idle `≤ 15d` → not yet eligible.
### Pre-reap warnings (the `warn_before` offsets)
An OWNED server inside a warning window gets an email notice (`3d`/`1d` before
the deadline, `warn_before` from `[archive]`) to the owner's **verified** email —
the same `[smtp]` relay felis-api uses. The `warned_3d_at` / `warned_1d_at`
stamps record a **delivered** notice:
- No `[smtp]` configured (or owner has no verified address): the run logs
`reaper: warning suppressed — no warner wired` / a delivery error and does
NOT stamp. Nothing is falsely recorded as sent, and the day SMTP is
configured the pending warning can still go out.
- Delivery failure (relay down): logged and retried on the next daily run —
bounded by the warning window, since the reap removes the candidate anyway.
- `warned=` in the run output counts DELIVERED notices, not attempts.
The reaper runs in the minecraft namespace and reads the **mirrors** of
`felis-smtp` and `felis-config` there (a `secretKeyRef` is namespace-local). The
installed `felis setup`'s "configure email" screen refreshes both mirrors when it
applies, so configuring SMTP after install is enough; a manual edit of the
control-namespace Secret alone is not. [GO-TESTED: the delivered/retried/
suppressed matrix in `internal/reaper`; live-drilled end to end against a local
SMTP sink.]
### Genuine false-delete risk vectors ### Genuine false-delete risk vectors
- **Stale `last_active_at`.** The keep-alive is `RecordJoin`, called from the - **Stale `last_active_at`.** The keep-alive is `RecordJoin`, called from the
+31 -2
View File
@@ -1,8 +1,9 @@
// Package mail is the SMTP implementation of the api.OTPMailer seam: it // Package mail is the SMTP implementation of the api.OTPMailer seam: it
// delivers the email one-time codes the passwordless doors mint (onboarding, // delivers the email one-time codes the passwordless doors mint (onboarding,
// email login, op-login) through the relay configured in felis.toml [smtp]. // email login, op-login) through the relay configured in felis.toml [smtp].
// It is deliberately tiny — one message shape, stdlib net/smtp — because the // It is deliberately tiny — two message shapes, stdlib net/smtp — because the
// only mail Felis ever sends is a six-digit code. // only mail Felis ever sends is a six-digit code plus the reaper's pre-deletion
// notice (SendNotice).
// //
// TLS posture: port 465 dials implicit TLS; any other port dials plaintext and // TLS posture: port 465 dials implicit TLS; any other port dials plaintext and
// upgrades via STARTTLS when the relay advertises it. AUTH is attempted only // upgrades via STARTTLS when the relay advertises it. AUTH is attempted only
@@ -53,6 +54,23 @@ func (s *SMTP) SendOTP(ctx context.Context, email, code string) error {
return c.Quit() return c.Quit()
} }
// SendNotice mails one operator-composed notice to email — the reaper's
// pre-deletion warning is its only caller. Subject and body are the caller's;
// the body is CRLF-normalized so a multi-line string renders as one text/plain
// message. Delivery errors surface exactly like SendOTP's, so the caller can
// retry on its own cadence.
func (s *SMTP) SendNotice(ctx context.Context, email, subject, body string) error {
c, err := s.connect(ctx)
if err != nil {
return err
}
defer c.Close()
if err := s.deliver(c, email, notice(s.From, email, subject, body, time.Now())); err != nil {
return err
}
return c.Quit()
}
// Ping proves the configured relay will actually ACCEPT mail from this sender, // Ping proves the configured relay will actually ACCEPT mail from this sender,
// by running a complete transaction — connect, (STARTTLS,) AUTH, MAIL FROM, // by running a complete transaction — connect, (STARTTLS,) AUTH, MAIL FROM,
// RCPT TO, DATA — and delivering a short self-test message to From itself. The // RCPT TO, DATA — and delivering a short self-test message to From itself. The
@@ -208,3 +226,14 @@ func selfTest(from string, now time.Time) []byte {
b.WriteString("Sent by `felis setup` when the SMTP relay was configured. / 由 `felis setup` 配置 SMTP 时发出。\r\n") b.WriteString("Sent by `felis setup` when the SMTP relay was configured. / 由 `felis setup` 配置 SMTP 时发出。\r\n")
return []byte(b.String()) return []byte(b.String())
} }
// notice renders an operator notice: the shared header block plus the caller's
// body, CRLF-normalized so every line obeys RFC 5322 regardless of which line
// endings the caller's format string produced.
func notice(from, to, subject, body string, now time.Time) []byte {
body = strings.ReplaceAll(strings.ReplaceAll(body, "\r\n", "\n"), "\n", "\r\n")
if !strings.HasSuffix(body, "\r\n") {
body += "\r\n"
}
return []byte(headers(from, to, subject, now) + body)
}
+27
View File
@@ -58,6 +58,33 @@ func TestSelfTestCarriesNoCode(t *testing.T) {
} }
} }
// TestNoticeShape pins the second message shape — the reaper's pre-deletion
// warning: CRLF throughout even when the caller's body used bare LFs, a
// Q-encoded subject when it carries non-ASCII, and the caller's text rendered
// verbatim between the header block and the wire.
func TestNoticeShape(t *testing.T) {
now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC)
msg := string(notice("[email protected]", "[email protected]",
"Felis: 服务器 survival 将回收", "line one\nline two\n", now))
if strings.Contains(strings.ReplaceAll(msg, "\r\n", ""), "\n") {
t.Error("notice contains a bare LF; every line must end CRLF")
}
headers, body, ok := strings.Cut(msg, "\r\n\r\n")
if !ok {
t.Fatal("notice has no blank line between headers and body")
}
if !strings.Contains(headers, "To: [email protected]") {
t.Errorf("headers missing To:\n%s", headers)
}
if !strings.Contains(headers, "Subject: =?utf-8?") {
t.Errorf("non-ASCII subject must be Q-encoded:\n%s", headers)
}
if !strings.Contains(body, "line one\r\nline two\r\n") {
t.Errorf("body must be CRLF-normalized verbatim text:\n%q", body)
}
}
// fakeRelay speaks just enough SMTP for net/smtp, answering 250 to MAIL FROM // fakeRelay speaks just enough SMTP for net/smtp, answering 250 to MAIL FROM
// and RCPT TO but dataVerdict at end-of-DATA. That split is the entire point: // and RCPT TO but dataVerdict at end-of-DATA. That split is the entire point:
// relays which validate sender identity (Fastmail among them) accept MAIL FROM // relays which validate sender identity (Fastmail among them) accept MAIL FROM
+14
View File
@@ -536,6 +536,20 @@ func reaperCronJob(p Params) *batchv1.CronJob {
"--config", configFilePath, "--config", configFilePath,
"--worlds-root", worldsMountPath, "--worlds-root", worldsMountPath,
}, },
// The [smtp] relay password for pre-reap warning emails — same optional
// Secret felis-api reads. Namespace caveat: a secretKeyRef is
// namespace-local, so this resolves against the minecraft-ns felis-smtp
// mirror that the "configure email" screen refreshes (the felis-config
// mirror it also refreshes is what puts [smtp] in this pod's config).
// Absent Secret ⇒ empty env ⇒ the reaper logs suppressed warnings
// instead of stamping them (never a failed pod).
Env: []corev1.EnvVar{
{Name: SMTPPasswordEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: SMTPSecretName},
Key: SMTPSecretPasswordKey,
Optional: boolPtr(true),
}}},
},
VolumeMounts: []corev1.VolumeMount{ VolumeMounts: []corev1.VolumeMount{
{Name: configVolume, MountPath: configMountPath, ReadOnly: true}, {Name: configVolume, MountPath: configMountPath, ReadOnly: true},
{Name: worldsVolume, MountPath: worldsMountPath, ReadOnly: true}, {Name: worldsVolume, MountPath: worldsMountPath, ReadOnly: true},
+13
View File
@@ -729,6 +729,19 @@ func TestReaperCronJob_Shape(t *testing.T) {
t.Errorf("reaper image = %q, want FelisImage %q", c.Image, p.FelisImage) t.Errorf("reaper image = %q, want FelisImage %q", c.Image, p.FelisImage)
} }
// The relay password for pre-reap warning emails: same optional Secret as
// felis-api, resolved against the minecraft-ns mirror. Optional so an install
// without SMTP still starts (the reaper then logs suppressed warnings).
smtpEnv := envVar(c.Env, SMTPPasswordEnv)
if smtpEnv == nil || smtpEnv.ValueFrom == nil || smtpEnv.ValueFrom.SecretKeyRef == nil {
t.Fatalf("reaper must wire %s from a secretKeyRef", SMTPPasswordEnv)
}
if ref := smtpEnv.ValueFrom.SecretKeyRef; ref.Name != SMTPSecretName || ref.Key != SMTPSecretPasswordKey {
t.Errorf("reaper %s ref = %s/%s, want %s/%s", SMTPPasswordEnv, ref.Name, ref.Key, SMTPSecretName, SMTPSecretPasswordKey)
} else if ref.Optional == nil || !*ref.Optional {
t.Errorf("reaper %s secretKeyRef must be optional", SMTPPasswordEnv)
}
// config: Secret, mounted read-only (it carries the DB URL). // config: Secret, mounted read-only (it carries the DB URL).
cfgVol := volumeByName(ps.Volumes, configVolume) cfgVol := volumeByName(ps.Volumes, configVolume)
if cfgVol == nil || cfgVol.Secret == nil || cfgVol.Secret.SecretName != configSecretName { if cfgVol == nil || cfgVol.Secret == nil || cfgVol.Secret.SecretName != configSecretName {
+24 -8
View File
@@ -203,7 +203,10 @@ type Cluster interface {
} }
// Warner delivers an impending-reap notice. It is optional and best-effort: a // Warner delivers an impending-reap notice. It is optional and best-effort: a
// nil Warner or a delivery error never blocks a reap (red line ⑤). // nil Warner or a delivery error never blocks a reap (red line ⑤). Warn returns
// nil only when the notice was handed to the delivery channel; an error (or a
// nil Warner) leaves warned_* unstamped, so the next daily run retries instead
// of silently burning the owner's only warning.
type Warner interface { type Warner interface {
Warn(ctx context.Context, ownerID, server, remaining string) error Warn(ctx context.Context, ownerID, server, remaining string) error
} }
@@ -433,9 +436,11 @@ func (r *Reaper) ensureCapacity(ctx context.Context, now time.Time, sum *Summary
// maybeWarn sends at most one impending-reap notice per run, honoring §18's // maybeWarn sends at most one impending-reap notice per run, honoring §18's
// elif precedence (earliest unsent warning first). Unowned servers are never // elif precedence (earliest unsent warning first). Unowned servers are never
// warned but are still reaped at the deadline (red line ⑤). A warner delivery // warned but are still reaped at the deadline (red line ⑤). The warned_* stamp
// failure is logged but the warned_* stamp still advances so the notice is not // records a DELIVERED notice: a nil Warner or a delivery error is logged and
// retried forever; a real join (RecordJoin) is what clears the stamps. // leaves the stamp untouched, so the next run retries — bounded by the warning
// window, since the reap itself removes the candidate. A real join (RecordJoin)
// clears the stamps when a player renews.
func (r *Reaper) maybeWarn(ctx context.Context, now time.Time, idle time.Duration, offs []time.Duration, c Candidate, sum *Summary) { func (r *Reaper) maybeWarn(ctx context.Context, now time.Time, idle time.Duration, offs []time.Duration, c Candidate, sum *Summary) {
if c.OwnerID == "" { if c.OwnerID == "" {
return return
@@ -449,10 +454,21 @@ func (r *Reaper) maybeWarn(ctx context.Context, now time.Time, idle time.Duratio
if !c.warnedAt(tier).IsZero() { if !c.warnedAt(tier).IsZero() {
continue // already sent this tier continue // already sent this tier
} }
if r.Warner != nil { if r.Warner == nil {
if err := r.Warner.Warn(ctx, c.OwnerID, c.Name, formatRemaining(offs[i])); err != nil { // No delivery channel is wired at all. Do not stamp: an operator who
r.log().Warn("reaper: warn delivery failed (best-effort)", "server", c.Name, "err", err) // wires one later must still be able to warn, and a stamp here would
} // have recorded a notice nobody received. Logged every run so silence
// is never mistaken for delivery.
r.log().Warn("reaper: warning suppressed — no warner wired",
"server", c.Name, "owner", c.OwnerID, "remaining", formatRemaining(offs[i]))
return
}
if err := r.Warner.Warn(ctx, c.OwnerID, c.Name, formatRemaining(offs[i])); err != nil {
// Best-effort: the reap still proceeds on schedule, but the stamp
// stays empty so the next daily run retries the delivery instead of
// permanently suppressing the owner's only notice.
r.log().Warn("reaper: warn delivery failed; will retry next run", "server", c.Name, "err", err)
return
} }
if err := r.Store.MarkWarned(ctx, c.Name, tier, now); err != nil { if err := r.Store.MarkWarned(ctx, c.Name, tier, now); err != nil {
r.log().Error("reaper: mark warned failed", "server", c.Name, "err", err) r.log().Error("reaper: mark warned failed", "server", c.Name, "err", err)
+49 -7
View File
@@ -472,6 +472,8 @@ func TestWarningsDerivedFromNonDefaultDeadline(t *testing.T) {
// past 7d but unowned -> never warned (red line ⑤) // past 7d but unowned -> never warned (red line ⑤)
Candidate{Name: "e", OwnerID: "", LastActiveAt: idleBy(8 * Day)}, Candidate{Name: "e", OwnerID: "", LastActiveAt: idleBy(8 * Day)},
) )
rw := &recordingWarner{}
r.Warner = rw
sum := mustRun(t, r) sum := mustRun(t, r)
if sum.WorldsReaped != 0 { if sum.WorldsReaped != 0 {
@@ -480,6 +482,9 @@ func TestWarningsDerivedFromNonDefaultDeadline(t *testing.T) {
if sum.Warned != 2 { if sum.Warned != 2 {
t.Fatalf("Warned = %d, want 2 (a:3d, b:1d)", sum.Warned) t.Fatalf("Warned = %d, want 2 (a:3d, b:1d)", sum.Warned)
} }
if len(rw.sent) != 2 {
t.Fatalf("deliveries = %d, want 2 (a:3d, b:1d)", len(rw.sent))
}
if cl.deletePVCCalls != 0 { if cl.deletePVCCalls != 0 {
t.Fatalf("a warning path deleted a PVC") t.Fatalf("a warning path deleted a PVC")
} }
@@ -494,20 +499,48 @@ func TestWarningsDerivedFromNonDefaultDeadline(t *testing.T) {
} }
} }
// Red line ⑤ (best-effort): a Warner delivery error does not abort the run, and // Red line ⑤ (best-effort) with delivery honesty: a Warner failure does not
// the warned_* stamp still advances (a real join, not a failed warn, is what // abort the run, and it does NOT stamp — the stamp records a DELIVERED notice,
// resets the clock). // so the next daily run retries (the warning window bounds the retries, and the
func TestWarningBestEffortOnDeliveryFailure(t *testing.T) { // reap clears the candidate either way).
func TestWarningDeliveryRetriedAfterFailure(t *testing.T) {
r, st, _, _ := newReaper(DefaultConfig(), r, st, _, _ := newReaper(DefaultConfig(),
Candidate{Name: "h", OwnerID: "u-h", LastActiveAt: idleBy(13 * Day)}) Candidate{Name: "h", OwnerID: "u-h", LastActiveAt: idleBy(13 * Day)})
r.Warner = failWarner{} r.Warner = failWarner{}
sum := mustRun(t, r) sum := mustRun(t, r)
if sum.Warned != 1 { if sum.Warned != 0 {
t.Fatalf("Warned = %d, want 1 despite delivery failure", sum.Warned) t.Fatalf("Warned = %d, want 0 (nothing was delivered)", sum.Warned)
}
if !st.byName["h"].Warned3dAt.IsZero() {
t.Fatal("a failed delivery must not stamp warned_3d_at")
}
// Next run with a working channel: the SAME warning goes out and stamps.
rw := &recordingWarner{}
r.Warner = rw
sum = mustRun(t, r)
if sum.Warned != 1 || len(rw.sent) != 1 {
t.Fatalf("retry: Warned=%d sent=%d, want 1/1", sum.Warned, len(rw.sent))
} }
if st.byName["h"].Warned3dAt.IsZero() { if st.byName["h"].Warned3dAt.IsZero() {
t.Fatalf("warned_3d_at not stamped after best-effort warn") t.Fatal("a delivered warning must stamp warned_3d_at")
}
}
// A nil Warner suppresses the warning WITHOUT stamping it: nothing was sent, so
// nothing is recorded as sent — and the day a channel is wired, the owner can
// still be warned.
func TestWarningSuppressedWithoutWarner(t *testing.T) {
r, st, _, _ := newReaper(DefaultConfig(),
Candidate{Name: "n", OwnerID: "u-n", LastActiveAt: idleBy(13 * Day)})
sum := mustRun(t, r)
if sum.Warned != 0 {
t.Fatalf("Warned = %d, want 0 with no warner wired", sum.Warned)
}
if !st.byName["n"].Warned3dAt.IsZero() || !st.byName["n"].Warned1dAt.IsZero() {
t.Fatal("a suppressed warning must not stamp either tier")
} }
} }
@@ -517,6 +550,15 @@ func (failWarner) Warn(context.Context, string, string, string) error {
return errors.New("smtp unavailable") return errors.New("smtp unavailable")
} }
// recordingWarner captures deliveries so the threshold tests exercise the real
// deliver-then-stamp path.
type recordingWarner struct{ sent []string }
func (w *recordingWarner) Warn(_ context.Context, ownerID, server, remaining string) error {
w.sent = append(w.sent, ownerID+"/"+server+"/"+remaining)
return nil
}
// §26 capacity: when the store is over its cap, the oldest backup is evicted // §26 capacity: when the store is over its cap, the oldest backup is evicted
// early (destructive — audited) to make room, then the reap proceeds. // early (destructive — audited) to make room, then the reap proceeds.
func TestCapacityEvictsOldestThenReaps(t *testing.T) { func TestCapacityEvictsOldestThenReaps(t *testing.T) {