feat(retention): felis-api 定时清理过期会话、验证码、挑战、绑定码等表，审计按 [audit] retention 保留，新增 felis db audit-export 导出归档
This commit is contained in:
9 files changed
+695
-21
No files matched your search
@@ -9,7 +9,9 @@ import (
|
||||
"net"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
)
|
||||
@@ -25,6 +27,7 @@ type Config struct {
|
||||
Archive ArchiveConfig `toml:"archive"`
|
||||
Offsite OffsiteConfig `toml:"offsite"`
|
||||
SMTP SMTPConfig `toml:"smtp"`
|
||||
Audit AuditConfig `toml:"audit"`
|
||||
// AuthSources is the [[auth_source]] array-of-tables: the third-party Yggdrasil
|
||||
// roots the Felis-nano hasJoined multiplexer federates over, in priority order
|
||||
// (config order = priority, so array-of-tables not a map — a map would lose order
|
||||
@@ -256,6 +259,63 @@ type ArchiveS3Config struct {
|
||||
SecretKeyRef string `toml:"secret_key_ref"`
|
||||
}
|
||||
|
||||
// AuditConfig is the [audit] table. Retention is how long felis-api keeps audit
|
||||
// rows before deleting them ("365d", "18mo", or "forever" to keep every row);
|
||||
// empty means DefaultAuditRetention. Export what must outlive it with
|
||||
// `felis db audit-export` first.
|
||||
type AuditConfig struct {
|
||||
Retention string `toml:"retention"`
|
||||
}
|
||||
|
||||
// DefaultAuditRetention keeps a year of audit rows; MinAuditRetention is the
|
||||
// shortest an install may set, since the manual-backup cooldown and an incident
|
||||
// investigation both read recent rows.
|
||||
const (
|
||||
DefaultAuditRetention = 365 * 24 * time.Hour
|
||||
MinAuditRetention = 30 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// RetentionPeriod resolves Retention: 0 keeps every row.
|
||||
func (a AuditConfig) RetentionPeriod() (time.Duration, error) {
|
||||
switch v := strings.TrimSpace(a.Retention); v {
|
||||
case "":
|
||||
return DefaultAuditRetention, nil
|
||||
case "forever":
|
||||
return 0, nil
|
||||
default:
|
||||
d, err := ParseSpan(v)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("config: [audit] retention %q must be a span such as 365d or 18mo, or forever", a.Retention)
|
||||
}
|
||||
if d < MinAuditRetention {
|
||||
return 0, fmt.Errorf("config: [audit] retention %q is shorter than the 30d minimum", a.Retention)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
}
|
||||
|
||||
// ParseSpan parses the human spans felis.toml uses for retention periods:
|
||||
// "3mo" (months of 30 days), "15d" (days), or any time.ParseDuration unit ("12h").
|
||||
func ParseSpan(s string) (time.Duration, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
switch {
|
||||
case strings.HasSuffix(s, "mo"):
|
||||
n, err := strconv.Atoi(strings.TrimSuffix(s, "mo"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return time.Duration(n) * 30 * 24 * time.Hour, nil
|
||||
case strings.HasSuffix(s, "d"):
|
||||
n, err := strconv.Atoi(strings.TrimSuffix(s, "d"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return time.Duration(n) * 24 * time.Hour, nil
|
||||
default:
|
||||
return time.ParseDuration(s)
|
||||
}
|
||||
}
|
||||
|
||||
// OffsiteConfig is the [offsite] table: the S3-compatible bucket, away from
|
||||
// this machine, that `felis offsite sync` (felis-offsite.timer on the host)
|
||||
// copies every world archive and the newest database bundles into, encrypted
|
||||
@@ -488,6 +548,9 @@ func (c *Config) Validate() error {
|
||||
if err := c.Offsite.validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := c.Audit.RetentionPeriod(); err != nil {
|
||||
return err
|
||||
}
|
||||
if h := c.Auth.ClientIPHeader; strings.ContainsAny(h, " :\t\r\n") {
|
||||
return fmt.Errorf("config: [auth] client_ip_header %q must be a bare header name such as CF-Connecting-IP or X-Forwarded-For", h)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/config"
|
||||
)
|
||||
@@ -695,3 +696,50 @@ prefix = "site-a"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAuditRetention: [audit] retention resolves to a year when unset, to 0 for
|
||||
// "forever", and to the named span otherwise; a span under 30 days or one that
|
||||
// does not parse fails at load.
|
||||
func TestAuditRetention(t *testing.T) {
|
||||
const head = `
|
||||
[server]
|
||||
root_domain = "mc.example.net"
|
||||
[database]
|
||||
url = "postgres://felis@db/felis"
|
||||
`
|
||||
const day = 24 * time.Hour
|
||||
for _, tc := range []struct {
|
||||
name, table string
|
||||
want time.Duration
|
||||
}{
|
||||
{"unset", "", 365 * day},
|
||||
{"empty", "[audit]\nretention = \"\"\n", 365 * day},
|
||||
{"forever", "[audit]\nretention = \"forever\"\n", 0},
|
||||
{"days", "[audit]\nretention = \"90d\"\n", 90 * day},
|
||||
{"months", "[audit]\nretention = \"18mo\"\n", 540 * day},
|
||||
{"hours", "[audit]\nretention = \"720h\"\n", 30 * day},
|
||||
{"exactly the minimum", "[audit]\nretention = \"30d\"\n", 30 * day},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg, err := config.Load(writeTOML(t, head+tc.table))
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
got, err := cfg.Audit.RetentionPeriod()
|
||||
if err != nil || got != tc.want {
|
||||
t.Fatalf("RetentionPeriod = %v, %v; want %v", got, err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, tc := range []struct{ value, wantErr string }{
|
||||
{"29d", `config: [audit] retention "29d" is shorter than the 30d minimum`},
|
||||
{"719h", `config: [audit] retention "719h" is shorter than the 30d minimum`},
|
||||
{"a year", `config: [audit] retention "a year" must be a span such as 365d or 18mo, or forever`},
|
||||
{"Forever", `config: [audit] retention "Forever" must be a span such as 365d or 18mo, or forever`},
|
||||
} {
|
||||
_, err := config.Load(writeTOML(t, head+"[audit]\nretention = \""+tc.value+"\"\n"))
|
||||
if err == nil || err.Error() != tc.wantErr {
|
||||
t.Errorf("retention %q: err = %v, want %q", tc.value, err, tc.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
//go:build pgint
|
||||
|
||||
package pgint
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/retention"
|
||||
)
|
||||
|
||||
func mustExec(t *testing.T, q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := db.Exec(q, args...); err != nil {
|
||||
t.Fatalf("%s: %v", q, err)
|
||||
}
|
||||
}
|
||||
|
||||
func rowExists(t *testing.T, q string, args ...any) bool {
|
||||
t.Helper()
|
||||
var ok bool
|
||||
if err := db.QueryRow(`SELECT EXISTS (`+q+`)`, args...).Scan(&ok); err != nil {
|
||||
t.Fatalf("%s: %v", q, err)
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
// Prune deletes each sign-in row a month after it was spent (expired, used or
|
||||
// revoked) and keeps the ones spent more recently and the live ones; audit rows
|
||||
// go once they are older than the policy's retention.
|
||||
func TestPruneDropsRowsSpentLongerThanGrace(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := mustNow()
|
||||
old := now.Add(-31 * 24 * time.Hour) // spent past the 30-day grace
|
||||
recent := now.Add(-29 * 24 * time.Hour) // spent inside it
|
||||
future := now.Add(time.Hour)
|
||||
u := newUser(t, "user", "prune")
|
||||
src1, src2, src3 := newUser(t, "user", "mig1"), newUser(t, "user", "mig2"), newUser(t, "user", "mig3")
|
||||
tag := suffix(t)
|
||||
id := func(s string) string { return s + "-" + tag }
|
||||
|
||||
for _, r := range []struct {
|
||||
name string
|
||||
expires, used, rev any
|
||||
}{
|
||||
{"s-expired-old", old, nil, nil},
|
||||
{"s-revoked-old", future, nil, old},
|
||||
{"s-expired-recent", recent, nil, nil},
|
||||
{"s-revoked-recent", future, nil, recent},
|
||||
{"s-live", future, nil, nil},
|
||||
} {
|
||||
mustExec(t, `INSERT INTO sessions (token_hash, user_id, expires_at, revoked_at) VALUES ($1, $2, $3, $4)`,
|
||||
id(r.name), u.ID, r.expires, r.rev)
|
||||
}
|
||||
for _, r := range []struct {
|
||||
name string
|
||||
expires, used any
|
||||
}{
|
||||
{"expired-old", old, nil},
|
||||
{"used-old", future, old},
|
||||
{"expired-recent", recent, nil},
|
||||
{"used-recent", future, recent},
|
||||
{"live", future, nil},
|
||||
} {
|
||||
mustExec(t, `INSERT INTO email_otps (id, user_id, email, code_hash, purpose, expires_at, consumed_at)
|
||||
VALUES ($1, $2, '[email protected]', 'h', 'login_email', $3, $4)`, id("otp-"+r.name), u.ID, r.expires, r.used)
|
||||
mustExec(t, `INSERT INTO webauthn_challenges (id, user_id, purpose, session_data, expires_at, consumed_at)
|
||||
VALUES ($1, $2, 'passkey_register', '\x00', $3, $4)`, id("wc-"+r.name), u.ID, r.expires, r.used)
|
||||
mustExec(t, `INSERT INTO webauthn_discoverable_challenges (id, session_data, expires_at, consumed_at)
|
||||
VALUES ($1, '\x00', $2, $3)`, id("wd-"+r.name), r.expires, r.used)
|
||||
mustExec(t, `INSERT INTO setup_tokens (token_hash, user_id, expires_at, consumed_at) VALUES ($1, $2, $3, $4)`,
|
||||
id("st-"+r.name), u.ID, r.expires, r.used)
|
||||
mustExec(t, `INSERT INTO op_login_requests (id, user_id, email, expires_at, consumed_at)
|
||||
VALUES ($1, $2, '[email protected]', $3, $4)`, id("op-"+r.name), u.ID, r.expires, r.used)
|
||||
}
|
||||
for _, r := range []struct {
|
||||
name string
|
||||
expires time.Time
|
||||
}{{"lc-old", old}, {"lc-recent", recent}, {"lc-live", future}} {
|
||||
mustExec(t, `INSERT INTO account_link_codes (code, mc_uuid, expires_at) VALUES ($1, $2, $3)`,
|
||||
id(r.name), testUUID(t), r.expires)
|
||||
}
|
||||
// A migration abandoned a month ago goes; one touched recently stays, and a
|
||||
// redeemed one is the record of the move and stays however old it is.
|
||||
mustExec(t, `INSERT INTO account_migrations (id, source_user_id, state, created_at, updated_at) VALUES ($1, $2, 'initiated', $3, $3)`,
|
||||
id("mig-stale"), src1.ID, old)
|
||||
mustExec(t, `INSERT INTO account_migrations (id, source_user_id, state, created_at, updated_at) VALUES ($1, $2, 'confirmed', $3, $4)`,
|
||||
id("mig-fresh"), src2.ID, old, recent)
|
||||
mustExec(t, `INSERT INTO account_migrations (id, source_user_id, state, redeemed_at, created_at, updated_at) VALUES ($1, $2, 'redeemed', $3, $3, $3)`,
|
||||
id("mig-redeemed"), src3.ID, old)
|
||||
mustExec(t, `INSERT INTO otp_failure_windows (user_id, purpose, window_start, failures) VALUES ($1, 'login_email', $2, 3)`, u.ID, old)
|
||||
mustExec(t, `INSERT INTO otp_failure_windows (user_id, purpose, window_start, failures) VALUES ($1, 'onboard_email', $2, 3)`, u.ID, recent)
|
||||
mustExec(t, `INSERT INTO audit_logs (actor, source, action, request_id, created_at) VALUES ('pgint', 'external', 'prune.test', $1, $2)`,
|
||||
id("audit-61d"), now.Add(-61*24*time.Hour))
|
||||
mustExec(t, `INSERT INTO audit_logs (actor, source, action, request_id, created_at) VALUES ('pgint', 'external', 'prune.test', $1, $2)`,
|
||||
id("audit-59d"), now.Add(-59*24*time.Hour))
|
||||
|
||||
if _, err := retention.Prune(ctx, db, now, retention.Policy{Audit: 60 * 24 * time.Hour}); err != nil {
|
||||
t.Fatalf("Prune: %v", err)
|
||||
}
|
||||
|
||||
for _, c := range []struct {
|
||||
q, key string
|
||||
want bool
|
||||
}{
|
||||
{`SELECT 1 FROM sessions WHERE token_hash = $1`, "s-expired-old", false},
|
||||
{`SELECT 1 FROM sessions WHERE token_hash = $1`, "s-revoked-old", false},
|
||||
{`SELECT 1 FROM sessions WHERE token_hash = $1`, "s-expired-recent", true},
|
||||
{`SELECT 1 FROM sessions WHERE token_hash = $1`, "s-revoked-recent", true},
|
||||
{`SELECT 1 FROM sessions WHERE token_hash = $1`, "s-live", true},
|
||||
{`SELECT 1 FROM account_link_codes WHERE code = $1`, "lc-old", false},
|
||||
{`SELECT 1 FROM account_link_codes WHERE code = $1`, "lc-recent", true},
|
||||
{`SELECT 1 FROM account_link_codes WHERE code = $1`, "lc-live", true},
|
||||
{`SELECT 1 FROM account_migrations WHERE id = $1`, "mig-stale", false},
|
||||
{`SELECT 1 FROM account_migrations WHERE id = $1`, "mig-fresh", true},
|
||||
{`SELECT 1 FROM account_migrations WHERE id = $1`, "mig-redeemed", true},
|
||||
{`SELECT 1 FROM audit_logs WHERE request_id = $1`, "audit-61d", false},
|
||||
{`SELECT 1 FROM audit_logs WHERE request_id = $1`, "audit-59d", true},
|
||||
} {
|
||||
if got := rowExists(t, c.q, id(c.key)); got != c.want {
|
||||
t.Errorf("%s: present = %v, want %v", c.key, got, c.want)
|
||||
}
|
||||
}
|
||||
for _, table := range []struct{ name, q, prefix string }{
|
||||
{"email_otps", `SELECT 1 FROM email_otps WHERE id = $1`, "otp-"},
|
||||
{"webauthn_challenges", `SELECT 1 FROM webauthn_challenges WHERE id = $1`, "wc-"},
|
||||
{"webauthn_discoverable_challenges", `SELECT 1 FROM webauthn_discoverable_challenges WHERE id = $1`, "wd-"},
|
||||
{"setup_tokens", `SELECT 1 FROM setup_tokens WHERE token_hash = $1`, "st-"},
|
||||
{"op_login_requests", `SELECT 1 FROM op_login_requests WHERE id = $1`, "op-"},
|
||||
} {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
want bool
|
||||
}{{"expired-old", false}, {"used-old", false}, {"expired-recent", true}, {"used-recent", true}, {"live", true}} {
|
||||
if got := rowExists(t, table.q, id(table.prefix+c.name)); got != c.want {
|
||||
t.Errorf("%s %s: present = %v, want %v", table.name, c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
if rowExists(t, `SELECT 1 FROM otp_failure_windows WHERE user_id = $1 AND purpose = 'login_email'`, u.ID) {
|
||||
t.Error("an OTP failure window that began a month ago survived")
|
||||
}
|
||||
if !rowExists(t, `SELECT 1 FROM otp_failure_windows WHERE user_id = $1 AND purpose = 'onboard_email'`, u.ID) {
|
||||
t.Error("an OTP failure window inside the grace was deleted")
|
||||
}
|
||||
}
|
||||
|
||||
// With no audit retention every audit row stays, and a table with more spent rows
|
||||
// than one DELETE takes is emptied of them in the same run.
|
||||
func TestPruneKeepsAuditForeverAndDrainsLargeBacklogs(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := mustNow()
|
||||
tag := suffix(t)
|
||||
mustExec(t, `INSERT INTO audit_logs (actor, source, action, request_id, created_at) VALUES ('pgint', 'external', 'prune.test', $1, $2)`,
|
||||
"ancient-"+tag, now.Add(-3000*24*time.Hour))
|
||||
mustExec(t, `INSERT INTO account_link_codes (code, mc_uuid, expires_at)
|
||||
SELECT $1 || n, gen_random_uuid(), $2 FROM generate_series(1, 5003) AS n`, "bulk-"+tag+"-", now.Add(-40*24*time.Hour))
|
||||
|
||||
res, err := retention.Prune(ctx, db, now, retention.Policy{})
|
||||
if err != nil {
|
||||
t.Fatalf("Prune: %v", err)
|
||||
}
|
||||
if !rowExists(t, `SELECT 1 FROM audit_logs WHERE request_id = $1`, "ancient-"+tag) {
|
||||
t.Error("an audit row was deleted with retention off")
|
||||
}
|
||||
if _, ok := res["audit_logs"]; ok {
|
||||
t.Errorf("audit_logs was pruned with retention off: %v", res)
|
||||
}
|
||||
var left int
|
||||
if err := db.QueryRow(`SELECT count(*) FROM account_link_codes WHERE code LIKE $1`, "bulk-"+tag+"-%").Scan(&left); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if left != 0 {
|
||||
t.Errorf("%d of 5003 spent bind codes left after one run", left)
|
||||
}
|
||||
if res["account_link_codes"] < 5003 {
|
||||
t.Errorf("account_link_codes deleted = %d, want at least 5003", res["account_link_codes"])
|
||||
}
|
||||
}
|
||||
|
||||
var idRE = regexp.MustCompile(`"id":\d+`)
|
||||
|
||||
// ExportAudit writes the rows inside [since, until) as JSON lines, oldest first,
|
||||
// with the optional columns left out when empty.
|
||||
func TestExportAuditWritesTheWindow(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
u := newUser(t, "user", "export")
|
||||
tag := suffix(t)
|
||||
base := time.Date(2031, 3, 4, 0, 0, 0, 0, time.UTC)
|
||||
mustExec(t, `INSERT INTO audit_logs (actor, source, action, request_id, created_at) VALUES ('before', 'external', 'x.before', $1, $2)`,
|
||||
"exp-"+tag, base.Add(-time.Second))
|
||||
mustExec(t, `INSERT INTO audit_logs (actor, actor_user_id, source, action, server_name, request_id, client_ip, user_agent, payload, created_at)
|
||||
VALUES ('[email protected]', $1, 'external', 'server.patch', 'survival', $2, '192.0.2.7', 'curl/8', '{"display_name":"Survival"}', $3)`,
|
||||
u.ID, "exp-"+tag, base.Add(2*time.Hour))
|
||||
mustExec(t, `INSERT INTO audit_logs (actor, source, action, created_at) VALUES ('internal', 'internal:velocity', 'server.wake', $1)`,
|
||||
base.Add(time.Hour))
|
||||
mustExec(t, `INSERT INTO audit_logs (actor, source, action, request_id, created_at) VALUES ('after', 'external', 'x.after', $1, $2)`,
|
||||
"exp-"+tag, base.Add(24*time.Hour))
|
||||
|
||||
var buf bytes.Buffer
|
||||
n, err := retention.ExportAudit(ctx, db, base, base.Add(24*time.Hour), &buf)
|
||||
if err != nil {
|
||||
t.Fatalf("ExportAudit: %v", err)
|
||||
}
|
||||
// Row ids are serial; blank them to compare the rest literally.
|
||||
got := idRE.ReplaceAllString(buf.String(), `"id":0`)
|
||||
want := `{"id":0,"created_at":"2031-03-04T01:00:00Z","actor":"internal","source":"internal:velocity","action":"server.wake"}` + "\n" +
|
||||
`{"id":0,"created_at":"2031-03-04T02:00:00Z","actor":"[email protected]","actor_user_id":"` + u.ID + `","source":"external","action":"server.patch","server_name":"survival","request_id":"exp-` + tag + `","client_ip":"192.0.2.7","user_agent":"curl/8","payload":{"display_name":"Survival"}}` + "\n"
|
||||
if n != 2 || got != want {
|
||||
t.Fatalf("ExportAudit wrote %d rows:\n%s\nwant 2:\n%s", n, got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
// Package retention deletes the control-plane rows that stopped mattering, so
|
||||
// the sign-in tables and the audit trail do not grow for as long as an install
|
||||
// runs.
|
||||
//
|
||||
// Every sign-in door leaves a row behind: a session per login, an email code or
|
||||
// passkey challenge per attempt, a bind code per /felis link, an op-login request
|
||||
// per staff sign-in, a setup token per invite. Each is spent once it expires,
|
||||
// is used, or is revoked, and a spent row only holds a hash nothing accepts any
|
||||
// more. Prune drops those rows Grace after they were spent: long enough that an
|
||||
// incident can still line up a session's device and address with the audit
|
||||
// trail, short enough that the tables stay the size of what is live.
|
||||
//
|
||||
// Audit rows are kept for the install's [audit] retention and then deleted, oldest
|
||||
// first. `felis db audit-export` (ExportAudit) writes the rows to a file first
|
||||
// for an install that must keep them longer.
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Grace is how long a spent sign-in row outlives the moment it was spent.
|
||||
const Grace = 30 * 24 * time.Hour
|
||||
|
||||
// batch bounds one DELETE, so a first run on an install that never pruned stays
|
||||
// clear of the pool's statement timeout and holds its row locks briefly.
|
||||
const batch = 5000
|
||||
|
||||
// sweep is one table's spent rows: every statement takes the cutoff as $1 and a
|
||||
// row limit as $2, and deletes at most $2 rows spent before the cutoff.
|
||||
type sweep struct {
|
||||
table string
|
||||
where string
|
||||
}
|
||||
|
||||
// sweeps name each sign-in table's spent rows. A session is spent once it
|
||||
// expired or was revoked; a single-use code or challenge once it expired or was
|
||||
// redeemed; a bind code once it expired (redeeming deletes it); a migration that
|
||||
// never completed once nobody touched it for Grace (a redeemed one is the record
|
||||
// of which account moved where and is kept); an OTP failure window once it began
|
||||
// Grace ago (a window lasts a day).
|
||||
var sweeps = []sweep{
|
||||
{"sessions", "expires_at < $1 OR revoked_at < $1"},
|
||||
{"email_otps", "expires_at < $1 OR consumed_at < $1"},
|
||||
{"webauthn_challenges", "expires_at < $1 OR consumed_at < $1"},
|
||||
{"webauthn_discoverable_challenges", "expires_at < $1 OR consumed_at < $1"},
|
||||
{"setup_tokens", "expires_at < $1 OR consumed_at < $1"},
|
||||
{"op_login_requests", "expires_at < $1 OR consumed_at < $1"},
|
||||
{"account_link_codes", "expires_at < $1"},
|
||||
{"account_migrations", "state <> 'redeemed' AND updated_at < $1"},
|
||||
{"otp_failure_windows", "window_start < $1"},
|
||||
}
|
||||
|
||||
// Policy is what Prune keeps.
|
||||
type Policy struct {
|
||||
// Audit is how long audit rows are kept; 0 keeps every row.
|
||||
Audit time.Duration
|
||||
}
|
||||
|
||||
// Result counts the rows Prune deleted, by table.
|
||||
type Result map[string]int64
|
||||
|
||||
// Total is the number of rows deleted across all tables.
|
||||
func (r Result) Total() int64 {
|
||||
var n int64
|
||||
for _, v := range r {
|
||||
n += v
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Prune deletes the spent sign-in rows and the audit rows past the policy's
|
||||
// retention. It stops at the first failing table and returns what it deleted
|
||||
// so far with the error; the next run picks up from there.
|
||||
func Prune(ctx context.Context, db *sql.DB, now time.Time, p Policy) (Result, error) {
|
||||
res := Result{}
|
||||
for _, s := range sweeps {
|
||||
n, err := deleteBatched(ctx, db, s.table, s.where, now.Add(-Grace))
|
||||
res[s.table] = n
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("prune %s: %w", s.table, err)
|
||||
}
|
||||
}
|
||||
if p.Audit > 0 {
|
||||
n, err := deleteBatched(ctx, db, "audit_logs", "created_at < $1", now.Add(-p.Audit))
|
||||
res["audit_logs"] = n
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("prune audit_logs: %w", err)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func deleteBatched(ctx context.Context, db *sql.DB, table, where string, cutoff time.Time) (int64, error) {
|
||||
q := fmt.Sprintf(`DELETE FROM %[1]s WHERE ctid IN (SELECT ctid FROM %[1]s WHERE %[2]s LIMIT $2)`, table, where)
|
||||
var total int64
|
||||
for {
|
||||
r, err := db.ExecContext(ctx, q, cutoff, batch)
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
n, err := r.RowsAffected()
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
total += n
|
||||
if n < batch {
|
||||
return total, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Loop runs Prune a minute after it starts and then every interval until ctx
|
||||
// ends, logging what each run deleted.
|
||||
func Loop(ctx context.Context, db *sql.DB, p Policy, every time.Duration, log *slog.Logger) {
|
||||
t := time.NewTimer(time.Minute)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
res, err := Prune(ctx, db, time.Now(), p)
|
||||
if err != nil {
|
||||
log.Error("retention: prune failed", "err", err, "deleted", res.Total())
|
||||
} else if res.Total() > 0 {
|
||||
args := []any{"deleted", res.Total()}
|
||||
for _, s := range sweeps {
|
||||
if res[s.table] > 0 {
|
||||
args = append(args, s.table, res[s.table])
|
||||
}
|
||||
}
|
||||
if res["audit_logs"] > 0 {
|
||||
args = append(args, "audit_logs", res["audit_logs"])
|
||||
}
|
||||
log.Info("retention: pruned spent rows", args...)
|
||||
}
|
||||
t.Reset(every)
|
||||
}
|
||||
}
|
||||
|
||||
// AuditRow is one audit_logs row as ExportAudit writes it.
|
||||
type AuditRow struct {
|
||||
ID int64 `json:"id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Actor string `json:"actor"`
|
||||
ActorUserID string `json:"actor_user_id,omitempty"`
|
||||
Source string `json:"source"`
|
||||
Action string `json:"action"`
|
||||
ServerName string `json:"server_name,omitempty"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
ClientIP string `json:"client_ip,omitempty"`
|
||||
UserAgent string `json:"user_agent,omitempty"`
|
||||
Payload json.RawMessage `json:"payload,omitempty"`
|
||||
}
|
||||
|
||||
// ExportAudit writes the audit rows created in [since, until) to w as JSON lines,
|
||||
// oldest first; a zero bound is open. It returns how many rows it wrote.
|
||||
func ExportAudit(ctx context.Context, db *sql.DB, since, until time.Time, w io.Writer) (int, error) {
|
||||
const q = `SELECT id, created_at, actor, COALESCE(actor_user_id, ''), source, action,
|
||||
COALESCE(server_name, ''), COALESCE(request_id, ''), COALESCE(host(client_ip), ''),
|
||||
COALESCE(user_agent, ''), COALESCE(payload::text, '')
|
||||
FROM audit_logs
|
||||
WHERE ($1::timestamptz IS NULL OR created_at >= $1) AND ($2::timestamptz IS NULL OR created_at < $2)
|
||||
ORDER BY created_at, id`
|
||||
rows, err := db.QueryContext(ctx, q, nullTime(since), nullTime(until))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
enc := json.NewEncoder(w)
|
||||
n := 0
|
||||
for rows.Next() {
|
||||
var (
|
||||
r AuditRow
|
||||
payload string
|
||||
)
|
||||
if err := rows.Scan(&r.ID, &r.CreatedAt, &r.Actor, &r.ActorUserID, &r.Source, &r.Action,
|
||||
&r.ServerName, &r.RequestID, &r.ClientIP, &r.UserAgent, &payload); err != nil {
|
||||
return n, err
|
||||
}
|
||||
r.CreatedAt = r.CreatedAt.UTC()
|
||||
if payload != "" {
|
||||
r.Payload = json.RawMessage(payload)
|
||||
}
|
||||
if err := enc.Encode(r); err != nil {
|
||||
return n, err
|
||||
}
|
||||
n++
|
||||
}
|
||||
return n, rows.Err()
|
||||
}
|
||||
|
||||
func nullTime(t time.Time) sql.NullTime {
|
||||
return sql.NullTime{Time: t, Valid: !t.IsZero()}
|
||||
}
|
||||
Reference in new issue
Block a user