feat(retention): felis-api 定时清理过期会话、验证码、挑战、绑定码等表，审计按 [audit] retention 保留，新增... (084ba1ed) · Commits · FelisMC / Felis · GitLab

cmd/felis/api.go

+15 −0

Changes for cmd/felis/api.go: 15 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -30,6 +30,7 @@ import (
		"felis.lolicon.best/internal/reaper"
		"felis.lolicon.best/internal/registryprune"
		"felis.lolicon.best/internal/restore"
		"felis.lolicon.best/internal/retention"
		"felis.lolicon.best/internal/submit"
		"k8s.io/apimachinery/pkg/runtime"
		utilruntime "k8s.io/apimachinery/pkg/util/runtime"
		@@ -84,6 +85,14 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
		return 1
		}

		// Load already refused a malformed [audit] retention.
		auditRetention, _ := cfg.Audit.RetentionPeriod()
		if auditRetention == 0 {
		fmt.Fprintln(stdout, "felis api: audit rows are kept forever ([audit] retention = \"forever\")")
		} else {
		fmt.Fprintf(stdout, "felis api: audit rows older than %d days are deleted ([audit] retention; export them first with felis db audit-export)\n", int(auditRetention/(24*time.Hour)))
		}

		ctx := ctrl.SetupSignalHandler()

		// Before anything serves: an api on a schema it was not built for answers with
		@@ -421,6 +430,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
		go pruner.Loop(ctx, registryPruneInterval)
		}
		go reapRejectedContexts(ctx, submissions, stderr)
		go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())

		servers := []*http.Server{internalSrv, externalSrv}
		if httpsSrv != nil {
		@@ -698,6 +708,11 @@ func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writ
		}
		}

		// retentionInterval spaces the runs that delete spent sign-in rows and audit rows
		// past [audit] retention. The rows are spent for weeks before they go, so a few
		// runs a day keep the tables flat.
		const retentionInterval = 6 * time.Hour

		// registryPruneInterval spaces the registry pruner's runs. The registry-gc
		// sidecar sweeps once a day, so pruning more often only changes which sweep frees
		// a layer.

cmd/felis/db.go

+86 −0

Changes for cmd/felis/db.go: 86 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -15,6 +15,7 @@ import (

		"felis.lolicon.best/internal/config"
		"felis.lolicon.best/internal/dbbackup"
		"felis.lolicon.best/internal/retention"
		)

		const dbUsage = `usage:
		@@ -24,6 +25,7 @@ const dbUsage = `usage:
		felis db verify [-dir dir] <bundle>
		felis db list [-dir dir]
		felis db check [-dir dir] [-max-age 26h]
		felis db audit-export [-config path] [-since date] [-until date] [-out file]
		`

		// defaultKeep is how many bundles of a label a backup leaves behind. Manual
		@@ -58,6 +60,8 @@ func cmdDB(args []string, stdout, stderr io.Writer) int {
		return dbList(fs, dir, rest, stdout, stderr)
		case "check":
		return dbCheck(fs, dir, rest, stdout, stderr)
		case "audit-export":
		return dbAuditExport(fs, rest, stdout, stderr)
		case "-h", "--help", "help":
		fmt.Fprint(stdout, dbUsage)
		return 0
		@@ -262,6 +266,88 @@ func dbList(fs flag.FlagSet, dir string, args []string, stdout, stderr io.Writ
		return 0
		}

		// dbAuditExport writes audit rows to a file (or stdout) as JSON lines, so an
		// install can keep them past [audit] retention, after which felis-api deletes them.
		func dbAuditExport(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
		cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
		sinceFlag := fs.String("since", "", "first day (or RFC 3339 instant) to export, inclusive; empty starts at the oldest row")
		untilFlag := fs.String("until", "", "day (or RFC 3339 instant) to stop before, exclusive; empty runs to the newest row")
		out := fs.String("out", "", "file to write (created 0600, never overwritten); empty writes to stdout")
		if err := fs.Parse(args); err != nil {
		return 2
		}
		if fs.NArg() > 0 {
		fmt.Fprint(stderr, dbUsage)
		return 2
		}
		since, err := parseExportBound(*sinceFlag)
		if err != nil {
		fmt.Fprintf(stderr, "felis db audit-export: -since: %v\n", err)
		return 2
		}
		until, err := parseExportBound(*untilFlag)
		if err != nil {
		fmt.Fprintf(stderr, "felis db audit-export: -until: %v\n", err)
		return 2
		}
		if !since.IsZero() && !until.IsZero() && !until.After(since) {
		fmt.Fprintf(stderr, "felis db audit-export: -until %s is not after -since %s\n", untilFlag, sinceFlag)
		return 2
		}
		url, err := dbDatabaseURL(*cfgPath)
		if err != nil {
		fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
		return 1
		}
		w := stdout
		var f *os.File
		if *out != "" {
		if f, err = os.OpenFile(*out, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600); err != nil {
		fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
		return 1
		}
		w = f
		}
		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
		defer cancel()
		n, err := exportAudit(ctx, url, since, until, w)
		if f != nil {
		if cerr := f.Close(); err == nil {
		err = cerr
		}
		}
		if err != nil {
		fmt.Fprintf(stderr, "felis db audit-export: %v (%d rows written)\n", err, n)
		return 1
		}
		fmt.Fprintf(stderr, "felis db audit-export: %d audit rows written\n", n)
		return 0
		}

		func exportAudit(ctx context.Context, url string, since, until time.Time, w io.Writer) (int, error) {
		drv, err := openStore(ctx, url, false)
		if err != nil {
		return 0, fmt.Errorf("open database: %w", err)
		}
		defer drv.Close()
		return retention.ExportAudit(ctx, drv.DB(), since, until, w)
		}

		// parseExportBound reads a -since/-until value: a day (midnight UTC) or an
		// RFC 3339 instant; empty is an open bound.
		func parseExportBound(v string) (time.Time, error) {
		if v == "" {
		return time.Time{}, nil
		}
		if t, err := time.Parse(time.DateOnly, v); err == nil {
		return t, nil
		}
		if t, err := time.Parse(time.RFC3339, v); err == nil {
		return t.UTC(), nil
		}
		return time.Time{}, fmt.Errorf("%q is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)", v)
		}

		func humanBytes(n int64) string {
		const unit = 1024
		if n < unit {

cmd/felis/db_test.go

+34 −0

Changes for cmd/felis/db_test.go: 34 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@ import (
		"io"
		"strings"
		"testing"
		"time"

		"felis.lolicon.best/internal/store"
		)
		@@ -149,3 +150,36 @@ func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
		}
		}
		}

		// audit-export takes a day or an RFC 3339 instant for each bound, and refuses a
		// malformed or inverted window before it opens the config or the database.
		func TestAuditExportBounds(t *testing.T) {
		for _, tc := range []struct{ in, want string }{
		{"", "0001-01-01T00:00:00Z"},
		{"2026-01-31", "2026-01-31T00:00:00Z"},
		{"2026-01-31T12:30:00+08:00", "2026-01-31T04:30:00Z"},
		} {
		got, err := parseExportBound(tc.in)
		if err != nil \|\| got.Format(time.RFC3339) != tc.want {
		t.Errorf("parseExportBound(%q) = %v, %v; want %s", tc.in, got, err, tc.want)
		}
		}
		if _, err := parseExportBound("31/01/2026"); err == nil \|\| err.Error() != `"31/01/2026" is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)` {
		t.Errorf("parseExportBound(31/01/2026) err = %v", err)
		}
		for _, tc := range []struct {
		args []string
		wantErr string
		}{
		{[]string{"db", "audit-export", "-since", "yesterday"}, `felis db audit-export: -since: "yesterday" is neither`},
		{[]string{"db", "audit-export", "-until", "2026-13-01"}, `felis db audit-export: -until: "2026-13-01" is neither`},
		{[]string{"db", "audit-export", "-since", "2026-02-01", "-until", "2026-02-01"}, "felis db audit-export: -until 2026-02-01 is not after -since 2026-02-01"},
		{[]string{"db", "audit-export", "extra"}, "felis db audit-export [-config path]"},
		} {
		var out, errBuf bytes.Buffer
		code := run(append(tc.args, "-config", "/nonexistent/felis.toml"), &out, &errBuf)
		if code != 2 \|\| !strings.Contains(errBuf.String(), tc.wantErr) {
		t.Errorf("%v: exit %d, stderr %q; want 2 and %q", tc.args, code, errBuf.String(), tc.wantErr)
		}
		}
		}

cmd/felis/reaper.go

+3 −21

Changes for cmd/felis/reaper.go: 3 added lines, 21 removed lines.
Original line number	Diff line number	Diff line
		@@ -317,27 +317,9 @@ func resolveWorldDir(ctx context.Context, cl client.Client, namespace, worldsRoo
		}
		}

		// parseSpanDuration parses the human spans used in felis.toml's [archive] table:
		// "3mo" (months≈30d), "15d" (days), or any time.ParseDuration unit ("12h").
		func parseSpanDuration(s string) (time.Duration, error) {
		s = strings.TrimSpace(s)
		switch {
		case strings.HasSuffix(s, "mo"):
		n, err := strconv.Atoi(strings.TrimSuffix(s, "mo"))
		if err != nil {
		return 0, err
		}
		return time.Duration(n) * 30 * 24 * time.Hour, nil
		case strings.HasSuffix(s, "d"):
		n, err := strconv.Atoi(strings.TrimSuffix(s, "d"))
		if err != nil {
		return 0, err
		}
		return time.Duration(n) * 24 * time.Hour, nil
		default:
		return time.ParseDuration(s)
		}
		}
		// parseSpanDuration parses the human spans used in felis.toml's [archive] table
		// (config.ParseSpan).
		func parseSpanDuration(s string) (time.Duration, error) { return config.ParseSpan(s) }

		// parseByteSize parses a Kubernetes-style quantity ("200Gi", "10G") into bytes.
		// An empty string means unlimited (0).

docs/troubleshooting.md

+28 −0

Changes for docs/troubleshooting.md: 28 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -2031,6 +2031,33 @@ A failed audit write does not fail the action; it logs `audit: lost ...` in
		`felis-api` and counts in `felis_audit_write_failures_total`
		(`FelisAuditWriteFailing`). The cause is almost always PostgreSQL (§16).

		### How long rows are kept

		`felis-api` prunes the database a minute after it starts and every 6 hours
		after that, and logs `retention: pruned spent rows` with a count per table:

		\| Rows \| Deleted \|
		\|---\|---\|
		\| sessions \| 30 days after they expired or were signed out \|
		\| email codes, passkey challenges, setup links, op-login requests \| 30 days after they expired or were used \|
		\| `/felis link` bind codes \| 30 days after they expired \|
		\| account migrations that never completed \| 30 days after their last step (completed ones stay) \|
		\| wrong-code windows (`otp_failure_windows`) \| 30 days after they began \|
		\| `audit_logs` \| once older than `[audit] retention` \|

		`[audit] retention` defaults to `365d`; it takes days (`90d`), months of 30
		days (`18mo`) or `forever`, and refuses anything under `30d`. The daily
		`felis db backup` bundles (§16) still hold the rows for as long as the bundles
		are kept. To keep audit rows past the retention, export them before they go:

		```sh
		sudo felis db audit-export -until 2026-01-01 -out /root/audit-2025.jsonl
		```

		`-since` and `-until` take a day (UTC midnight) or an RFC 3339 instant; the
		window is `[since, until)`. Each line is one row as JSON, oldest first. The
		file is created `0600` and an existing file is never overwritten.

		### Optional: a Cloudflare rate limiting rule in front

		The limits above live in the API, so they hold on any edge. Behind Cloudflare
		@@ -2080,3 +2107,4 @@ for 10 seconds (the Free plan's limits).
		\| Right code refused; `otp_account_locked` / `FelisOTPAccountLocked` \| §17 \|
		\| `FelisSignInFailures` / who is guessing, from where \| §17 \|
		\| `FelisAuditWriteFailing` \| §17 \|
		\| How long sessions, codes and audit rows are kept; export audit rows \| §17 \|