feat(retention): felis-api 定时清理过期会话、验证码、挑战、绑定码等表,审计按 [audit] retention 保留,新增 felis db audit-export 导出归档
This commit is contained in:
9 files changed
+695
-21
No files matched your search
@@ -30,6 +30,7 @@ import (
|
||||
"felis.lolicon.best/internal/reaper"
|
||||
"felis.lolicon.best/internal/registryprune"
|
||||
"felis.lolicon.best/internal/restore"
|
||||
"felis.lolicon.best/internal/retention"
|
||||
"felis.lolicon.best/internal/submit"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
@@ -84,6 +85,14 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
|
||||
return 1
|
||||
}
|
||||
|
||||
// Load already refused a malformed [audit] retention.
|
||||
auditRetention, _ := cfg.Audit.RetentionPeriod()
|
||||
if auditRetention == 0 {
|
||||
fmt.Fprintln(stdout, "felis api: audit rows are kept forever ([audit] retention = \"forever\")")
|
||||
} else {
|
||||
fmt.Fprintf(stdout, "felis api: audit rows older than %d days are deleted ([audit] retention; export them first with felis db audit-export)\n", int(auditRetention/(24*time.Hour)))
|
||||
}
|
||||
|
||||
ctx := ctrl.SetupSignalHandler()
|
||||
|
||||
// Before anything serves: an api on a schema it was not built for answers with
|
||||
@@ -421,6 +430,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
|
||||
go pruner.Loop(ctx, registryPruneInterval)
|
||||
}
|
||||
go reapRejectedContexts(ctx, submissions, stderr)
|
||||
go retention.Loop(ctx, drv.DB(), retention.Policy{Audit: auditRetention}, retentionInterval, slog.Default())
|
||||
|
||||
servers := []*http.Server{internalSrv, externalSrv}
|
||||
if httpsSrv != nil {
|
||||
@@ -698,6 +708,11 @@ func reapRejectedContexts(ctx context.Context, m *submit.Manager, stderr io.Writ
|
||||
}
|
||||
}
|
||||
|
||||
// retentionInterval spaces the runs that delete spent sign-in rows and audit rows
|
||||
// past [audit] retention. The rows are spent for weeks before they go, so a few
|
||||
// runs a day keep the tables flat.
|
||||
const retentionInterval = 6 * time.Hour
|
||||
|
||||
// registryPruneInterval spaces the registry pruner's runs. The registry-gc
|
||||
// sidecar sweeps once a day, so pruning more often only changes which sweep frees
|
||||
// a layer.
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"felis.lolicon.best/internal/config"
|
||||
"felis.lolicon.best/internal/dbbackup"
|
||||
"felis.lolicon.best/internal/retention"
|
||||
)
|
||||
|
||||
const dbUsage = `usage:
|
||||
@@ -24,6 +25,7 @@ const dbUsage = `usage:
|
||||
felis db verify [-dir dir] <bundle>
|
||||
felis db list [-dir dir]
|
||||
felis db check [-dir dir] [-max-age 26h]
|
||||
felis db audit-export [-config path] [-since date] [-until date] [-out file]
|
||||
`
|
||||
|
||||
// defaultKeep is how many bundles of a label a backup leaves behind. Manual
|
||||
@@ -58,6 +60,8 @@ func cmdDB(args []string, stdout, stderr io.Writer) int {
|
||||
return dbList(fs, dir, rest, stdout, stderr)
|
||||
case "check":
|
||||
return dbCheck(fs, dir, rest, stdout, stderr)
|
||||
case "audit-export":
|
||||
return dbAuditExport(fs, rest, stdout, stderr)
|
||||
case "-h", "--help", "help":
|
||||
fmt.Fprint(stdout, dbUsage)
|
||||
return 0
|
||||
@@ -262,6 +266,88 @@ func dbList(fs *flag.FlagSet, dir *string, args []string, stdout, stderr io.Writ
|
||||
return 0
|
||||
}
|
||||
|
||||
// dbAuditExport writes audit rows to a file (or stdout) as JSON lines, so an
|
||||
// install can keep them past [audit] retention, after which felis-api deletes them.
|
||||
func dbAuditExport(fs *flag.FlagSet, args []string, stdout, stderr io.Writer) int {
|
||||
cfgPath := fs.String("config", "/etc/felis/felis.toml", "path to felis.toml")
|
||||
sinceFlag := fs.String("since", "", "first day (or RFC 3339 instant) to export, inclusive; empty starts at the oldest row")
|
||||
untilFlag := fs.String("until", "", "day (or RFC 3339 instant) to stop before, exclusive; empty runs to the newest row")
|
||||
out := fs.String("out", "", "file to write (created 0600, never overwritten); empty writes to stdout")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
if fs.NArg() > 0 {
|
||||
fmt.Fprint(stderr, dbUsage)
|
||||
return 2
|
||||
}
|
||||
since, err := parseExportBound(*sinceFlag)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis db audit-export: -since: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
until, err := parseExportBound(*untilFlag)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis db audit-export: -until: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
if !since.IsZero() && !until.IsZero() && !until.After(since) {
|
||||
fmt.Fprintf(stderr, "felis db audit-export: -until %s is not after -since %s\n", *untilFlag, *sinceFlag)
|
||||
return 2
|
||||
}
|
||||
url, err := dbDatabaseURL(*cfgPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
w := stdout
|
||||
var f *os.File
|
||||
if *out != "" {
|
||||
if f, err = os.OpenFile(*out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600); err != nil {
|
||||
fmt.Fprintf(stderr, "felis db audit-export: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
w = f
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
n, err := exportAudit(ctx, url, since, until, w)
|
||||
if f != nil {
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "felis db audit-export: %v (%d rows written)\n", err, n)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintf(stderr, "felis db audit-export: %d audit rows written\n", n)
|
||||
return 0
|
||||
}
|
||||
|
||||
func exportAudit(ctx context.Context, url string, since, until time.Time, w io.Writer) (int, error) {
|
||||
drv, err := openStore(ctx, url, false)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("open database: %w", err)
|
||||
}
|
||||
defer drv.Close()
|
||||
return retention.ExportAudit(ctx, drv.DB(), since, until, w)
|
||||
}
|
||||
|
||||
// parseExportBound reads a -since/-until value: a day (midnight UTC) or an
|
||||
// RFC 3339 instant; empty is an open bound.
|
||||
func parseExportBound(v string) (time.Time, error) {
|
||||
if v == "" {
|
||||
return time.Time{}, nil
|
||||
}
|
||||
if t, err := time.Parse(time.DateOnly, v); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
if t, err := time.Parse(time.RFC3339, v); err == nil {
|
||||
return t.UTC(), nil
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("%q is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)", v)
|
||||
}
|
||||
|
||||
func humanBytes(n int64) string {
|
||||
const unit = 1024
|
||||
if n < unit {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/store"
|
||||
)
|
||||
@@ -149,3 +150,36 @@ func TestPreMigrateBackupOnlyGuardsAPopulatedDatabase(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// audit-export takes a day or an RFC 3339 instant for each bound, and refuses a
|
||||
// malformed or inverted window before it opens the config or the database.
|
||||
func TestAuditExportBounds(t *testing.T) {
|
||||
for _, tc := range []struct{ in, want string }{
|
||||
{"", "0001-01-01T00:00:00Z"},
|
||||
{"2026-01-31", "2026-01-31T00:00:00Z"},
|
||||
{"2026-01-31T12:30:00+08:00", "2026-01-31T04:30:00Z"},
|
||||
} {
|
||||
got, err := parseExportBound(tc.in)
|
||||
if err != nil || got.Format(time.RFC3339) != tc.want {
|
||||
t.Errorf("parseExportBound(%q) = %v, %v; want %s", tc.in, got, err, tc.want)
|
||||
}
|
||||
}
|
||||
if _, err := parseExportBound("31/01/2026"); err == nil || err.Error() != `"31/01/2026" is neither a day (2026-01-31) nor an RFC 3339 instant (2026-01-31T12:00:00Z)` {
|
||||
t.Errorf("parseExportBound(31/01/2026) err = %v", err)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
args []string
|
||||
wantErr string
|
||||
}{
|
||||
{[]string{"db", "audit-export", "-since", "yesterday"}, `felis db audit-export: -since: "yesterday" is neither`},
|
||||
{[]string{"db", "audit-export", "-until", "2026-13-01"}, `felis db audit-export: -until: "2026-13-01" is neither`},
|
||||
{[]string{"db", "audit-export", "-since", "2026-02-01", "-until", "2026-02-01"}, "felis db audit-export: -until 2026-02-01 is not after -since 2026-02-01"},
|
||||
{[]string{"db", "audit-export", "extra"}, "felis db audit-export [-config path]"},
|
||||
} {
|
||||
var out, errBuf bytes.Buffer
|
||||
code := run(append(tc.args, "-config", "/nonexistent/felis.toml"), &out, &errBuf)
|
||||
if code != 2 || !strings.Contains(errBuf.String(), tc.wantErr) {
|
||||
t.Errorf("%v: exit %d, stderr %q; want 2 and %q", tc.args, code, errBuf.String(), tc.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-21
@@ -317,27 +317,9 @@ func resolveWorldDir(ctx context.Context, cl client.Client, namespace, worldsRoo
|
||||
}
|
||||
}
|
||||
|
||||
// parseSpanDuration parses the human spans used in felis.toml's [archive] table:
|
||||
// "3mo" (months≈30d), "15d" (days), or any time.ParseDuration unit ("12h").
|
||||
func parseSpanDuration(s string) (time.Duration, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
switch {
|
||||
case strings.HasSuffix(s, "mo"):
|
||||
n, err := strconv.Atoi(strings.TrimSuffix(s, "mo"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return time.Duration(n) * 30 * 24 * time.Hour, nil
|
||||
case strings.HasSuffix(s, "d"):
|
||||
n, err := strconv.Atoi(strings.TrimSuffix(s, "d"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return time.Duration(n) * 24 * time.Hour, nil
|
||||
default:
|
||||
return time.ParseDuration(s)
|
||||
}
|
||||
}
|
||||
// parseSpanDuration parses the human spans used in felis.toml's [archive] table
|
||||
// (config.ParseSpan).
|
||||
func parseSpanDuration(s string) (time.Duration, error) { return config.ParseSpan(s) }
|
||||
|
||||
// parseByteSize parses a Kubernetes-style quantity ("200Gi", "10G") into bytes.
|
||||
// An empty string means unlimited (0).
|
||||
|
||||
Reference in new issue
Block a user