feat(registry): 开启 manifest 删除，GC sidecar 在 gate 只读窗口内回收 blob

internal/registrygate/gate.go

@@ -11,7 +11,13 @@
 //     and none of them should carry a credential;
 //   - the "platform" principal (the installer) may write anything;
 //   - the "build" principal (the push step of a build Job) may write any repository
-//     outside the platform-reserved ones (felis/…, mirror/…), and may not delete.
+//     outside the platform-reserved ones (felis/…, mirror/…), and may not delete;
+//   - the "prune" principal (felis-api's registry pruner) may only delete a
+//     manifest by digest, the one write that frees space.
+//
+// The gate also holds the registry read-only while the GC sidecar runs registry
+// garbage-collect (maint.go): a blob pushed during the sweep could be deleted
+// under a manifest that is about to reference it.
 //
 // Before this gate any pod that could reach the registry — a game server running a
 // tenant's plugin, or a Dockerfile RUN step inside Kaniko — could overwrite
@@ -27,17 +33,24 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"regexp"
 	"strings"
 	"time"
 )
 
+var digestRE = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
+
 // Principal names. They are the basic-auth usernames and the file names under the
 // gate's auth directory (cmd/felis registry-gate --auth-dir).
 const (
 	PrincipalPlatform = "platform"
 	PrincipalBuild    = "build"
+	PrincipalPrune    = "prune"
 )
 
+// Principals lists every principal the gate knows, for loading their tokens.
+var Principals = []string{PrincipalPlatform, PrincipalBuild, PrincipalPrune}
+
 // ReservedRepoRoots are the first path components the build principal may never
 // write: felis/ holds the control-plane and game images the platform runs, mirror/
 // holds the Trivy DB mirrors the scan gate trusts. A build that could overwrite
@@ -59,6 +72,7 @@ type Gate struct {
 
 	proxy  *httputil.ReverseProxy
 	health *http.Client
+	maint  maintenance
 }
 
 // New builds a Gate for upstream.
@@ -76,6 +90,8 @@ func New(upstream *url.URL, tokens map[string]string, log *slog.Logger) *Gate {
 	rp.FlushInterval = -1
 	g.proxy = rp
 	g.health = &http.Client{Timeout: 3 * time.Second}
+	g.maint.now = time.Now
+	g.maint.quiet = DefaultQuiet
 	return g
 }
 
@@ -135,6 +151,15 @@ func (g *Gate) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusForbidden, "DENIED", reason)
 		return
 	}
+	if !g.maint.beginWrite() {
+		// Retry-After is what imagepush and docker push back off on; a GC sweep
+		// over a few GiB takes well under a minute.
+		w.Header().Set("Retry-After", "30")
+		writeError(w, http.StatusServiceUnavailable, "UNAVAILABLE",
+			"the registry is read-only while garbage collection runs; retry shortly")
+		return
+	}
+	defer g.maint.endWrite()
 	g.proxy.ServeHTTP(w, r)
 }
 
@@ -181,11 +206,30 @@ func Authorize(principal, method, path string) string {
 			}
 		}
 		return ""
+	case PrincipalPrune:
+		// Deleting a manifest only unlinks it; the blobs go at the next GC. The
+		// pruner never needs anything else, so a leaked prune token can neither
+		// plant an image nor delete a blob a live manifest still references.
+		if method != http.MethodDelete || !isManifestDigestPath(path) {
+			return "the prune principal may only delete a manifest by digest"
+		}
+		return ""
 	default:
 		return "unknown principal"
 	}
 }
 
+// isManifestDigestPath reports whether path is /v2/<repo>/manifests/sha256:<hex>.
+func isManifestDigestPath(path string) bool {
+	rest, ok := strings.CutPrefix(path, "/v2/")
+	if !ok {
+		return false
+	}
+	seg := strings.Split(rest, "/")
+	n := len(seg)
+	return n >= 3 && seg[n-2] == "manifests" && digestRE.MatchString(seg[n-1])
+}
+
 // RepoFromPath extracts the repository name from a registry API v2 path, or ""
 // when the path addresses no repository (/v2/, /v2/_catalog). The shapes are the
 // distribution API's: <name>/manifests/<ref>, <name>/blobs/<digest>,

internal/registrygate/maint.go

@@ -0,0 +1,196 @@
+package registrygate
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// registry garbage-collect is only safe on a registry nobody writes to: it marks
+// the blobs every manifest references, then deletes the rest, and a layer pushed
+// between the two phases is deleted under the manifest that arrives next. The GC
+// sidecar therefore asks the gate for a read-only window over a loopback-only
+// maintenance listener (MaintHandler), and the gate grants it only once writes
+// have been quiet for a while, so a push that is between two of its requests is
+// not cut in half.
+//
+// While the window is open every write answers 503 with Retry-After; reads keep
+// working, so running servers and kubelet re-pulls never notice. The window is a
+// lease: a GC sidecar that dies mid-sweep cannot leave the registry read-only for
+// longer than the lease it asked for.
+
+// DefaultQuiet is how long writes must have been idle before a read-only window is
+// granted. A push issues its requests back to back; two minutes of silence means
+// no push is mid-way.
+const DefaultQuiet = 2 * time.Minute
+
+// maxLease bounds a read-only window. A sweep over a few GiB takes seconds; an
+// hour covers a large registry on a slow disk.
+const maxLease = time.Hour
+
+type maintenance struct {
+	mu        sync.Mutex
+	inflight  int
+	lastWrite time.Time
+	until     time.Time
+	quiet     time.Duration
+	stateFile string
+	now       func() time.Time
+}
+
+// beginWrite admits a write unless a read-only window is open.
+func (m *maintenance) beginWrite() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	now := m.now()
+	if now.Before(m.until) {
+		return false
+	}
+	m.inflight++
+	m.lastWrite = now
+	return true
+}
+
+func (m *maintenance) endWrite() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.inflight--
+	m.lastWrite = m.now()
+}
+
+// acquire opens (or extends) a read-only window for lease. It refuses while a
+// write is in flight or the last one finished less than quiet ago.
+func (m *maintenance) acquire(lease time.Duration) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	now := m.now()
+	if !now.Before(m.until) {
+		if m.inflight > 0 {
+			return fmt.Errorf("%d write(s) in flight", m.inflight)
+		}
+		if idle := now.Sub(m.lastWrite); idle < m.quiet {
+			return fmt.Errorf("last write %s ago, waiting for %s of quiet", idle.Round(time.Second), m.quiet)
+		}
+	}
+	m.until = now.Add(lease)
+	m.persist()
+	return nil
+}
+
+func (m *maintenance) release() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.until = time.Time{}
+	m.persist()
+}
+
+func (m *maintenance) readOnly() (bool, time.Time) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.now().Before(m.until), m.until
+}
+
+// persist records the window's end in stateFile, so a gate container restarted
+// mid-sweep comes back read-only instead of admitting writes into a running GC.
+// Called with mu held.
+func (m *maintenance) persist() {
+	if m.stateFile == "" {
+		return
+	}
+	if m.until.IsZero() {
+		_ = os.Remove(m.stateFile)
+		return
+	}
+	tmp := m.stateFile + ".tmp"
+	if err := os.WriteFile(tmp, []byte(strconv.FormatInt(m.until.Unix(), 10)), 0o600); err == nil {
+		_ = os.Rename(tmp, m.stateFile)
+	}
+}
+
+// SetMaintenanceState makes the gate keep its read-only window in path (on a
+// volume that outlives the container) and resumes a window a previous run of the
+// gate left open.
+func (g *Gate) SetMaintenanceState(path string) error {
+	g.maint.mu.Lock()
+	defer g.maint.mu.Unlock()
+	g.maint.stateFile = path
+	b, err := os.ReadFile(path)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	sec, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
+	if err != nil {
+		return fmt.Errorf("maintenance state %s: %w", path, err)
+	}
+	if until := time.Unix(sec, 0); g.maint.now().Before(until) {
+		g.maint.until = until
+	}
+	return nil
+}
+
+// SetQuiet overrides DefaultQuiet (tests and drills shorten it).
+func (g *Gate) SetQuiet(d time.Duration) {
+	g.maint.mu.Lock()
+	g.maint.quiet = d
+	g.maint.mu.Unlock()
+}
+
+// MaintHandler serves the GC sidecar's side of the handshake. It carries no
+// authentication, so it must only ever listen on the pod's loopback:
+//
+//	POST /readonly?lease=<seconds>  200 once the window is open, 409 while writes are not quiet
+//	POST /readwrite                 200, the window is closed
+//	GET  /readonly                  200 while read-only, 409 otherwise
+//
+// POST-only verbs keep the sidecar's busybox wget (which cannot send DELETE) able
+// to drive it.
+func (g *Gate) MaintHandler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("POST /readonly", func(w http.ResponseWriter, r *http.Request) {
+		lease := maxLease
+		if s := r.URL.Query().Get("lease"); s != "" {
+			sec, err := strconv.Atoi(s)
+			if err != nil || sec <= 0 {
+				http.Error(w, "lease must be a positive number of seconds", http.StatusBadRequest)
+				return
+			}
+			lease = min(time.Duration(sec)*time.Second, maxLease)
+		}
+		if err := g.maint.acquire(lease); err != nil {
+			http.Error(w, "busy: "+err.Error(), http.StatusConflict)
+			return
+		}
+		_, until := g.maint.readOnly()
+		if g.Log != nil {
+			g.Log.Info("registry read-only for garbage collection", "until", until.UTC().Format(time.RFC3339))
+		}
+		fmt.Fprintf(w, "read-only until %s\n", until.UTC().Format(time.RFC3339))
+	})
+	mux.HandleFunc("POST /readwrite", func(w http.ResponseWriter, r *http.Request) {
+		g.maint.release()
+		if g.Log != nil {
+			g.Log.Info("registry writable again")
+		}
+		fmt.Fprintln(w, "writable")
+	})
+	mux.HandleFunc("GET /readonly", func(w http.ResponseWriter, r *http.Request) {
+		if ro, until := g.maint.readOnly(); ro {
+			fmt.Fprintf(w, "read-only until %s\n", until.UTC().Format(time.RFC3339))
+			return
+		}
+		http.Error(w, "writable", http.StatusConflict)
+	})
+	return mux
+}
+
+// MaintStatePath is where cmd/felis keeps the window inside the maintenance
+// volume.
+func MaintStatePath(dir string) string { return filepath.Join(dir, "readonly-until") }

internal/registrygate/maint_test.go

@@ -0,0 +1,208 @@
+package registrygate
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+const testDigest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+
+type clock struct {
+	mu sync.Mutex
+	t  time.Time
+}
+
+func (c *clock) now() time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.t
+}
+
+func (c *clock) advance(d time.Duration) {
+	c.mu.Lock()
+	c.t = c.t.Add(d)
+	c.mu.Unlock()
+}
+
+func newMaintGate(t *testing.T) (*Gate, *httptest.Server, *httptest.Server, *upstreamLog, *clock) {
+	t.Helper()
+	up := &upstreamLog{}
+	upSrv := httptest.NewServer(up.handler())
+	t.Cleanup(upSrv.Close)
+	target, _ := url.Parse(upSrv.URL)
+	g := New(target, map[string]string{
+		PrincipalPlatform: "plat-secret", PrincipalBuild: "build-secret", PrincipalPrune: "prune-secret",
+	}, nil)
+	clk := &clock{t: time.Date(2026, 9, 24, 3, 0, 0, 0, time.UTC)}
+	g.maint.now = clk.now
+	gs := httptest.NewServer(g)
+	t.Cleanup(gs.Close)
+	ms := httptest.NewServer(g.MaintHandler())
+	t.Cleanup(ms.Close)
+	return g, gs, ms, up, clk
+}
+
+func post(t *testing.T, srv *httptest.Server, path string) int {
+	t.Helper()
+	resp, err := http.Post(srv.URL+path, "text/plain", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp.Body.Close()
+	return resp.StatusCode
+}
+
+func TestPrunePrincipalMayOnlyDeleteManifestsByDigest(t *testing.T) {
+	up2 := &upstreamLog{}
+	upSrv := httptest.NewServer(up2.handler())
+	t.Cleanup(upSrv.Close)
+	target, _ := url.Parse(upSrv.URL)
+	g := New(target, map[string]string{PrincipalPrune: "prune-secret"}, nil)
+	srv := httptest.NewServer(g)
+	t.Cleanup(srv.Close)
+
+	for _, p := range []string{
+		"/v2/user-uploads/s1/manifests/" + testDigest,
+		"/v2/felis/felis/manifests/" + testDigest,
+	} {
+		if resp := do(t, srv, http.MethodDelete, p, PrincipalPrune, "prune-secret"); resp.StatusCode != http.StatusOK {
+			t.Errorf("prune DELETE %s = %d, want it proxied", p, resp.StatusCode)
+		}
+	}
+	for _, c := range []struct{ method, path string }{
+		{http.MethodDelete, "/v2/user-uploads/s1/manifests/latest"},
+		{http.MethodDelete, "/v2/user-uploads/s1/blobs/" + testDigest},
+		{http.MethodDelete, "/v2/user-uploads/s1/manifests/sha256:short"},
+		{http.MethodPut, "/v2/user-uploads/s1/manifests/" + testDigest},
+		{http.MethodPost, "/v2/user-uploads/s1/blobs/uploads/"},
+		{http.MethodPatch, "/v2/user-uploads/s1/blobs/uploads/abc"},
+	} {
+		if resp := do(t, srv, c.method, c.path, PrincipalPrune, "prune-secret"); resp.StatusCode != http.StatusForbidden {
+			t.Errorf("prune %s %s = %d, want 403", c.method, c.path, resp.StatusCode)
+		}
+	}
+	if n := up2.count(); n != 2 {
+		t.Fatalf("upstream saw %d requests, want the 2 allowed deletes: %v", n, up2.seen)
+	}
+}
+
+func TestReadOnlyWindowWaitsForQuietThenRefusesWrites(t *testing.T) {
+	_, gs, ms, up, clk := newMaintGate(t)
+	put := "/v2/user-uploads/s1/manifests/latest"
+
+	if resp := do(t, gs, http.MethodPut, put, PrincipalBuild, "build-secret"); resp.StatusCode != http.StatusCreated {
+		t.Fatalf("write before any window = %d, want 201", resp.StatusCode)
+	}
+	// A write just finished: a push may be between two of its requests.
+	if code := post(t, ms, "/readonly?lease=600"); code != http.StatusConflict {
+		t.Fatalf("readonly right after a write = %d, want 409", code)
+	}
+	clk.advance(DefaultQuiet)
+	if code := post(t, ms, "/readonly?lease=600"); code != http.StatusOK {
+		t.Fatalf("readonly after %s of quiet = %d, want 200", DefaultQuiet, code)
+	}
+
+	before := up.count()
+	resp := do(t, gs, http.MethodPut, put, PrincipalPlatform, "plat-secret")
+	if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") == "" {
+		t.Fatalf("write during the window = %d Retry-After=%q, want 503 with Retry-After", resp.StatusCode, resp.Header.Get("Retry-After"))
+	}
+	if resp := do(t, gs, http.MethodGet, "/v2/felis/felis/manifests/b1", "", ""); resp.StatusCode != http.StatusOK {
+		t.Fatalf("read during the window = %d, want 200", resp.StatusCode)
+	}
+	if up.count() != before+1 {
+		t.Fatalf("the refused write reached the registry: %v", up.seen)
+	}
+	resp, err := http.Get(ms.URL + "/readonly")
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("GET /readonly during the window = %d, want 200", resp.StatusCode)
+	}
+
+	if code := post(t, ms, "/readwrite"); code != http.StatusOK {
+		t.Fatalf("readwrite = %d", code)
+	}
+	if resp := do(t, gs, http.MethodPut, put, PrincipalBuild, "build-secret"); resp.StatusCode != http.StatusCreated {
+		t.Fatalf("write after the window = %d, want 201", resp.StatusCode)
+	}
+}
+
+func TestReadOnlyWindowIsALease(t *testing.T) {
+	_, gs, ms, _, clk := newMaintGate(t)
+	if code := post(t, ms, "/readonly?lease=60"); code != http.StatusOK {
+		t.Fatalf("readonly on an idle gate = %d, want 200", code)
+	}
+	put := "/v2/user-uploads/s1/manifests/latest"
+	if resp := do(t, gs, http.MethodPut, put, PrincipalBuild, "build-secret"); resp.StatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("write inside the lease = %d, want 503", resp.StatusCode)
+	}
+	// A GC sidecar that died mid-sweep never releases; the lease does.
+	clk.advance(61 * time.Second)
+	if resp := do(t, gs, http.MethodPut, put, PrincipalBuild, "build-secret"); resp.StatusCode != http.StatusCreated {
+		t.Fatalf("write after the lease expired = %d, want 201", resp.StatusCode)
+	}
+	for _, bad := range []string{"0", "-5", "soon"} {
+		if code := post(t, ms, "/readonly?lease="+bad); code != http.StatusBadRequest {
+			t.Errorf("lease=%s = %d, want 400", bad, code)
+		}
+	}
+}
+
+func TestReadOnlyWindowSurvivesAGateRestart(t *testing.T) {
+	dir := t.TempDir()
+	state := MaintStatePath(dir)
+	g, _, ms, _, clk := newMaintGate(t)
+	if err := g.SetMaintenanceState(state); err != nil {
+		t.Fatal(err)
+	}
+	if code := post(t, ms, "/readonly?lease=600"); code != http.StatusOK {
+		t.Fatalf("readonly = %d", code)
+	}
+	if _, err := os.Stat(state); err != nil {
+		t.Fatalf("window not persisted: %v", err)
+	}
+
+	// The restarted gate reads the window back and keeps refusing writes.
+	g2, gs2, _, _, _ := newMaintGate(t)
+	g2.maint.now = clk.now
+	if err := g2.SetMaintenanceState(state); err != nil {
+		t.Fatal(err)
+	}
+	if resp := do(t, gs2, http.MethodPut, "/v2/user-uploads/s1/manifests/latest", PrincipalBuild, "build-secret"); resp.StatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("write on the restarted gate = %d, want 503", resp.StatusCode)
+	}
+
+	if code := post(t, ms, "/readwrite"); code != http.StatusOK {
+		t.Fatalf("readwrite = %d", code)
+	}
+	if _, err := os.Stat(state); !os.IsNotExist(err) {
+		t.Fatalf("state file left after release: %v", err)
+	}
+
+	// An expired window in the file is ignored.
+	if err := os.WriteFile(state, []byte("1"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	g3, gs3, _, _, _ := newMaintGate(t)
+	if err := g3.SetMaintenanceState(state); err != nil {
+		t.Fatal(err)
+	}
+	if resp := do(t, gs3, http.MethodPut, "/v2/user-uploads/s1/manifests/latest", PrincipalBuild, "build-secret"); resp.StatusCode != http.StatusCreated {
+		t.Fatalf("write with an expired window on file = %d, want 201", resp.StatusCode)
+	}
+	if err := os.WriteFile(state, []byte("not-a-number"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := g3.SetMaintenanceState(state); err == nil || !strings.Contains(err.Error(), "maintenance state") {
+		t.Fatalf("a corrupt state file = %v, want an error naming it", err)
+	}
+}