diff --git a/cmd/felis/api.go b/cmd/felis/api.go index c2ab081..a0a2b91 100644 --- a/cmd/felis/api.go +++ b/cmd/felis/api.go @@ -10,6 +10,7 @@ import ( "os" "regexp" "strings" + "sync" "sync/atomic" "time" @@ -415,15 +416,17 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int { } go reapRejectedContexts(ctx, submissions, stderr) + servers := []*http.Server{internalSrv, externalSrv} + if httpsSrv != nil { + servers = append(servers, httpsSrv) + } + for _, srv := range servers { + srv.RegisterOnShutdown(a.CloseStreams) + } + select { case <-ctx.Done(): - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _ = internalSrv.Shutdown(shutdownCtx) - _ = externalSrv.Shutdown(shutdownCtx) - if httpsSrv != nil { - _ = httpsSrv.Shutdown(shutdownCtx) - } + shutdownServers(servers, apiShutdownGrace, stderr) return 0 case err := <-errc: if err != nil && err != http.ErrServerClosed { @@ -443,8 +446,30 @@ const ( // apiIdleTimeout caps how long a kept-alive connection may sit idle between // requests before the server closes it, bounding idle-connection exhaustion. apiIdleTimeout = 120 * time.Second + // apiShutdownGrace is how long the listeners drain after SIGTERM. The pod gets + // the Kubernetes default of 30s before SIGKILL; this leaves the rest for the + // process to exit. + apiShutdownGrace = 20 * time.Second ) +// shutdownServers drains every listener at once under one deadline: in turn, a +// slow first listener would spend the time the others needed. Log streams end +// through RegisterOnShutdown (API.CloseStreams); what is still running when the +// deadline passes is cut off with the process. +func shutdownServers(servers []*http.Server, grace time.Duration, stderr io.Writer) { + ctx, cancel := context.WithTimeout(context.Background(), grace) + defer cancel() + var wg sync.WaitGroup + for _, srv := range servers { + wg.Go(func() { + if err := srv.Shutdown(ctx); err != nil { + fmt.Fprintf(stderr, "felis api: shutdown %s: %v\n", srv.Addr, err) + } + }) + } + wg.Wait() +} + // newAPIServer builds an http.Server with hardened header/idle timeouts (gosec // G112) shared by all three felis-api listeners (internal, external, https). // WriteTimeout and ReadTimeout are deliberately LEFT UNSET: the external and https diff --git a/cmd/felis/api_test.go b/cmd/felis/api_test.go index a9990ec..17faf05 100644 --- a/cmd/felis/api_test.go +++ b/cmd/felis/api_test.go @@ -4,8 +4,12 @@ import ( "context" "errors" "fmt" + "io" + "net" "net/http" + "sync/atomic" "testing" + "time" "felis.lolicon.best/internal/api" "felis.lolicon.best/internal/build" @@ -97,6 +101,50 @@ func TestNewAPIServerSetsHardenedTimeouts(t *testing.T) { } } +// Every listener drains at once, and the shutdown hook (API.CloseStreams in +// cmdAPI) runs on each: two listeners each holding a request that ends when +// the hook fires take one hook's worth of time, well inside the deadline. +func TestShutdownServersDrainsListenersTogether(t *testing.T) { + release := make(chan struct{}) + var hooks int32 + started := make(chan struct{}, 2) + var servers []*http.Server + for range 2 { + srv := newAPIServer("", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + started <- struct{}{} + <-release + })) + srv.RegisterOnShutdown(func() { + if atomic.AddInt32(&hooks, 1) == 1 { + time.AfterFunc(100*time.Millisecond, func() { close(release) }) + } + }) + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv.Addr = l.Addr().String() + go func() { _ = srv.Serve(l) }() + go func() { + if resp, err := http.Get("http://" + srv.Addr); err == nil { + resp.Body.Close() + } + }() + servers = append(servers, srv) + } + <-started + <-started + + begun := time.Now() + shutdownServers(servers, 5*time.Second, io.Discard) + if took := time.Since(begun); took > 2*time.Second { + t.Fatalf("shutdown took %v", took) + } + if got := atomic.LoadInt32(&hooks); got != 2 { + t.Fatalf("shutdown hook ran %d times, want once per listener", got) + } +} + type fakeRefStore struct { images []build.Image builds []build.Build diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e4beaf7..f6397d1 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -31,6 +31,28 @@ info: and the panel). Admin-tier external operations additionally require the admin Access path. See `x-felis-face` / `x-felis-tier` on each operation. + Behaviour every operation shares, and so not repeated under each: + + * Every response carries `X-Request-Id` (a well-formed inbound one is kept), + `X-Content-Type-Options: nosniff`, `X-Frame-Options: DENY`, + `Referrer-Policy: no-referrer` and `Content-Security-Policy: default-src 'none'`. + `Strict-Transport-Security` is added when the request came through the TLS + edge (`X-Forwarded-Proto: https`). + * A path no operation serves is `404 not_found`; a path served under other + methods is `405 method_not_allowed` with an `Allow` header. + * A POST/PUT/PATCH/DELETE a browser sends from another site (`Sec-Fetch-Site` + `same-site` or `cross-site`, or an `Origin` whose host is not the request's) + is `403 cross_site`, before authentication. Callers that send neither header + (the plugins, scripts) are unaffected. + * A JSON body over 1 MiB is `413 too_large`. A request body must keep arriving: + after 30 s it has to average 16 KiB/s or the connection is closed. + * Event streams (the console and build logs) tag each line with `id:` (unix + seconds); an EventSource that reconnects with `Last-Event-ID` within the hour + resumes from that second instead of the tailed backlog. The server re-checks + the caller every minute and ends the stream with `event: revoked` once the + session or the access is gone; a stream also closes after 30 minutes and on + server shutdown, and the client simply reconnects. + servers: - url: https://api-internal.{root_domain} description: >- @@ -1443,9 +1465,16 @@ paths: security: [{ accessJWT: [] }] parameters: - { name: name, in: path, required: true, schema: { type: string } } + - name: Last-Event-ID + in: header + required: false + description: The id of the last line received; resumes the stream from that second (within the hour). + schema: { type: string } responses: '200': - description: An event stream of log lines. + description: >- + An event stream of log lines (`id:` + `data:` per line, `:` comments as + keep-alives), ended by `event: revoked` when access is withdrawn. content: text/event-stream: schema: { type: string } @@ -1871,7 +1900,11 @@ paths: get: tags: [servers] operationId: status - summary: Status of your own server. + summary: Status of a server; the full record for its owner and staff. + description: >- + Anyone signed in may ask. The owner and staff get the whole projection; + anyone else gets what the game's own server list shows: name, subdomain, + displayName, phase, ready, playersOnline and playersMax. x-felis-face: [external] x-felis-tier: app security: [{ accessJWT: [] }] @@ -1879,7 +1912,7 @@ paths: - { name: name, in: path, required: true, schema: { type: string } } responses: '200': - description: The server's status projection. + description: The server's status projection (trimmed for non-owners). content: application/json: schema: { $ref: '#/components/schemas/ServerInfo' } @@ -4707,9 +4740,16 @@ paths: security: [{ accessJWT: [] }] parameters: - { name: id, in: path, required: true, schema: { type: string } } + - name: Last-Event-ID + in: header + required: false + description: The id of the last line received; resumes the stream from that second (within the hour). + schema: { type: string } responses: '200': - description: An event stream of build log lines. + description: >- + An event stream of build log lines (`id:` + `data:` per line), ended by + `event: revoked` when the caller is no longer staff. content: text/event-stream: schema: { type: string } diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 1264096..1cc5198 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1238,6 +1238,26 @@ All four mandated metrics have real producers; scrape them when triaging: - `felis_reaper_worlds_deleted_total` — increments only after a world PVC is actually deleted post-backup (§10); a spike here means worlds crossed the 15d idle line — cross-check that join events are flowing (§10 risk vectors). +- `felis_http_requests_total{face,method,route,code}` and + `felis_http_request_duration_seconds{face,route}` — every API request, by the + route pattern it matched (`/api/v1/servers/{name}/status`, never the raw + path; `unmatched` for a path no route serves). `face` is `internal` or + `external`. Log streams count as requests but stay out of the latency + histogram. A rise in `code=~"5.."` on one route narrows a failure to one + handler; `route="unmatched"` rising is someone scanning. +- The Go runtime and process series (`go_*`, `process_*`) of `felis-api`: + goroutines, heap, open file descriptors. Goroutines that climb without + falling back usually mean streams or uploads that never end. + +The API also writes one access-log line per request to its log, in logfmt: +`face`, `method`, `route`, `path`, `status`, `duration_ms`, `bytes`, +`request_id` (the id in every error envelope) and `principal` (the user id, +once signed in). Successful probes and scrapes are left out. + +```bash +kubectl -n felis logs deploy/felis-api | grep 'msg=request' | grep 'status=5' +kubectl -n felis logs deploy/felis-api | grep 'request_id=' +``` ### Scraping @@ -1250,7 +1270,8 @@ annotated Service endpoints picks them up as is. `felis_build_info{component="operator"}`, and controller-runtime's `controller_runtime_reconcile_*` / `workqueue_*` series. - `felis-api` internal face `:8081/metrics` (Service `felis-api-internal`) — - `felis_build_info{component="api"}`, + `felis_build_info{component="api"}`, the `felis_http_*` request series, the + `go_*`/`process_*` runtime series, `felis_image_build_failures_total`, and the sign-in series of §17 (`felis_mail_total`, `felis_rate_limited_total`, `felis_auth_otp_lockouts_total`, `felis_auth_failures_total`, diff --git a/internal/api/api.go b/internal/api/api.go index f5ca662..027e6e6 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -14,7 +14,9 @@ package api import ( "context" + "log/slog" "net/http" + "strings" "sync" "time" @@ -181,6 +183,10 @@ type API struct { // X-Forwarded-For behind an operator proxy). Empty means the TCP peer. ClientIPHeader string + // AccessLog receives one line per API request (observe.go). Nil logs logfmt + // to stderr. + AccessLog *slog.Logger + // Now is the clock, injectable for tests. Defaults to time.Now. Now func() time.Time @@ -200,6 +206,26 @@ type API struct { authDoorBuckets *bucketSet mailOnce sync.Once mailBuckets *bucketSet + + drainInit sync.Once + drainClose sync.Once + drain chan struct{} +} + +// streamsClosing is closed once CloseStreams runs. +func (a *API) streamsClosing() <-chan struct{} { + a.drainInit.Do(func() { a.drain = make(chan struct{}) }) + return a.drain +} + +// CloseStreams ends every log stream this API is relaying, now and from now on. +// http.Server.Shutdown waits for handlers to return and cancels nothing, so a +// console left open would hold the process until the pod's grace period ran out; +// register this with RegisterOnShutdown. The EventSource on the other end +// reconnects, and resumes from its Last-Event-ID on the next instance. +func (a *API) CloseStreams() { + a.streamsClosing() + a.drainClose.Do(func() { close(a.drain) }) } // panelURL returns the public player-console origin ("https://console."), @@ -625,14 +651,14 @@ func (a *API) externalAPIRoutes() []apiRoute { // InternalHandler builds the internal-face http.Handler: service-token auth, no // Zero Trust (spec §14 red line). /healthz and /readyz are unauthenticated. func (a *API) InternalHandler() http.Handler { - return a.buildFace(a.internalAPIRoutes(), a.requireInternal) + return a.buildFace("internal", a.internalAPIRoutes(), a.requireInternal) } // ExternalHandler builds the external-face http.Handler: Access-JWT auth on every // /api/v1 route, with admin-tier routes additionally gated by the admin Access // path inside their handlers. func (a *API) ExternalHandler() http.Handler { - return a.buildFace(a.externalAPIRoutes(), a.requireExternal) + return a.buildFace("external", a.externalAPIRoutes(), a.requireExternal) } // buildFace assembles one face from its route table. Public routes are mounted @@ -641,7 +667,7 @@ func (a *API) ExternalHandler() http.Handler { // adminOnly, and Owner routes in ownerOnly. Because both faces are built from the // same table the OpenAPI parity test reads, the served surface and the documented // surface cannot drift apart without failing the build. -func (a *API) buildFace(routes []apiRoute, guard func(http.Handler) http.Handler) http.Handler { +func (a *API) buildFace(face string, routes []apiRoute, guard func(http.Handler) http.Handler) http.Handler { mux := http.NewServeMux() auth := http.NewServeMux() for _, rt := range routes { @@ -651,7 +677,7 @@ func (a *API) buildFace(routes []apiRoute, guard func(http.Handler) http.Handler if rt.AuthDoor { h = a.throttleAuthDoor(h) } - mux.HandleFunc(pattern, h) + mux.HandleFunc(pattern, tagRoute(rt.Pattern, h)) continue } h := rt.h @@ -667,22 +693,61 @@ func (a *API) buildFace(routes []apiRoute, guard func(http.Handler) http.Handler if !rt.SetupAllowed { h = a.requireOnboarded(h) } - auth.HandleFunc(pattern, h) + auth.HandleFunc(pattern, tagRoute(rt.Pattern, h)) } guarded := guard(auth) mux.Handle("/api/v1/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if _, pattern := auth.Handler(r); pattern == "" { - http.NotFound(w, r) + _, pattern := auth.Handler(r) + if pattern == "" { + writeNoRoute(w, r, mux, auth) return } + // Named before the guard runs, so a refused request is counted under + // the route it asked for. + noteRoute(r, pattern) guarded.ServeHTTP(w, r) })) - return a.baseChain(mux) + top := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, pattern := mux.Handler(r); pattern == "" { + writeNoRoute(w, r, mux, auth) + return + } + mux.ServeHTTP(w, r) + }) + return a.baseChain(face, top) } -// baseChain wraps a handler in the cross-cutting middleware shared by both faces. -func (a *API) baseChain(h http.Handler) http.Handler { - return withRequestID(withRecover(h)) +// baseChain wraps a handler in the cross-cutting middleware shared by both faces: +// the request id, then the access log and metrics (which see the final status of +// everything inside), the response security headers, the cross-site write fence, +// the request-body read deadline, and panic recovery. +func (a *API) baseChain(face string, h http.Handler) http.Handler { + return withRequestID(a.observe(face, withSecurityHeaders(rejectCrossSiteWrites(withBodyDeadline(withRecover(h)))))) +} + +// writeNoRoute answers a request no route took, in the API's error envelope: 405 +// with an Allow header when the path exists under other methods, 404 otherwise. +// ServeMux's own answers are plain text and turn a wrong method on a guarded +// route into a 404, since the guarded routes sit behind one catch-all. +func writeNoRoute(w http.ResponseWriter, r *http.Request, muxes ...*http.ServeMux) { + var allow []string + for _, m := range []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete} { + probe := r.Clone(r.Context()) + probe.Method = m + for _, mux := range muxes { + if _, p := mux.Handler(probe); p != "" && p != "/api/v1/" { + allow = append(allow, m) + break + } + } + } + if len(allow) > 0 { + w.Header().Set("Allow", strings.Join(allow, ", ")) + writeError(w, r, newError(http.StatusMethodNotAllowed, "method_not_allowed", + "%s is not allowed here; use %s", r.Method, strings.Join(allow, ", "))) + return + } + writeError(w, r, newError(http.StatusNotFound, "not_found", "no such endpoint")) } // requireOnboarded fences an authenticated route behind the setup-lockdown: a @@ -734,6 +799,7 @@ type ctxKey int const ( ctxKeyRequestID ctxKey = iota ctxKeyPrincipal + ctxKeyReqInfo ) func requestIDFromContext(ctx context.Context) string { diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 2071d37..5c41abe 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "log/slog" "net/http" "net/http/httptest" "sort" @@ -1646,9 +1647,12 @@ func (okInternal) Authenticate(*http.Request) error { return nil } func newTestAPI(repo Repo, cl Cluster) *API { return &API{Repo: repo, Cluster: cl, Internal: okInternal{}, RootDomain: testRoot, - Now: func() time.Time { return time.Unix(1_700_000_000, 0) }} + Now: func() time.Time { return time.Unix(1_700_000_000, 0) }, + AccessLog: discardLog} } +var discardLog = slog.New(slog.NewTextHandler(io.Discard, nil)) + func do(h http.Handler, method, target, body string, headers map[string]string) *httptest.ResponseRecorder { var r *http.Request if body == "" { diff --git a/internal/api/handlers_internal.go b/internal/api/handlers_internal.go index 5d4f6ae..132ad5c 100644 --- a/internal/api/handlers_internal.go +++ b/internal/api/handlers_internal.go @@ -3,6 +3,7 @@ package api import ( "context" "errors" + "log" "net/http" "felis.lolicon.best/internal/apis/felis/v1alpha1" @@ -24,7 +25,10 @@ func (a *API) handleReadyz(w http.ResponseWriter, r *http.Request) { } for name, check := range checks { if err := check(r.Context()); err != nil { - writeError(w, r, newError(http.StatusServiceUnavailable, "not_ready", "%s: %v", name, err)) + // The cause goes to the log; the probe answer names only the dependency, + // so a driver error (hosts, users, SQL) never reaches a caller. + log.Printf("api: readyz: %s: %v (request_id=%s)", name, err, requestIDFromContext(r.Context())) + writeError(w, r, newError(http.StatusServiceUnavailable, "not_ready", "%s is unavailable", name)) return } } diff --git a/internal/api/handlers_logstream.go b/internal/api/handlers_logstream.go index 412c4bd..68fd260 100644 --- a/internal/api/handlers_logstream.go +++ b/internal/api/handlers_logstream.go @@ -1,6 +1,7 @@ package api import ( + "context" "errors" "net/http" @@ -80,7 +81,11 @@ func (a *API) handleServerConsole(w http.ResponseWriter, r *http.Request) { // Open the follow stream. Every error must be resolved HERE, into a normal JSON // envelope, because relayLogStream commits the 200 + SSE headers and no error // body can follow it. - src, err := a.Logs.StreamLogs(r.Context(), name) + ctx := r.Context() + if since, ok := logSinceFromRequest(r, a.now()); ok { + ctx = withLogSince(ctx, since) + } + src, err := a.Logs.StreamLogs(ctx, name) switch { case errors.Is(err, ErrConsoleUnavailable): writeError(w, r, newError(http.StatusServiceUnavailable, "console_unavailable", @@ -99,5 +104,35 @@ func (a *API) handleServerConsole(w http.ResponseWriter, r *http.Request) { } a.audit(r, "console.attach", name) - relayLogStream(w, r, src) + relayLogStream(w, r, src, a.streamRecheck(r, func(ctx context.Context, p *Principal) error { + rec, err := a.Repo.ServerByName(ctx, name) + switch { + case errors.Is(err, ErrNotFound): + return errForbidden // the server is gone, and the grant with it + case err != nil: + return nil + case !a.isOwnerOrAdmin(p, rec): + return errForbidden + } + return nil + }), a.streamsClosing()) +} + +// streamRecheck builds the streamGuard for an external-face stream: it re-runs the +// authentication the stream opened with (the session may have been revoked or +// expired, the account disabled), the op.console staff gate, and then allow for the +// route's own rule. An unreachable session store is not a verdict (see streamGuard). +func (a *API) streamRecheck(r *http.Request, allow func(ctx context.Context, p *Principal) error) streamGuard { + return func(ctx context.Context) error { + p, err := a.External.Authenticate(r) + switch { + case errors.Is(err, errAuthBackend): + return nil + case err != nil || p == nil: + return errUnauthorized + case hostIsAdminConsole(r, a.RootDomain, a.AdminHostname) && !p.IsAdmin(): + return errForbidden + } + return allow(ctx, p) + } } diff --git a/internal/api/handlers_logstream_test.go b/internal/api/handlers_logstream_test.go index f1065b6..08bc40f 100644 --- a/internal/api/handlers_logstream_test.go +++ b/internal/api/handlers_logstream_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "os" + "strconv" "strings" "sync" "testing" @@ -376,7 +377,7 @@ type interleaveWriter struct { func (b *interleaveWriter) Write(p []byte) (int, error) { s := string(p) - if strings.HasPrefix(s, "data:") { + if strings.Contains(s, "\ndata:") { b.sawData = true } if s == sseHeartbeat { @@ -552,7 +553,7 @@ type firstDataWriter struct { func (b *firstDataWriter) Write(p []byte) (int, error) { n, err := b.ResponseWriter.Write(p) - if strings.HasPrefix(string(p), "data:") { + if strings.Contains(string(p), "\ndata:") { b.once.Do(func() { close(b.data) }) } return n, err @@ -687,7 +688,7 @@ func TestRelayLogStreamWriteDeadlineSeversStalledReader(t *testing.T) { done := make(chan struct{}) go func() { - relayLogStream(w, r, src) + relayLogStream(w, r, src, nil, nil) close(done) }() @@ -762,7 +763,7 @@ func TestRelayLogStreamClearsWriteDeadlineOnReturn(t *testing.T) { w := newDeadlineRecordWriter() r := httptest.NewRequest("GET", "/api/v1/servers/survival/console", nil) - relayLogStream(w, r, src) + relayLogStream(w, r, src, nil, nil) last, sawPositive := w.finalDeadline() if !sawPositive { @@ -775,3 +776,245 @@ func TestRelayLogStreamClearsWriteDeadlineOnReturn(t *testing.T) { t.Fatal("relay returned without closing the source") } } + +// switchExternal is an Authenticator a test can revoke mid-stream. +type switchExternal struct { + mu sync.Mutex + p *Principal + err error +} + +func (s *switchExternal) Authenticate(*http.Request) (*Principal, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.p, s.err +} + +func (s *switchExternal) set(p *Principal, err error) { + s.mu.Lock() + s.p, s.err = p, err + s.mu.Unlock() +} + +// vanishingServerRepo lets a test delete every server while a stream reads them. +type vanishingServerRepo struct { + *fakeRepo + mu sync.Mutex + gone bool +} + +func (v *vanishingServerRepo) vanish() { + v.mu.Lock() + v.gone = true + v.mu.Unlock() +} + +func (v *vanishingServerRepo) ServerByName(ctx context.Context, name string) (*ServerRecord, error) { + v.mu.Lock() + gone := v.gone + v.mu.Unlock() + if gone { + return nil, ErrNotFound + } + return v.fakeRepo.ServerByName(ctx, name) +} + +func shrinkStreamTimers(t *testing.T, recheck, lifetime time.Duration) { + t.Helper() + origRecheck, origLife := streamRecheckEvery, streamMaxLifetime + streamRecheckEvery, streamMaxLifetime = recheck, lifetime + t.Cleanup(func() { streamRecheckEvery, streamMaxLifetime = origRecheck, origLife }) +} + +// A stream is authorized when it opens; a session revoked (or a server handed to +// someone else) afterwards must not keep the console flowing. The relay re-asks +// on a timer and ends with "event: revoked" once the answer changes. +func TestServerConsoleStreamEndsWhenAccessIsWithdrawn(t *testing.T) { + owner := &Principal{UserID: "owner1", Email: "owner1@example.net", Role: "user"} + stranger := &Principal{UserID: "someone", Email: "someone@example.net", Role: "user"} + + for _, tc := range []struct { + name string + revoke func(ext *switchExternal, repo *vanishingServerRepo) + }{ + {"session revoked", func(ext *switchExternal, _ *vanishingServerRepo) { ext.set(nil, errUnauthorized) }}, + {"caller no longer owns the server", func(ext *switchExternal, _ *vanishingServerRepo) { ext.set(stranger, nil) }}, + {"server deleted", func(_ *switchExternal, repo *vanishingServerRepo) { repo.vanish() }}, + } { + t.Run(tc.name, func(t *testing.T) { + shrinkStreamTimers(t, 20*time.Millisecond, time.Hour) + repo := &vanishingServerRepo{fakeRepo: newFakeRepo()} + repo.byName["survival"] = &ServerRecord{Name: "survival", OwnerID: "owner1"} + ext := &switchExternal{p: owner} + a := newTestAPI(repo, newFakeCluster()) + a.External = ext + started := make(chan struct{}) + streamer := &fakeLogStreamer{srcFromCtx: func(ctx context.Context) io.ReadCloser { + return &ctxBlockingReadCloser{ctx: ctx, first: []byte("boot\n"), firstRead: started, closed: make(chan struct{})} + }} + a.Logs = streamer + + done := make(chan *httptest.ResponseRecorder) + go func() { done <- do(a.ExternalHandler(), "GET", "/api/v1/servers/survival/console", "", nil) }() + <-started + tc.revoke(ext, repo) + + select { + case w := <-done: + body := w.Body.String() + if !strings.Contains(body, "data: boot\n\n") || !strings.HasSuffix(body, sseRevoked) { + t.Fatalf("body = %q, want the boot line then %q", body, sseRevoked) + } + case <-time.After(2 * time.Second): + t.Fatal("the stream kept running after its grant was withdrawn") + } + }) + } +} + +// A store outage is no verdict on the caller: the stream keeps going (the lifetime +// cap still bounds it), and it ends with a plain close, which EventSource answers by +// reconnecting through the full auth path. +func TestServerConsoleStreamOutlivesAnAuthOutageUntilItsLifetime(t *testing.T) { + shrinkStreamTimers(t, 10*time.Millisecond, 80*time.Millisecond) + owner := &Principal{UserID: "owner1", Email: "owner1@example.net", Role: "user"} + repo := newFakeRepo() + repo.byName["survival"] = &ServerRecord{Name: "survival", OwnerID: "owner1"} + ext := &switchExternal{p: owner} + a := newTestAPI(repo, newFakeCluster()) + a.External = ext + started := make(chan struct{}) + a.Logs = &fakeLogStreamer{srcFromCtx: func(ctx context.Context) io.ReadCloser { + return &ctxBlockingReadCloser{ctx: ctx, first: []byte("boot\n"), firstRead: started, closed: make(chan struct{})} + }} + + begun := time.Now() + done := make(chan *httptest.ResponseRecorder) + go func() { done <- do(a.ExternalHandler(), "GET", "/api/v1/servers/survival/console", "", nil) }() + <-started + ext.set(nil, errAuthBackend) + + select { + case w := <-done: + if strings.Contains(w.Body.String(), "event: revoked") { + t.Fatalf("an auth outage revoked the stream: %q", w.Body.String()) + } + if since := time.Since(begun); since < 80*time.Millisecond { + t.Fatalf("stream ended after %v, before its lifetime", since) + } + case <-time.After(2 * time.Second): + t.Fatal("the stream outlived streamMaxLifetime") + } +} + +// CloseStreams (registered with RegisterOnShutdown) ends an open console at +// once, and a stream opened after it ends as soon as it starts. +func TestServerConsoleStreamEndsOnCloseStreams(t *testing.T) { + owner := &Principal{UserID: "owner1", Email: "owner1@example.net", Role: "user"} + repo := newFakeRepo() + repo.byName["survival"] = &ServerRecord{Name: "survival", OwnerID: "owner1"} + a := newTestAPI(repo, newFakeCluster()) + a.External = staticExternal{p: owner} + var closed []chan struct{} + a.Logs = &fakeLogStreamer{srcFromCtx: func(ctx context.Context) io.ReadCloser { + c := make(chan struct{}) + closed = append(closed, c) + return &ctxBlockingReadCloser{ctx: ctx, first: []byte("boot\n"), firstRead: make(chan struct{}), closed: c} + }} + + open := func() chan *httptest.ResponseRecorder { + done := make(chan *httptest.ResponseRecorder, 1) + go func() { done <- do(a.ExternalHandler(), "GET", "/api/v1/servers/survival/console", "", nil) }() + return done + } + wait := func(done chan *httptest.ResponseRecorder, what string) { + t.Helper() + select { + case w := <-done: + if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "event: revoked") { + t.Fatalf("%s: code %d body %q, want a plain close", what, w.Code, w.Body.String()) + } + case <-time.After(2 * time.Second): + t.Fatalf("%s: the stream did not end", what) + } + } + + first := open() + select { + case <-first: + t.Fatal("the stream ended before CloseStreams") + case <-time.After(50 * time.Millisecond): + } + a.CloseStreams() + a.CloseStreams() // every listener's shutdown calls it + wait(first, "open stream") + wait(open(), "stream opened after CloseStreams") + for i, c := range closed { + select { + case <-c: + default: + t.Fatalf("source %d was never closed", i) + } + } +} + +// A reconnecting EventSource sends back the id of the last line it saw; the +// handler resumes the follow from that second instead of replaying the backlog. +func TestServerConsoleStreamResumesFromLastEventID(t *testing.T) { + owner := &Principal{UserID: "owner1", Email: "owner1@example.net", Role: "user"} + repo := newFakeRepo() + repo.byName["survival"] = &ServerRecord{Name: "survival", OwnerID: "owner1"} + a := newTestAPI(repo, newFakeCluster()) + a.External = staticExternal{p: owner} + streamer := &fakeLogStreamer{src: &recordReadCloser{r: strings.NewReader("line\n")}} + a.Logs = streamer + + last := a.now().Add(-2 * time.Minute).Truncate(time.Second) + w := do(a.ExternalHandler(), "GET", "/api/v1/servers/survival/console", "", + map[string]string{"Last-Event-ID": strconv.FormatInt(last.Unix(), 10)}) + if w.Code != http.StatusOK { + t.Fatalf("code = %d body %s", w.Code, w.Body.String()) + } + if got, ok := logSinceFromContext(streamer.gotCtx); !ok || !got.Equal(last) { + t.Fatalf("streamer since = %v (%v), want %v", got, ok, last) + } + if !strings.Contains(w.Body.String(), "id: ") { + t.Fatalf("relayed lines carry no id for the next resume: %q", w.Body.String()) + } +} + +func TestLogSinceFromRequest(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + for _, tc := range []struct { + header string + want time.Time + ok bool + }{ + {"", time.Time{}, false}, + {"garbage", time.Time{}, false}, + {strconv.FormatInt(now.Add(-time.Minute).Unix(), 10), now.Add(-time.Minute), true}, + {strconv.FormatInt(now.Add(time.Minute).Unix(), 10), time.Time{}, false}, + {strconv.FormatInt(now.Add(-2*maxResumeAge).Unix(), 10), time.Time{}, false}, + } { + r := httptest.NewRequest("GET", "/", nil) + if tc.header != "" { + r.Header.Set("Last-Event-ID", tc.header) + } + got, ok := logSinceFromRequest(r, now) + if ok != tc.ok || !got.Equal(tc.want) { + t.Errorf("Last-Event-ID %q: got %v %v, want %v %v", tc.header, got, ok, tc.want, tc.ok) + } + } +} + +func TestPodLogOptionsTailOrResume(t *testing.T) { + opts := podLogOptions(context.Background(), "minecraft", 200) + if opts.TailLines == nil || *opts.TailLines != 200 || opts.SinceTime != nil || !opts.Follow { + t.Fatalf("fresh attach options = %+v, want a followed 200-line tail", opts) + } + since := time.Unix(1_800_000_000, 0) + opts = podLogOptions(withLogSince(context.Background(), since), "minecraft", 200) + if opts.TailLines != nil || opts.SinceTime == nil || !opts.SinceTime.Time.Equal(since) { + t.Fatalf("resume options = %+v, want sinceTime %v and no tail", opts, since) + } +} diff --git a/internal/api/handlers_user.go b/internal/api/handlers_user.go index 1ea38d4..c4ee5b0 100644 --- a/internal/api/handlers_user.go +++ b/internal/api/handlers_user.go @@ -176,9 +176,36 @@ func (a *API) handleStatus(w http.ResponseWriter, r *http.Request) { a.writeLookupError(w, r, err) return } + // The internal face (no principal) is the platform itself. On the external + // face the full record (image, resources, endpoint) is the owner's and the + // staff's; anyone else signed in sees what the game's own server list shows. + if p := principalFromContext(r.Context()); p != nil { + rec, err := a.Repo.ServerByName(r.Context(), name) + if err != nil && !errors.Is(err, ErrNotFound) { + writeError(w, r, err) + return + } + if !a.isOwnerOrAdmin(p, rec) { + info = publicServerInfo(info) + } + } writeJSON(w, http.StatusOK, info) } +// publicServerInfo keeps the fields any signed-in caller may see of a server +// that is not theirs. +func publicServerInfo(s *ServerInfo) *ServerInfo { + return &ServerInfo{ + Name: s.Name, + Subdomain: s.Subdomain, + DisplayName: s.DisplayName, + Phase: s.Phase, + Ready: s.Ready, + PlayersOnline: s.PlayersOnline, + PlayersMax: s.PlayersMax, + } +} + // handleMe returns the calling principal's own identity (spec §14 tiering). The // panel reads it once at boot to decide which navigation surfaces to render: // the User-Side for everyone, the Admin/SysAdmin sides only when is_admin. This diff --git a/internal/api/images.go b/internal/api/images.go index 3a96064..d20bfbd 100644 --- a/internal/api/images.go +++ b/internal/api/images.go @@ -137,7 +137,11 @@ func (a *API) handleBuildLogs(w http.ResponseWriter, r *http.Request) { } defer release() - src, err := a.BuildLogs.StreamLogs(r.Context(), id) + ctx := r.Context() + if since, ok := logSinceFromRequest(r, a.now()); ok { + ctx = withLogSince(ctx, since) + } + src, err := a.BuildLogs.StreamLogs(ctx, id) switch { case errors.Is(err, ErrNotFound): writeError(w, r, newError(http.StatusNotFound, "not_found", @@ -153,7 +157,12 @@ func (a *API) handleBuildLogs(w http.ResponseWriter, r *http.Request) { return } a.audit(r, "image.build.logs", id) - relayLogStream(w, r, src) + relayLogStream(w, r, src, a.streamRecheck(r, func(_ context.Context, p *Principal) error { + if !p.IsAdmin() { + return errForbidden + } + return nil + }), a.streamsClosing()) } // handleCancelBuild cancels an in-flight build (admin-tier). A build that has diff --git a/internal/api/jobstatus.go b/internal/api/jobstatus.go index bad09e7..09666ff 100644 --- a/internal/api/jobstatus.go +++ b/internal/api/jobstatus.go @@ -17,8 +17,8 @@ type AsyncJob struct { Kind string `json:"kind"` // "backup" | "restore" State string `json:"state"` // "running" | "succeeded" | "failed" Message string `json:"message,omitempty"` - StartedAt time.Time `json:"started_at,omitempty"` - FinishedAt time.Time `json:"finished_at,omitempty"` + StartedAt time.Time `json:"started_at,omitzero"` + FinishedAt time.Time `json:"finished_at,omitzero"` } // JobStatusReader reads the newest backup/restore Jobs for a server, newest diff --git a/internal/api/logstream.go b/internal/api/logstream.go index df1aa7e..c604112 100644 --- a/internal/api/logstream.go +++ b/internal/api/logstream.go @@ -5,6 +5,7 @@ import ( "context" "io" "net/http" + "strconv" "time" "felis.lolicon.best/internal/apis/felis/v1alpha1" @@ -75,6 +76,80 @@ var heartbeatInterval = 25 * time.Second // reassigns it. var writeTimeout = 30 * time.Second +// A stream is authorized once, when it opens, and then may run for hours. So the +// relay asks again every streamRecheckEvery (the caller's streamGuard: is the +// session still live, does the caller still own the server) and ends the stream +// after streamMaxLifetime regardless. A withdrawn grant gets an "event: revoked" +// before the close, which the panel treats as final; the lifetime cap is a plain +// close, which EventSource answers by reconnecting through the full auth path, +// resuming from its Last-Event-ID (see logSinceFromRequest) instead of replaying +// the backlog. Both are vars only so a test can shrink them. +var ( + streamRecheckEvery = time.Minute + streamMaxLifetime = 30 * time.Minute +) + +// streamGuard re-checks a running stream's authorization. nil means it still +// holds; an error ends the stream with "event: revoked". A guard that cannot +// reach its store should return nil: an outage is no verdict on the caller, and +// streamMaxLifetime still bounds how long the stream can outlive a revocation. +type streamGuard func(ctx context.Context) error + +// sseRevoked is the last event a stream whose grant was withdrawn receives. +const sseRevoked = "event: revoked\ndata: access to this stream was withdrawn\n\n" + +// maxResumeAge bounds how far back a Last-Event-ID may resume: past it the pod +// has likely restarted anyway, and the tailed backlog is the better start. +const maxResumeAge = time.Hour + +type logSinceKey struct{} + +// withLogSince asks the LogStreamer to start the follow at since instead of the +// tailed backlog. It rides the context so the LogStreamer interface (and its test +// fakes) stays one method. +func withLogSince(ctx context.Context, since time.Time) context.Context { + return context.WithValue(ctx, logSinceKey{}, since) +} + +func logSinceFromContext(ctx context.Context) (time.Time, bool) { + t, ok := ctx.Value(logSinceKey{}).(time.Time) + return t, ok && !t.IsZero() +} + +// logSinceFromRequest reads the resume point an EventSource sends on reconnect. +// Every relayed line carries "id: ", so Last-Event-ID is the second +// the client last heard from; resuming at that second may repeat a line or two +// from it, which beats replaying 200 lines of backlog on every reconnect. Anything +// unparsable, in the future, or older than maxResumeAge is ignored. +func logSinceFromRequest(r *http.Request, now time.Time) (time.Time, bool) { + raw := r.Header.Get("Last-Event-ID") + if raw == "" { + return time.Time{}, false + } + secs, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return time.Time{}, false + } + t := time.Unix(secs, 0) + if t.After(now) || now.Sub(t) > maxResumeAge { + return time.Time{}, false + } + return t, true +} + +// podLogOptions is the follow request both streamers send: the tailed backlog, +// or everything since the resume point when the client is reconnecting. +func podLogOptions(ctx context.Context, container string, tail int64) *corev1.PodLogOptions { + opts := &corev1.PodLogOptions{Container: container, Follow: true} + if since, ok := logSinceFromContext(ctx); ok { + st := metav1.NewTime(since) + opts.SinceTime = &st + return opts + } + opts.TailLines = &tail + return opts +} + // relayLogStream is the shared §8 read-side relay: it copies a line-oriented log // source to the client as Server-Sent Events (spec §262 SSE, NOT WebSocket). It // is the single reusable artifact the server console (handleServerConsole) and, @@ -91,7 +166,11 @@ var writeTimeout = 30 * time.Second // r.Context(), so a client disconnect cancels it, the underlying Read errors, the // scan loop exits, and the deferred Close releases the upstream stream (no leaked // apiserver connection). -func relayLogStream(w http.ResponseWriter, r *http.Request, src io.ReadCloser) { +// +// still, when non-nil, is asked every streamRecheckEvery whether the caller may +// keep reading; the stream also ends after streamMaxLifetime (see streamGuard), +// and when closing is closed (API.CloseStreams, at shutdown). +func relayLogStream(w http.ResponseWriter, r *http.Request, src io.ReadCloser, still streamGuard, closing <-chan struct{}) { defer src.Close() // SSE needs per-event flushing; without a Flusher the bytes buffer and never @@ -162,6 +241,10 @@ func relayLogStream(w http.ResponseWriter, r *http.Request, src io.ReadCloser) { ticker := time.NewTicker(heartbeatInterval) defer ticker.Stop() + recheck := time.NewTicker(streamRecheckEvery) + defer recheck.Stop() + expire := time.NewTimer(streamMaxLifetime) + defer expire.Stop() for { select { @@ -174,9 +257,21 @@ func relayLogStream(w http.ResponseWriter, r *http.Request, src io.ReadCloser) { // One log line → one SSE "data:" event, written under a per-write deadline // so a stalled reader severs the stream (writeChunk → false) instead of // pinning this goroutine; the deferred Close then tears the upstream down. - if !writeChunk(rc, w, "data: "+line+"\n\n") { + // The id is the resume point a reconnecting EventSource sends back. + if !writeChunk(rc, w, "id: "+strconv.FormatInt(time.Now().Unix(), 10)+"\ndata: "+line+"\n\n") { return } + case <-recheck.C: + if still != nil && still(ctx) != nil { + writeChunk(rc, w, sseRevoked) + return + } + case <-expire.C: + // A plain close: the client reconnects and is authorized afresh. + return + case <-closing: + // The server is shutting down; the client reconnects to the next one. + return case <-ticker.C: // No line for a whole interval: emit a comment so the connection stays // warm past the proxy idle timeout — same deadline-guarded write, so a @@ -282,12 +377,8 @@ func (k *K8sLogStreamer) StreamLogs(ctx context.Context, name string) (io.ReadCl return nil, ErrNotFound } - tail := k.tailLines - stream, err := k.clientset.CoreV1().Pods(k.namespace).GetLogs(podName, &corev1.PodLogOptions{ - Container: serverLogContainer, - Follow: true, - TailLines: &tail, - }).Stream(ctx) + stream, err := k.clientset.CoreV1().Pods(k.namespace).GetLogs(podName, + podLogOptions(ctx, serverLogContainer, k.tailLines)).Stream(ctx) if err != nil { return nil, ErrConsoleUnavailable } @@ -354,12 +445,8 @@ func (k *K8sBuildLogStreamer) StreamLogs(ctx context.Context, buildID string) (i // at most one Pod, so the first match is the build's Pod. podName := pods.Items[0].Name - tail := k.tailLines - stream, err := k.clientset.CoreV1().Pods(k.namespace).GetLogs(podName, &corev1.PodLogOptions{ - Container: build.ContainerKaniko, - Follow: true, - TailLines: &tail, - }).Stream(ctx) + stream, err := k.clientset.CoreV1().Pods(k.namespace).GetLogs(podName, + podLogOptions(ctx, build.ContainerKaniko, k.tailLines)).Stream(ctx) if err != nil { return nil, ErrConsoleUnavailable } diff --git a/internal/api/metrics.go b/internal/api/metrics.go index 01d6e9b..b38562a 100644 --- a/internal/api/metrics.go +++ b/internal/api/metrics.go @@ -6,6 +6,7 @@ import ( "felis.lolicon.best/internal/metrics" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -21,6 +22,12 @@ func newAPIMetricsHandler() http.Handler { // A fresh registry cannot already hold a collector, so Register's // AlreadyRegistered tolerance arm never triggers here. _ = metrics.Register(reg) + // The request series (observe.go) and the runtime: goroutines, heap, GC and + // the process's fds and RSS, which is where a leaked stream or a slow + // connection pile-up shows first. + reg.MustRegister(httpRequestsTotal, httpRequestDuration, + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) return promhttp.HandlerFor(reg, promhttp.HandlerOpts{}) } diff --git a/internal/api/middleware.go b/internal/api/middleware.go index 201786c..91e1104 100644 --- a/internal/api/middleware.go +++ b/internal/api/middleware.go @@ -5,9 +5,13 @@ import ( "crypto/rand" "encoding/hex" "errors" + "io" "log" "net/http" + "net/url" "runtime/debug" + "strings" + "time" ) // withRequestID assigns a request id (honoring a WELL-FORMED inbound X-Request-Id) @@ -84,6 +88,128 @@ func withRecover(next http.Handler) http.Handler { }) } +// withSecurityHeaders sets the response headers every API answer carries. The +// API serves JSON, event streams and downloads, never a page, so its CSP allows +// nothing and no frame may embed it. HSTS goes out on what the Cloudflare edge +// served over HTTPS (viaTLSEdge). +func withSecurityHeaders(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := w.Header() + h.Set("X-Content-Type-Options", "nosniff") + h.Set("X-Frame-Options", "DENY") + h.Set("Referrer-Policy", "no-referrer") + h.Set("Content-Security-Policy", "default-src 'none'; frame-ancestors 'none'") + if viaTLSEdge(r) { + h.Set("Strict-Transport-Security", "max-age=31536000") + } + next.ServeHTTP(w, r) + }) +} + +// viaTLSEdge reports a request the Cloudflare edge took over HTTPS. The tunnel +// reaches the origin over TLS as well (the self-signed NodePort listener), so +// r.TLS says nothing about what the browser saw; the edge's X-Forwarded-Proto +// does. A browser that comes straight to the self-signed listener sends no such +// header, and would drop the policy anyway: HSTS that arrives over a certificate +// error, or for an IP-literal host, is ignored (RFC 6797 §8.1). +func viaTLSEdge(r *http.Request) bool { + return strings.EqualFold(r.Header.Get("X-Forwarded-Proto"), "https") +} + +// rejectCrossSiteWrites refuses a state-changing request a browser sent from +// another site. The session cookie is SameSite=Lax, which keeps it off cross-site +// POSTs but still sends it from a sibling subdomain (same-site), and a player's +// server can serve pages under the install's root domain. Browsers say where a +// request came from in Sec-Fetch-Site; one too old for that still sends Origin on +// a POST. A request with neither is not from a browser (the plugins, the CLI) and +// carries no ambient cookie to abuse. +func rejectCrossSiteWrites(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet, http.MethodHead, http.MethodOptions: + default: + if !sameOriginRequest(r) { + writeError(w, r, newError(http.StatusForbidden, "cross_site", + "requests from another site may not change anything here")) + return + } + } + next.ServeHTTP(w, r) + }) +} + +func sameOriginRequest(r *http.Request) bool { + switch r.Header.Get("Sec-Fetch-Site") { + case "same-origin", "none": + return true + case "": + default: // same-site, cross-site + return false + } + origin := r.Header.Get("Origin") + if origin == "" { + return true + } + u, err := url.Parse(origin) + return err == nil && u.Host != "" && strings.EqualFold(u.Host, r.Host) +} + +// Request bodies get a read deadline that grows with what has arrived: bodyGrace +// to get going, plus the time the bytes so far would take at bodyMinRate. A JSON +// body (1 MiB at most) has well under a minute; a 1 GiB build context uploads as +// long as it keeps averaging bodyMinRate; a client trickling a byte now and then +// to hold a connection and a goroutine is cut off. The server sets no ReadTimeout +// (it would cut the long SSE responses), so without this nothing bounded the body. +// Vars only so a test can shrink them. +var ( + bodyGrace = 30 * time.Second + bodyMinRate = float64(16 << 10) // bytes per second +) + +func withBodyDeadline(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Body == nil || r.Body == http.NoBody { + next.ServeHTTP(w, r) + return + } + rc := http.NewResponseController(w) + start := time.Now() + if err := rc.SetReadDeadline(start.Add(bodyGrace)); err != nil { + next.ServeHTTP(w, r) // a writer without deadlines (tests) + return + } + r.Body = &deadlineBody{ReadCloser: r.Body, rc: rc, start: start} + next.ServeHTTP(w, r) + }) +} + +// deadlineBody moves the connection's read deadline as the body arrives, and +// clears it once the body ends: from then on net/http reads the connection only +// to notice a disconnect, and a deadline meant for the body would cancel the +// request's context under a handler that is still working. +type deadlineBody struct { + io.ReadCloser + rc *http.ResponseController + start time.Time + read int64 + done bool +} + +func (b *deadlineBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + b.read += int64(n) + switch { + case b.done: + case err != nil: + b.done = true + _ = b.rc.SetReadDeadline(time.Time{}) + case n > 0: + allowed := bodyGrace + time.Duration(float64(b.read)/bodyMinRate*float64(time.Second)) + _ = b.rc.SetReadDeadline(b.start.Add(allowed)) + } + return n, err +} + // requireInternal enforces service-token auth for the internal face. It never // applies Zero Trust (spec §14 red line). func (a *API) requireInternal(next http.Handler) http.Handler { diff --git a/internal/api/observe.go b/internal/api/observe.go new file mode 100644 index 0000000..09a7383 --- /dev/null +++ b/internal/api/observe.go @@ -0,0 +1,183 @@ +package api + +import ( + "context" + "log/slog" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// Request observability: one access-log line and two series per API request. +// +// The route label is the matched ServeMux pattern ("/api/v1/servers/{name}"), +// never the raw path, so a scan of random URLs cannot grow the series set; a +// request no route matched is "unmatched". The method label is folded the same +// way (methodLabel). Stream routes (text/event-stream) count toward the request +// total but stay out of the duration histogram, where an attachment that lasts +// half an hour would only bury the latency of everything else. + +var ( + httpRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "felis", + Name: "http_requests_total", + Help: "API requests served, by face, method, route pattern and status code.", + }, []string{"face", "method", "route", "code"}) + + httpRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "felis", + Name: "http_request_duration_seconds", + Help: "API request latency, by face and route pattern (streams excluded).", + Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30}, + }, []string{"face", "route"}) +) + +// defaultAccessLog writes logfmt lines to stderr, next to the API's other logs. +var defaultAccessLog = slog.New(slog.NewTextHandler(os.Stderr, nil)) + +// routeUnmatched labels a request no route pattern matched. +const routeUnmatched = "unmatched" + +// reqInfo is filled in by the handlers a request passes through, for observe to +// read once the response is done: the matched route, and the principal once the +// face's guard has resolved one. +type reqInfo struct { + route string + principal string +} + +func reqInfoFrom(ctx context.Context) *reqInfo { + info, _ := ctx.Value(ctxKeyReqInfo).(*reqInfo) + return info +} + +// noteRoute records the matched route pattern (without its method). +func noteRoute(r *http.Request, pattern string) { + if info := reqInfoFrom(r.Context()); info != nil { + if _, path, ok := strings.Cut(pattern, " "); ok { + pattern = path + } + info.route = pattern + } +} + +// tagRoute wraps a route's handler so the request records its pattern and, on +// the authenticated faces, who made it. +func tagRoute(pattern string, h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + noteRoute(r, pattern) + if info := reqInfoFrom(r.Context()); info != nil { + if p := principalFromContext(r.Context()); p != nil { + info.principal = p.UserID + } + } + h(w, r) + } +} + +// methodLabel folds anything but the methods the API serves into one value. +func methodLabel(m string) string { + switch m { + case http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, + http.MethodPatch, http.MethodDelete, http.MethodOptions: + return m + } + return "OTHER" +} + +// quietRoute reports whether a successful request is left out of the access log: +// the probes and the scrape, which arrive every few seconds and say nothing. +func quietRoute(route string, status int) bool { + if status >= 400 { + return false + } + switch route { + case "/healthz", "/readyz", "/metrics": + return true + } + return false +} + +// observe records every request that reaches face: the access-log line and the +// felis_http_* series. +func (a *API) observe(face string, next http.Handler) http.Handler { + logger := a.AccessLog + if logger == nil { + logger = defaultAccessLog + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + info := &reqInfo{} + rec := &statusRecorder{ResponseWriter: w} + next.ServeHTTP(rec, r.WithContext(context.WithValue(r.Context(), ctxKeyReqInfo, info))) + + status := rec.status + if status == 0 { + status = http.StatusOK + } + route := info.route + if route == "" { + route = routeUnmatched + } + elapsed := time.Since(start) + httpRequestsTotal.WithLabelValues(face, methodLabel(r.Method), route, strconv.Itoa(status)).Inc() + if !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/event-stream") { + httpRequestDuration.WithLabelValues(face, route).Observe(elapsed.Seconds()) + } + if quietRoute(route, status) { + return + } + logger.LogAttrs(r.Context(), slog.LevelInfo, "request", + slog.String("face", face), + slog.String("method", r.Method), + slog.String("route", route), + slog.String("path", r.URL.Path), + slog.Int("status", status), + slog.Int64("duration_ms", elapsed.Milliseconds()), + slog.Int64("bytes", rec.bytes), + slog.String("request_id", requestIDFromContext(r.Context())), + slog.String("principal", info.principal), + ) + }) +} + +// statusRecorder notes the status and size of a response. It passes flushes +// through with their error (the SSE relay's write-deadline guard depends on +// seeing a failed flush) and unwraps for http.ResponseController, so deadlines +// still reach the connection. +type statusRecorder struct { + http.ResponseWriter + status int + bytes int64 +} + +func (s *statusRecorder) WriteHeader(code int) { + if s.status == 0 { + s.status = code + } + s.ResponseWriter.WriteHeader(code) +} + +func (s *statusRecorder) Write(p []byte) (int, error) { + if s.status == 0 { + s.status = http.StatusOK + } + n, err := s.ResponseWriter.Write(p) + s.bytes += int64(n) + return n, err +} + +func (s *statusRecorder) FlushError() error { + if s.status == 0 { + s.status = http.StatusOK + } + return http.NewResponseController(s.ResponseWriter).Flush() +} + +func (s *statusRecorder) Flush() { _ = s.FlushError() } + +func (s *statusRecorder) Unwrap() http.ResponseWriter { return s.ResponseWriter } diff --git a/internal/api/observe_test.go b/internal/api/observe_test.go new file mode 100644 index 0000000..7543f97 --- /dev/null +++ b/internal/api/observe_test.go @@ -0,0 +1,310 @@ +package api + +import ( + "bytes" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + dto "github.com/prometheus/client_model/go" +) + +func counterValue(t *testing.T, labels ...string) float64 { + t.Helper() + var m dto.Metric + if err := httpRequestsTotal.WithLabelValues(labels...).Write(&m); err != nil { + t.Fatal(err) + } + return m.GetCounter().GetValue() +} + +// accessLines decodes the JSON access-log lines written to buf. +func accessLines(t *testing.T, buf *bytes.Buffer) []map[string]any { + t.Helper() + var out []map[string]any + for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") { + if line == "" { + continue + } + var m map[string]any + if err := json.Unmarshal([]byte(line), &m); err != nil { + t.Fatalf("access log line %q: %v", line, err) + } + out = append(out, m) + } + return out +} + +// TestObserveLabelsByRoutePattern pins the access log and the request series to +// the matched route pattern: a server name never becomes a label, a path no route +// matched is "unmatched", the caller is named, and the probes stay out of the log. +func TestObserveLabelsByRoutePattern(t *testing.T) { + repo := newFakeRepo() + repo.byName["survival"] = &ServerRecord{Name: "survival", OwnerID: "owner1"} + cl := newFakeCluster() + cl.byName["survival"] = &ServerInfo{Name: "survival", Phase: "Running"} + a := newTestAPI(repo, cl) + a.External = staticExternal{p: &Principal{UserID: "owner1", Email: "owner1@example.net", Role: "user"}} + var buf bytes.Buffer + a.AccessLog = slog.New(slog.NewJSONHandler(&buf, nil)) + h := a.ExternalHandler() + + const route = "/api/v1/servers/{name}/status" + before := counterValue(t, "external", "GET", route, "200") + if w := do(h, "GET", "/api/v1/servers/survival/status", "", map[string]string{"X-Request-Id": "trace-1"}); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body) + } + if got := counterValue(t, "external", "GET", route, "200") - before; got != 1 { + t.Fatalf("felis_http_requests_total{route=%q} grew by %v, want 1", route, got) + } + + unmatched := counterValue(t, "external", "GET", routeUnmatched, "404") + if w := do(h, "GET", "/api/v1/no/such/thing", "", nil); w.Code != http.StatusNotFound { + t.Fatalf("unknown path = %d, want 404", w.Code) + } + if got := counterValue(t, "external", "GET", routeUnmatched, "404") - unmatched; got != 1 { + t.Fatalf("unknown path counted %v times under %q, want 1", got, routeUnmatched) + } + + do(h, "GET", "/healthz", "", nil) + + lines := accessLines(t, &buf) + if len(lines) != 2 { + t.Fatalf("access log has %d lines, want 2 (the probe is quiet): %s", len(lines), buf.String()) + } + first := lines[0] + for k, want := range map[string]any{ + "msg": "request", "face": "external", "method": "GET", "route": route, + "path": "/api/v1/servers/survival/status", "status": float64(200), + "request_id": "trace-1", "principal": "owner1", + } { + if first[k] != want { + t.Errorf("access log %s = %v, want %v", k, first[k], want) + } + } + if first["bytes"].(float64) <= 0 { + t.Errorf("access log bytes = %v, want the body size", first["bytes"]) + } + if lines[1]["route"] != routeUnmatched || lines[1]["status"] != float64(404) { + t.Errorf("unknown path logged as %v", lines[1]) + } +} + +// TestNoRouteEnvelopes: a path no route serves is a 404 envelope, and a known path +// asked with the wrong method is a 405 that names the methods it does take. +func TestNoRouteEnvelopes(t *testing.T) { + a := newTestAPI(newFakeRepo(), newFakeCluster()) + a.External = staticExternal{p: &Principal{UserID: "owner1", Email: "owner1@example.net", Role: "user"}} + + for _, face := range []struct { + name string + h http.Handler + }{{"external", a.ExternalHandler()}, {"internal", a.InternalHandler()}} { + w := do(face.h, "GET", "/api/v1/definitely-not-a-route", "", nil) + if w.Code != http.StatusNotFound || errCode(w.Body.Bytes()) != "not_found" { + t.Fatalf("%s unknown path = %d %s, want 404 not_found", face.name, w.Code, w.Body) + } + w = do(face.h, "DELETE", "/healthz", "", nil) + if w.Code != http.StatusMethodNotAllowed || errCode(w.Body.Bytes()) != "method_not_allowed" { + t.Fatalf("%s DELETE /healthz = %d %s, want 405 method_not_allowed", face.name, w.Code, w.Body) + } + if got := w.Header().Get("Allow"); got != "GET" { + t.Fatalf("%s DELETE /healthz Allow = %q, want GET", face.name, got) + } + } + + // An authenticated route answers 405 the same way. + w := do(a.ExternalHandler(), "PUT", "/api/v1/servers/survival/status", "", nil) + if w.Code != http.StatusMethodNotAllowed || w.Header().Get("Allow") != "GET" { + t.Fatalf("PUT status = %d Allow %q, want 405 Allow GET", w.Code, w.Header().Get("Allow")) + } +} + +// TestOversizedJSONBodyIs413: a JSON body past maxBodyBytes is refused as too +// large, where it used to read as a malformed one. +func TestOversizedJSONBodyIs413(t *testing.T) { + r := httptest.NewRequest("POST", "/x", strings.NewReader(`{"name":"`+strings.Repeat("a", maxBodyBytes)+`"}`)) + w := httptest.NewRecorder() + var v struct{ Name string } + err := decodeJSON(w, r, &v) + writeError(w, r, err) + if w.Code != http.StatusRequestEntityTooLarge || errCode(w.Body.Bytes()) != "too_large" { + t.Fatalf("oversized body = %d %s, want 413 too_large", w.Code, w.Body) + } +} + +// TestSecurityHeaders: every API answer forbids sniffing, framing and referrers, +// and HSTS goes out on what the edge served over HTTPS. +func TestSecurityHeaders(t *testing.T) { + h := newTestAPI(newFakeRepo(), newFakeCluster()).ExternalHandler() + + w := do(h, "GET", "/healthz", "", nil) + for k, want := range map[string]string{ + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "DENY", + "Referrer-Policy": "no-referrer", + "Content-Security-Policy": "default-src 'none'; frame-ancestors 'none'", + } { + if got := w.Header().Get(k); got != want { + t.Errorf("%s = %q, want %q", k, got, want) + } + } + if got := w.Header().Get("Strict-Transport-Security"); got != "" { + t.Errorf("HSTS on a direct request = %q, want none", got) + } + + // The tunnel reaches the origin over TLS as well. + r := httptest.NewRequest("GET", "https://console."+testRoot+"/healthz", nil) + r.Header.Set("X-Forwarded-Proto", "https") + w = httptest.NewRecorder() + h.ServeHTTP(w, r) + if got := w.Header().Get("Strict-Transport-Security"); got == "" { + t.Error("no HSTS behind the TLS edge") + } +} + +// TestCrossSiteWritesRefused: a browser write from another site (a sibling +// subdomain included) is refused before auth; reads, same-origin writes and +// non-browser callers pass. +func TestCrossSiteWritesRefused(t *testing.T) { + a := newTestAPI(newFakeRepo(), newFakeCluster()) + a.External = staticExternal{p: &Principal{UserID: "owner1", Email: "owner1@example.net", Role: "user"}} + h := a.ExternalHandler() + const host = "console." + testRoot + + send := func(method string, hdr map[string]string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, "/api/v1/servers/survival/stop", nil) + r.Host = host + for k, v := range hdr { + r.Header.Set(k, v) + } + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return w + } + + for _, c := range []struct { + name string + hdr map[string]string + }{ + {"cross-site", map[string]string{"Sec-Fetch-Site": "cross-site"}}, + {"sibling subdomain", map[string]string{"Sec-Fetch-Site": "same-site", "Origin": "https://evil." + testRoot}}, + {"foreign Origin only", map[string]string{"Origin": "https://evil.example.org"}}, + {"null Origin", map[string]string{"Origin": "null"}}, + } { + w := send("POST", c.hdr) + if w.Code != http.StatusForbidden || errCode(w.Body.Bytes()) != "cross_site" { + t.Errorf("%s: POST = %d %s, want 403 cross_site", c.name, w.Code, w.Body) + } + } + + for _, c := range []struct { + name string + hdr map[string]string + }{ + {"same-origin", map[string]string{"Sec-Fetch-Site": "same-origin", "Origin": "https://" + host}}, + {"typed URL", map[string]string{"Sec-Fetch-Site": "none"}}, + {"Origin matches Host", map[string]string{"Origin": "https://" + host}}, + {"no browser headers", nil}, + } { + if w := send("POST", c.hdr); w.Code == http.StatusForbidden && errCode(w.Body.Bytes()) == "cross_site" { + t.Errorf("%s: POST refused as cross-site", c.name) + } + } + + if w := send("GET", map[string]string{"Sec-Fetch-Site": "cross-site"}); w.Code == http.StatusForbidden && errCode(w.Body.Bytes()) == "cross_site" { + t.Error("a cross-site GET was refused; only writes are fenced") + } +} + +// TestBodyDeadlineCutsATrickle runs a real server: a body that stops arriving is +// cut off at the grace period, and a body that did arrive clears the deadline so +// the request context outlives it while the handler keeps working. +func TestBodyDeadlineCutsATrickle(t *testing.T) { + oldGrace, oldRate := bodyGrace, bodyMinRate + bodyGrace, bodyMinRate = 150*time.Millisecond, 1<<20 + defer func() { bodyGrace, bodyMinRate = oldGrace, oldRate }() + + type outcome struct { + readErr error + ctxErr error + } + results := make(chan outcome, 1) + srv := httptest.NewServer(withBodyDeadline(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := io.ReadAll(r.Body) + if err == nil { + time.Sleep(3 * bodyGrace) + } + results <- outcome{readErr: err, ctxErr: r.Context().Err()} + }))) + defer srv.Close() + + // A whole body, then a handler slower than the grace period. + resp, err := http.Post(srv.URL, "application/json", strings.NewReader(`{"ok":true}`)) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if o := <-results; o.readErr != nil || o.ctxErr != nil { + t.Fatalf("complete body: read err %v, ctx err %v; want both nil", o.readErr, o.ctxErr) + } + + // A body that sends a few bytes and stalls. + pr, pw := io.Pipe() + defer pw.Close() + go func() { _, _ = pw.Write([]byte(`{"slow":`)) }() + req, _ := http.NewRequest("POST", srv.URL, pr) + go func() { + if resp, err := http.DefaultClient.Do(req); err == nil { + resp.Body.Close() + } + }() + select { + case o := <-results: + if o.readErr == nil { + t.Fatal("a stalled body read to the end") + } + case <-time.After(5 * time.Second): + t.Fatal("a stalled body was never cut off") + } +} + +// TestStatusTrimmedForOthers: GET /servers/{name}/status gives the owner the whole +// record and any other signed-in user what the game's server list shows. +func TestStatusTrimmedForOthers(t *testing.T) { + repo := newFakeRepo() + repo.byName["survival"] = &ServerRecord{Name: "survival", OwnerID: "owner1"} + cl := newFakeCluster() + cl.byName["survival"] = &ServerInfo{Name: "survival", Subdomain: "survival", Phase: "Running", Ready: true, + Image: "paper-1.21", JavaMemory: "4G", EndpointAddress: "10.0.0.7:25565", PlayersOnline: 2, PlayersMax: 20} + + status := func(uid string) map[string]any { + a := newTestAPI(repo, cl) + a.External = staticExternal{p: &Principal{UserID: uid, Email: uid + "@example.net", Role: "user"}} + w := do(a.ExternalHandler(), "GET", "/api/v1/servers/survival/status", "", nil) + if w.Code != http.StatusOK { + t.Fatalf("%s: status = %d %s", uid, w.Code, w.Body) + } + var m map[string]any + _ = json.Unmarshal(w.Body.Bytes(), &m) + return m + } + + if m := status("owner1"); m["image"] != "paper-1.21" || m["endpointAddress"] != "10.0.0.7:25565" { + t.Fatalf("owner sees %v, want the whole record", m) + } + m := status("stranger") + for _, hidden := range []string{"image", "javaMemory", "endpointAddress"} { + if _, ok := m[hidden]; ok { + t.Errorf("a stranger sees %s", hidden) + } + } + if m["phase"] != "Running" || m["playersOnline"] != float64(2) { + t.Errorf("a stranger sees %v, want phase and player count", m) + } +} diff --git a/internal/api/util.go b/internal/api/util.go index dbf4337..b0f92a0 100644 --- a/internal/api/util.go +++ b/internal/api/util.go @@ -2,6 +2,7 @@ package api import ( "encoding/json" + "errors" "mime" "net/http" "strings" @@ -31,11 +32,17 @@ func requireJSONContentType(r *http.Request) error { } // decodeJSON strictly decodes a small request body into v, rejecting unknown -// fields and trailing data so malformed callers fail fast with 400. +// fields and trailing data so malformed callers fail fast with 400, and a body +// past maxBodyBytes with 413. func decodeJSON(w http.ResponseWriter, r *http.Request, v any) error { dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)) dec.DisallowUnknownFields() if err := dec.Decode(v); err != nil { + var tooBig *http.MaxBytesError + if errors.As(err, &tooBig) { + return newError(http.StatusRequestEntityTooLarge, "too_large", + "request body is larger than %d bytes", tooBig.Limit) + } return newError(http.StatusBadRequest, "bad_request", "invalid request body: %v", err) } if dec.More() { diff --git a/internal/offsite/status.go b/internal/offsite/status.go index 83eca7f..81568d6 100644 --- a/internal/offsite/status.go +++ b/internal/offsite/status.go @@ -21,7 +21,7 @@ const StaleAfter = 12 * time.Hour type Status struct { LastAttempt time.Time `json:"last_attempt"` // LastSuccess is the last run in which every step succeeded. - LastSuccess time.Time `json:"last_success,omitempty"` + LastSuccess time.Time `json:"last_success,omitzero"` LastError string `json:"last_error,omitempty"` Endpoint string `json:"endpoint"` Bucket string `json:"bucket"` diff --git a/internal/panel/panel.go b/internal/panel/panel.go index 74f6a2f..2c29ad4 100644 --- a/internal/panel/panel.go +++ b/internal/panel/panel.go @@ -3,7 +3,9 @@ package panel import ( "bytes" + "crypto/sha256" "embed" + "encoding/base64" "encoding/json" "errors" "io" @@ -119,6 +121,7 @@ func Handler(api http.Handler, rootDomain, panelHost, adminHost, version string) build: parseBuildVersion(version), files: files, fileServer: http.FileServer(http.FS(files)), + csp: contentSecurityPolicy(files), } } @@ -130,9 +133,66 @@ type handler struct { build buildInfo files fs.FS fileServer http.Handler + csp string +} + +// inlineScript matches a `) + +// contentSecurityPolicy is the policy the console's pages are served under. The +// bundle is same-origin, so scripts run only from 'self' plus, by hash, the inline +// scripts index.html carries as built (hashed here, at start, so a rebuilt page +// cannot drift from its policy). Styles allow inline: the UI library sets style +// attributes and injects style tags. Images take data: and blob: for rendered +// QR codes and downloads; every request the page makes goes to its own origin. +func contentSecurityPolicy(files fs.FS) string { + scripts := "'self'" + if index, err := fs.ReadFile(files, "index.html"); err == nil { + for _, m := range inlineScript.FindAllSubmatch(index, -1) { + sum := sha256.Sum256(m[1]) + scripts += " 'sha256-" + base64.StdEncoding.EncodeToString(sum[:]) + "'" + } + } + return strings.Join([]string{ + "default-src 'self'", + "script-src " + scripts, + "style-src 'self' 'unsafe-inline'", + "img-src 'self' data: blob:", + "font-src 'self' data:", + "connect-src 'self'", + "object-src 'none'", + "base-uri 'none'", + "form-action 'self'", + "frame-ancestors 'none'", + }, "; ") +} + +// setPageHeaders marks every response the console serves: no framing (the +// passkey and approval buttons must not sit under someone else's page), no +// sniffing, no referrer carrying a setup token off the site, and HSTS when the +// request came through a TLS edge. The API sets its own, stricter set on /api/. +func (h *handler) setPageHeaders(w http.ResponseWriter, r *http.Request) { + hdr := w.Header() + hdr.Set("Content-Security-Policy", h.csp) + hdr.Set("X-Content-Type-Options", "nosniff") + hdr.Set("X-Frame-Options", "DENY") + hdr.Set("Referrer-Policy", "no-referrer") + hdr.Set("Permissions-Policy", "camera=(), microphone=(), geolocation=(), payment=()") + // The tunnel reaches this listener over TLS too, so what the browser saw is + // the edge's X-Forwarded-Proto. A browser straight on the self-signed listener + // sends none, and HSTS over a certificate error is ignored (RFC 6797 §8.1). + if strings.EqualFold(r.Header.Get("X-Forwarded-Proto"), "https") { + hdr.Set("Strict-Transport-Security", "max-age=31536000") + } } func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/healthz" || r.URL.Path == "/readyz" || strings.HasPrefix(r.URL.Path, "/api/") { + h.api.ServeHTTP(w, r) + return + } + h.setPageHeaders(w, r) // WeChat/QQ in-app browsers cannot run WebAuthn, so steer their document // navigations to a "open in your system browser" interstitial before the SPA // (which is built around passkey enrollment) ever loads. See webview.go. @@ -140,8 +200,6 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } switch { - case r.URL.Path == "/healthz" || r.URL.Path == "/readyz" || strings.HasPrefix(r.URL.Path, "/api/"): - h.api.ServeHTTP(w, r) case r.URL.Path == "/config.json": w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "no-store") @@ -153,8 +211,16 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { Build: h.build, }) case h.hasStaticFile(r.URL.Path): + // Vite names every file under assets/ by its content hash, so a cached + // copy can never go stale; everything else revalidates. + if strings.HasPrefix(r.URL.Path, "/assets/") { + w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") + } else { + w.Header().Set("Cache-Control", "no-cache") + } h.fileServer.ServeHTTP(w, r) default: + w.Header().Set("Cache-Control", "no-cache") h.serveIndex(w, r) } } diff --git a/internal/panel/panel_test.go b/internal/panel/panel_test.go index a27e61f..fbe91b1 100644 --- a/internal/panel/panel_test.go +++ b/internal/panel/panel_test.go @@ -1,11 +1,14 @@ package panel import ( + "crypto/sha256" + "encoding/base64" "encoding/json" "net/http" "net/http/httptest" "strings" "testing" + "testing/fstest" ) func TestHandlerServesPanelAndConfig(t *testing.T) { @@ -86,3 +89,59 @@ func TestParseBuildVersionSplitsBothStampForms(t *testing.T) { }) } } + +// The console's pages go out under a CSP that admits the inline theme script by +// its hash (and nothing else inline), cannot be framed, and cache by name: +// hashed assets forever, the page itself never without revalidation. +func TestHandlerSetsPageSecurityAndCacheHeaders(t *testing.T) { + h := Handler(http.NotFoundHandler(), "example.test", "", "", "v1.2.3") + + w := httptest.NewRecorder() + // Through the tunnel: TLS to the origin as well, the edge's scheme in XFP. + r := httptest.NewRequest(http.MethodGet, "https://console.example.test/servers/survival", nil) + r.Header.Set("X-Forwarded-Proto", "https") + h.ServeHTTP(w, r) + csp := w.Header().Get("Content-Security-Policy") + for _, want := range []string{"script-src 'self'", "frame-ancestors 'none'", "object-src 'none'", "base-uri 'none'"} { + if !strings.Contains(csp, want) { + t.Errorf("CSP %q lacks %q", csp, want) + } + } + if strings.Contains(csp, "'unsafe-inline'") && !strings.Contains(csp, "style-src 'self' 'unsafe-inline'") { + t.Errorf("CSP %q allows inline outside styles", csp) + } + for k, want := range map[string]string{ + "X-Frame-Options": "DENY", + "X-Content-Type-Options": "nosniff", + "Referrer-Policy": "no-referrer", + "Strict-Transport-Security": "max-age=31536000", + "Cache-Control": "no-cache", + } { + if got := w.Header().Get(k); got != want { + t.Errorf("%s = %q, want %q", k, got, want) + } + } + + // Straight to the self-signed listener: no HSTS. + w = httptest.NewRecorder() + r = httptest.NewRequest(http.MethodGet, "https://10.0.0.7:30443/", nil) + h.ServeHTTP(w, r) + if got := w.Header().Get("Strict-Transport-Security"); got != "" { + t.Errorf("HSTS on the self-signed listener: %q", got) + } +} + +func TestContentSecurityPolicyHashesInlineScripts(t *testing.T) { + script := "\n document.documentElement.classList.add(\"dark\");\n " + files := fstest.MapFS{"index.html": {Data: []byte( + "" + + ``)}} + sum := sha256.Sum256([]byte(script)) + want := "script-src 'self' 'sha256-" + base64.StdEncoding.EncodeToString(sum[:]) + "';" + if csp := contentSecurityPolicy(files); !strings.Contains(csp, want) { + t.Fatalf("CSP %q, want %q", csp, want) + } + if csp := contentSecurityPolicy(fstest.MapFS{}); !strings.Contains(csp, "script-src 'self';") { + t.Fatalf("CSP without an index = %q", csp) + } +} diff --git a/internal/watchdog/watchdog.go b/internal/watchdog/watchdog.go index 62227d7..e65654d 100644 --- a/internal/watchdog/watchdog.go +++ b/internal/watchdog/watchdog.go @@ -67,12 +67,12 @@ type Alert struct { FirstSeen time.Time `json:"first_seen"` // Notified is when the alert was last mailed, at NotifiedSeverity; zero // while it is pending. - Notified time.Time `json:"notified,omitempty"` + Notified time.Time `json:"notified,omitzero"` NotifiedSeverity Severity `json:"notified_severity,omitempty"` // ClearedAt is when a mailed alert was first seen gone. It is mailed as // resolved only after staying gone for resolveAfter, so a value hovering at // its threshold does not mail on every crossing. - ClearedAt time.Time `json:"cleared_at,omitempty"` + ClearedAt time.Time `json:"cleared_at,omitzero"` } // State is what the watchdog keeps between runs. diff --git a/panel/src/lib/logstream.test.ts b/panel/src/lib/logstream.test.ts index 96ea5e9..c01638c 100644 --- a/panel/src/lib/logstream.test.ts +++ b/panel/src/lib/logstream.test.ts @@ -18,6 +18,7 @@ class FakeEventSource implements EventSourceLike { readyState = 0; // CONNECTING closed = false; readonly url: string; + private readonly listeners = new Map void>>(); constructor(url: string) { this.url = url; @@ -28,6 +29,13 @@ class FakeEventSource implements EventSourceLike { this.readyState = 2; // CLOSED } + addEventListener(type: string, listener: (ev: unknown) => void): void { + this.listeners.set(type, [...(this.listeners.get(type) ?? []), listener]); + } + emitEvent(type: string): void { + for (const fn of this.listeners.get(type) ?? []) fn({ data: "" }); + } + // Test drivers mirroring what the browser would invoke. emitOpen(): void { this.readyState = 1; // OPEN @@ -181,4 +189,18 @@ describe("LogStreamController", () => { expect(ctrl.getSnapshot().lines).toHaveLength(0); expect(ctrl.getSnapshot().status).toBe("open"); }); + + it("ends for good when the server revokes the stream", () => { + const { factory, created } = makeFactory(); + const ctrl = new LogStreamController({ url: "/api/v1/servers/s/console", factory }); + ctrl.open(); + const es = created[0]; + es.emitOpen(); + es.emitMessage("[12:00:00] [Server thread/INFO]: hello"); + es.emitEvent("revoked"); + expect(ctrl.getSnapshot().status).toBe("ended"); + expect(es.closed).toBe(true); + expect(ctrl.getSnapshot().lines).toHaveLength(1); + expect(created).toHaveLength(1); + }); }); diff --git a/panel/src/lib/logstream.ts b/panel/src/lib/logstream.ts index b8df2a0..d5a63f7 100644 --- a/panel/src/lib/logstream.ts +++ b/panel/src/lib/logstream.ts @@ -59,6 +59,7 @@ export interface EventSourceLike { onerror: ((ev: any) => void) | null; readyState: number; close(): void; + addEventListener(type: string, listener: (ev: any) => void): void; } /** Builds an EventSource for a URL (production: the browser ctor; tests: a fake). */ @@ -175,6 +176,14 @@ export class LogStreamController { this.es = es; es.onopen = () => this.setStatus("open"); es.onmessage = (ev) => this.pushLine(ev.data); + // The server re-checks a running stream's grant (session still live, caller + // still owns the server) and sends "revoked" before it closes. That close is + // final: an auto-retry would only be refused again. + es.addEventListener("revoked", () => { + if (this.es !== es) return; + this.disconnect(); + this.setStatus("ended"); + }); es.onerror = () => { // Native EventSource auto-reconnects on a transient drop (readyState // returns to CONNECTING); only a fatal response — non-200 / wrong