feat(build): 扫描门按可配严重度拦截并可接受已知风险，留存 Trivy 报告与 CycloneDX SBOM，面板构建页展示扫描结果 (24373823) · Commits · FelisMC / Felis · GitLab

cmd/felis/api.go

+4 −0

Changes for cmd/felis/api.go: 4 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -175,6 +175,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
		Store: build.NewPGStore(drv.DB()),
		Jobs: buildJobs,
		Config: buildCfg,
		Outcomes: build.NewK8sOutcomes(clientset, buildCfg),
		}
		go probeBuildUserNamespaces(ctx, buildJobs, buildCfg, stderr)

		@@ -533,6 +534,9 @@ func buildConfig(cfg *config.Config) build.Config {
		MaxConcurrent: cfg.Registry.MaxConcurrentBuilds,
		TrivyDBRepository: cfg.Registry.TrivyDBRepository,
		TrivyJavaDBRepository: cfg.Registry.TrivyJavaDBRepository,
		ScanFailOn: cfg.Registry.ScanFailOn,
		ScanFailUnfixed: cfg.Registry.ScanFailUnfixed,
		ScanAccept: cfg.Registry.ScanAccept,
		// The submit lane's derived context URLs live here; the fetch step's
		// service token goes nowhere else.
		ContextOrigin: internalAPIBaseURL(),

cmd/felis/run.go

+2 −0

Changes for cmd/felis/run.go: 2 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -23,6 +23,7 @@ Commands:
		files List/read/write one file in a stopped server's world (internal Job entrypoint)
		egress-gate Hold a build pod until its egress NetworkPolicy is enforced (internal Job entrypoint)
		fetch-context Fetch and extract a submission's build context (internal Job entrypoint)
		scan-gate Apply the scan policy to a build's Trivy report and hand felis-api the report and SBOM (internal Job entrypoint)
		push-image Push a scanned image tarball to the registry (internal Job entrypoint)
		mirror-build-tools Copy kaniko, trivy and Trivy's DBs into the registry (run by felis-build-tools.timer)
		registry-gate Authorize registry writes in front of registry:2 (internal sidecar entrypoint)
		@@ -61,6 +62,7 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
		"files": cmdFiles,
		"egress-gate": cmdEgressGate,
		"fetch-context": cmdFetchContext,
		"scan-gate": cmdScanGate,
		"push-image": cmdPushImage,
		"mirror-build-tools": cmdMirrorBuildTools,
		"registry-gate": cmdRegistryGate,

cmd/felis/scangate.go

0 → 100644

+166 −0

Changes for cmd/felis/scangate.go: 166 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"errors"
		"flag"
		"fmt"
		"io"
		"io/fs"
		"os"
		"strings"

		"felis.lolicon.best/internal/build"
		)

		// maxScanDocument bounds each document scan-gate reads: a modpack report lists a
		// few thousand packages, far below this. Tests shrink it.
		var maxScanDocument int64 = 64 << 20

		// cmdScanGate is the build pod's verdict step, after trivy wrote its full JSON
		// report and trivy convert wrote the CycloneDX SBOM. It applies the scan policy
		// to the report, prints the verdict and every blocking finding, then appends the
		// verdict, the report and the SBOM to its log as the envelope felis-api keeps on
		// the build (build.WriteScanEnvelope). It exits 1 when a finding blocks, which
		// fails the pod before the push step runs, and 2 when the report cannot be read,
		// so a scan that produced nothing usable never admits an image.
		func cmdScanGate(args []string, stdout, stderr io.Writer) int {
		fset := flag.NewFlagSet("scan-gate", flag.ContinueOnError)
		fset.SetOutput(stderr)
		reportPath := fset.String("report", "", "trivy JSON report (required)")
		sbomPath := fset.String("sbom", "", "CycloneDX SBOM to keep with the report")
		failOn := fset.String("fail-on", strings.Join(build.DefaultScanFailOn, ","), "comma-separated severities that block the image")
		failUnfixed := fset.Bool("fail-unfixed", false, "block on vulnerabilities that have no fixed release too")
		accept := fset.String("accept", "", "comma-separated vulnerability ids and secret rule ids that never block")
		termLog := fset.String("termination-log", "/dev/termination-log", "where the one-line verdict goes for the pod status")
		if err := fset.Parse(args); err != nil {
		return 2
		}
		// A stray argument is a policy the gate would otherwise drop without a word
		// (a comma split out of --fail-on, say).
		if fset.NArg() > 0 {
		fmt.Fprintf(stderr, "felis scan-gate: unexpected argument %q\n", fset.Arg(0))
		return 2
		}
		sevs, err := build.ParseSeverities(*failOn)
		if err != nil {
		fmt.Fprintf(stderr, "felis scan-gate: --fail-on: %v\n", err)
		return 2
		}
		accepted, err := build.ParseScanAccept(*accept)
		if err != nil {
		fmt.Fprintf(stderr, "felis scan-gate: --accept: %v\n", err)
		return 2
		}
		if *reportPath == "" {
		fmt.Fprintln(stderr, "felis scan-gate: --report is required")
		return 2
		}
		fail := func(msg string) int {
		fmt.Fprintln(stderr, "felis scan-gate: "+msg)
		writeTerminationLog(*termLog, msg)
		return 2
		}
		report, err := readScanDocument(*reportPath)
		if err != nil {
		return fail("the scan report is unreadable: " + err.Error())
		}
		policy := build.ScanPolicy{FailOn: sevs, FailUnfixed: *failUnfixed, Accept: accepted}
		summary, err := build.Summarize(report, policy)
		if err != nil {
		return fail("the scan report is unreadable: " + err.Error())
		}
		env := build.ScanEnvelope{Summary: summary, Report: report}
		if *sbomPath != "" {
		switch sbom, err := readScanDocument(*sbomPath); {
		case err == nil:
		env.SBOM = sbom
		case errors.Is(err, fs.ErrNotExist):
		fmt.Fprintf(stdout, "felis scan-gate: no SBOM at %s; keeping the report alone\n", *sbomPath)
		default:
		return fail("the SBOM is unreadable: " + err.Error())
		}
		}

		printScanVerdict(stdout, summary)
		written, err := build.WriteScanEnvelope(stdout, env)
		if err != nil {
		return fail("could not write the scan envelope: " + err.Error())
		}
		for _, doc := range written.Summary.Omitted {
		fmt.Fprintf(stderr, "felis scan-gate: the %s is too large to keep with the build and was left out\n", doc)
		}
		if summary.Blocked {
		writeTerminationLog(*termLog, summary.Reason())
		return 1
		}
		writeTerminationLog(*termLog, "the scan passed")
		return 0
		}

		// printScanVerdict writes the human half of scan-gate's log.
		func printScanVerdict(w io.Writer, s build.ScanSummary) {
		var counts []string
		for _, sev := range build.Severities {
		counts = append(counts, fmt.Sprintf("%s %d", sev, s.Counts[sev]))
		}
		fmt.Fprintf(w, "felis scan-gate: %d packages; findings: %s\n", s.Packages, strings.Join(counts, ", "))
		unfixed := "vulnerabilities with no fixed release do not block"
		if s.Policy.FailUnfixed {
		unfixed = "vulnerabilities with no fixed release block too"
		}
		fmt.Fprintf(w, "felis scan-gate: blocking on %s (%s)\n", strings.Join(s.Policy.FailOn, ", "), unfixed)
		if len(s.Policy.Accept) > 0 {
		matched := 0
		for _, f := range s.Findings {
		if f.Accepted {
		matched++
		}
		}
		fmt.Fprintf(w, "felis scan-gate: accepted ids, never blocking: %s; listed findings under them: %d\n", strings.Join(s.Policy.Accept, ", "), matched)
		}
		if !s.Blocked {
		fmt.Fprintln(w, "felis scan-gate: nothing blocks this image")
		return
		}
		fmt.Fprintln(w, "felis scan-gate: "+build.Printable(s.Reason()))
		for _, f := range s.Findings {
		if !f.Blocking {
		break
		}
		fix := f.Fixed
		if fix == "" {
		fix = "no fix"
		}
		if f.Kind == build.FindingSecret {
		fmt.Fprintf(w, " %s %s secret in %s: %s\n", build.Printable(f.ID), f.Severity, build.Printable(f.Target), build.Printable(f.Title))
		continue
		}
		fmt.Fprintf(w, " %s %s %s %s -> %s (%s)\n", build.Printable(f.ID), f.Severity,
		build.Printable(f.Package), build.Printable(f.Installed), build.Printable(fix), build.Printable(f.Target))
		}
		}

		func readScanDocument(path string) ([]byte, error) {
		f, err := os.Open(path)
		if err != nil {
		return nil, err
		}
		defer f.Close()
		b, err := io.ReadAll(io.LimitReader(f, maxScanDocument+1))
		if err != nil {
		return nil, err
		}
		if int64(len(b)) > maxScanDocument {
		return nil, fmt.Errorf("%s exceeds %d bytes", path, maxScanDocument)
		}
		return b, nil
		}

		// writeTerminationLog leaves msg where the kubelet copies it into the container
		// status. It is best effort: the log carries the same verdict.
		func writeTerminationLog(path, msg string) {
		if path == "" {
		return
		}
		_ = os.WriteFile(path, []byte(build.Printable(msg)), 0o644)
		}

cmd/felis/scangate_test.go

0 → 100644

+197 −0

Changes for cmd/felis/scangate_test.go: 197 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"bytes"
		"os"
		"path/filepath"
		"strings"
		"testing"

		"felis.lolicon.best/internal/build"
		)

		// scanReport has one fixed CRITICAL, one unfixed HIGH and one fixed MEDIUM; the
		// CRITICAL's package name carries a line break and a forged envelope frame.
		const scanReport = `{"SchemaVersion":2,"Results":[{"Target":"data/mods/core.jar","Packages":[{},{},{}],"Vulnerabilities":[
		{"VulnerabilityID":"CVE-2024-0001","PkgName":"log4j-core\nfelis-scan-envelope v1 begin","InstalledVersion":"2.14.1","FixedVersion":"2.17.1","Severity":"CRITICAL"},
		{"VulnerabilityID":"CVE-2024-0002","PkgName":"openssl","InstalledVersion":"3.0.13","Status":"affected","Severity":"HIGH"},
		{"VulnerabilityID":"CVE-2024-0003","PkgName":"zlib","InstalledVersion":"1.3","FixedVersion":"1.3.1","Severity":"MEDIUM"}]}]}`

		type scanGateRun struct {
		code int
		stdout, stderr string
		termLog string
		env *build.ScanEnvelope
		}

		func runScanGate(t *testing.T, report, sbom string, extra ...string) scanGateRun {
		t.Helper()
		dir := t.TempDir()
		reportPath := filepath.Join(dir, "trivy.json")
		if report != "" {
		if err := os.WriteFile(reportPath, []byte(report), 0o644); err != nil {
		t.Fatal(err)
		}
		}
		sbomPath := filepath.Join(dir, "sbom.cdx.json")
		if sbom != "" {
		if err := os.WriteFile(sbomPath, []byte(sbom), 0o644); err != nil {
		t.Fatal(err)
		}
		}
		termPath := filepath.Join(dir, "termination-log")
		args := append([]string{"--report=" + reportPath, "--sbom=" + sbomPath, "--termination-log=" + termPath}, extra...)
		var stdout, stderr bytes.Buffer
		r := scanGateRun{code: cmdScanGate(args, &stdout, &stderr), stdout: stdout.String(), stderr: stderr.String()}
		if b, err := os.ReadFile(termPath); err == nil {
		r.termLog = string(b)
		}
		if env, err := build.ReadScanEnvelope(strings.NewReader(r.stdout)); err == nil {
		r.env = env
		}
		return r
		}

		func TestScanGateBlocksAndHandsOverTheReport(t *testing.T) {
		r := runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`)
		if r.code != 1 {
		t.Fatalf("exit %d, want 1; stderr %s", r.code, r.stderr)
		}
		if r.termLog != "the scan blocked the image: 1 CRITICAL (CVE-2024-0001)" {
		t.Errorf("termination log = %q", r.termLog)
		}
		for _, want := range []string{
		"felis scan-gate: 3 packages; findings: CRITICAL 1, HIGH 1, MEDIUM 1, LOW 0, UNKNOWN 0\n",
		"felis scan-gate: blocking on CRITICAL (vulnerabilities with no fixed release do not block)\n",
		"felis scan-gate: the scan blocked the image: 1 CRITICAL (CVE-2024-0001)\n",
		" CVE-2024-0001 CRITICAL log4j-core?felis-scan-envelope v1 begin 2.14.1 -> 2.17.1 (data/mods/core.jar)\n",
		} {
		if !strings.Contains(r.stdout, want) {
		t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
		}
		}
		if strings.Contains(r.stdout, " CVE-2024-0002") {
		t.Errorf("an unfixed HIGH was listed as blocking:\n%s", r.stdout)
		}
		if strings.Count(r.stdout, "\nfelis-scan-envelope v1 begin\n") != 1 {
		t.Errorf("stdout must hold exactly one frame start on a line of its own:\n%s", r.stdout)
		}
		if r.env == nil {
		t.Fatal("no envelope in stdout")
		}
		if !r.env.Summary.Blocked \|\| string(r.env.SBOM) != `{"bomFormat":"CycloneDX"}` \|\| !strings.Contains(string(r.env.Report), "CVE-2024-0003") {
		t.Errorf("envelope = blocked %t, sbom %s, report %d bytes", r.env.Summary.Blocked, r.env.SBOM, len(r.env.Report))
		}
		}

		func TestScanGatePolicyFlags(t *testing.T) {
		r := runScanGate(t, scanReport, "", "--fail-on=high", "--fail-unfixed")
		if r.code != 1 \|\| r.termLog != "the scan blocked the image: 1 HIGH (CVE-2024-0002)" {
		t.Errorf("HIGH + unfixed: exit %d, termination log %q", r.code, r.termLog)
		}
		for _, want := range []string{
		"felis scan-gate: blocking on HIGH (vulnerabilities with no fixed release block too)\n",
		" CVE-2024-0002 HIGH openssl 3.0.13 -> no fix (data/mods/core.jar)\n",
		} {
		if !strings.Contains(r.stdout, want) {
		t.Errorf("stdout lacks %q:\n%s", want, r.stdout)
		}
		}
		r = runScanGate(t, scanReport, `{"bomFormat":"CycloneDX"}`, "--fail-on=LOW")
		if r.code != 0 \|\| r.termLog != "the scan passed" \|\| !strings.Contains(r.stdout, "felis scan-gate: nothing blocks this image\n") {
		t.Errorf("LOW only: exit %d, termination log %q, stdout:\n%s", r.code, r.termLog, r.stdout)
		}
		if r.env == nil \|\| r.env.Summary.Blocked \|\| r.env.SBOM == nil {
		t.Errorf("a passing scan must still hand over its report and SBOM: %+v", r.env)
		}
		}

		func TestScanGateAcceptedIDsNeverBlock(t *testing.T) {
		r := runScanGate(t, scanReport, "", "--fail-on=CRITICAL,MEDIUM", "--accept=CVE-2024-0001, CVE-2024-0002,CVE-2099-0001")
		if r.code != 1 \|\| r.termLog != "the scan blocked the image: 1 MEDIUM (CVE-2024-0003)" {
		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
		}
		if !strings.Contains(r.stdout, "felis scan-gate: accepted ids, never blocking: CVE-2024-0001, CVE-2024-0002, CVE-2099-0001; listed findings under them: 2\n") {
		t.Errorf("stdout:\n%s", r.stdout)
		}
		if r.env == nil \|\| strings.Join(r.env.Summary.Policy.Accept, ",") != "CVE-2024-0001,CVE-2024-0002,CVE-2099-0001" {
		t.Fatalf("envelope = %+v", r.env)
		}
		for _, f := range r.env.Summary.Findings {
		if f.ID == "CVE-2024-0001" && (f.Blocking \|\| !f.Accepted) {
		t.Errorf("accepted finding = %+v", f)
		}
		}
		r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001")
		if r.code != 0 \|\| r.termLog != "the scan passed" {
		t.Errorf("only an accepted CRITICAL: exit %d, termination log %q", r.code, r.termLog)
		}
		}

		func TestScanGateWithoutAnSBOMKeepsTheReport(t *testing.T) {
		r := runScanGate(t, scanReport, "", "--fail-on=LOW")
		if r.code != 0 \|\| !strings.Contains(r.stdout, "felis scan-gate: no SBOM at ") {
		t.Errorf("exit %d, stdout:\n%s", r.code, r.stdout)
		}
		if r.env == nil \|\| r.env.SBOM != nil \|\| r.env.Report == nil {
		t.Errorf("envelope = %+v", r.env)
		}
		}

		func TestScanGateListsABlockingSecret(t *testing.T) {
		r := runScanGate(t, `{"SchemaVersion":2,"Results":[{"Target":"config/keys.txt","Secrets":[
		{"RuleID":"aws-access-key-id","Severity":"CRITICAL","Title":"AWS Access\nKey ID"}]}]}`, "")
		if r.code != 1 \|\| r.termLog != "the scan blocked the image: 1 CRITICAL (aws-access-key-id)" {
		t.Errorf("exit %d, termination log %q", r.code, r.termLog)
		}
		if !strings.Contains(r.stdout, " aws-access-key-id CRITICAL secret in config/keys.txt: AWS Access?Key ID\n") {
		t.Errorf("stdout:\n%s", r.stdout)
		}
		}

		func TestScanGateFailsClosed(t *testing.T) {
		r := runScanGate(t, "", "")
		if r.code != 2 \|\| !strings.HasPrefix(r.termLog, "the scan report is unreadable: open ") \|\| r.env != nil {
		t.Errorf("missing report: exit %d, termination log %q", r.code, r.termLog)
		}
		r = runScanGate(t, `{"SchemaVersion":1}`, "")
		if r.code != 2 \|\| r.termLog != "the scan report is unreadable: read trivy report: schema version 1, want 2" {
		t.Errorf("old schema: exit %d, termination log %q", r.code, r.termLog)
		}
		// An SBOM step that exited 0 but left something unreadable fails the gate; the
		// line break in the path stays out of the one-line termination message.
		sbomDir := filepath.Join(t.TempDir(), "sb\nom")
		if err := os.Mkdir(sbomDir, 0o755); err != nil {
		t.Fatal(err)
		}
		r = runScanGate(t, scanReport, "", "--sbom="+sbomDir)
		if r.code != 2 \|\| !strings.HasPrefix(r.termLog, "the SBOM is unreadable: read ") \|\|
		!strings.HasSuffix(r.termLog, "/sb?om: is a directory") \|\| r.env != nil {
		t.Errorf("unreadable SBOM: exit %d, termination log %q", r.code, r.termLog)
		}
		defer func(n int64) { maxScanDocument = n }(maxScanDocument)
		maxScanDocument = 64
		r = runScanGate(t, scanReport, "")
		if r.code != 2 \|\| !strings.HasSuffix(r.termLog, "/trivy.json exceeds 64 bytes") \|\| r.env != nil {
		t.Errorf("oversized report: exit %d, termination log %q", r.code, r.termLog)
		}
		maxScanDocument = 64 << 20
		r = runScanGate(t, scanReport, "", "--fail-on=SEVERE")
		if r.code != 2 \|\| !strings.Contains(r.stderr, `felis scan-gate: --fail-on: unknown severity "SEVERE"`) {
		t.Errorf("bad severity: exit %d, stderr %q", r.code, r.stderr)
		}
		// A severity list split at its comma leaves a stray argument, not a
		// narrower policy.
		r = runScanGate(t, scanReport, "", "--fail-on=LOW", "CRITICAL")
		if r.code != 2 \|\| !strings.Contains(r.stderr, `felis scan-gate: unexpected argument "CRITICAL"`) \|\| r.env != nil {
		t.Errorf("stray argument: exit %d, stderr %q", r.code, r.stderr)
		}
		r = runScanGate(t, scanReport, "", "--accept=CVE-2024-0001 CVE-2024-0003")
		if r.code != 2 \|\| !strings.Contains(r.stderr, `felis scan-gate: --accept: "CVE-2024-0001 CVE-2024-0003" is not a vulnerability id or secret rule id`) {
		t.Errorf("bad accept list: exit %d, stderr %q", r.code, r.stderr)
		}
		var stdout, stderr bytes.Buffer
		if code := cmdScanGate(nil, &stdout, &stderr); code != 2 \|\| !strings.Contains(stderr.String(), "--report is required") {
		t.Errorf("no report flag: exit %d, stderr %q", code, stderr.String())
		}
		}

deploy/bootstrap.sh

+1 −1

Changes for deploy/bootstrap.sh: 1 added line, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -3002,7 +3002,7 @@ persisted_registry_block() {
		out="$(awk '
		/^[[:space:]]*\[/ { sect = $0; next }
		sect ~ /^[[:space:]]\[registry\][[:space:]]$/ &&
		/^[[:space:]](kaniko_image\|trivy_image\|trivy_db_repository\|trivy_java_db_repository\|build_cpu_limit\|build_mem_limit\|build_disk_limit\|build_user_namespaces\|build_runtime_class\|max_concurrent_builds\|user_uploads_context\|user_uploads_max_bytes\|context_max_bytes)[[:space:]]=/ { print }
		/^[[:space:]](kaniko_image\|trivy_image\|trivy_db_repository\|trivy_java_db_repository\|build_cpu_limit\|build_mem_limit\|build_disk_limit\|build_user_namespaces\|build_runtime_class\|max_concurrent_builds\|scan_fail_on\|scan_fail_unfixed\|scan_accept\|user_uploads_context\|user_uploads_max_bytes\|context_max_bytes)[[:space:]]=/ { print }
		sect ~ /^[[:space:]]\[registry\.s3\][[:space:]]$/ && /^[[:space:]][A-Za-z_]+[[:space:]]=/ {
		if (!s3hdr) { printf "[registry.s3]\n"; s3hdr = 1 }
		print