feat(metrics): record felis_image_build_failures_total on failed builds
Wire the build subsystem to the felis_image_build_failures_total counter (spec §23). It advances at the two terminal-failure producers: finishAt (the Sync JobFailed/JobUnknown verdict — a kaniko failure or a CRITICAL CVE from trivy's --exit-code 1) and Submit's job-creation bypass path, which records its failure directly without going through finishAt. Cancellations and successful builds are deliberately not counted. A delta-asserting test exercises both Inc sites plus a successful-build negative control that proves the StatusFailed guard discriminates rather than firing on every terminal write, all over the existing in-memory Store/Jobs fakes.
This commit is contained in:
2 files changed
+71
No files matched your search
@@ -36,6 +36,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"felis.lolicon.best/internal/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Status mirrors the build_status enum (spec §6).
|
// Status mirrors the build_status enum (spec §6).
|
||||||
@@ -307,6 +309,12 @@ func (b *Builder) Submit(ctx context.Context, req Request) (*Build, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
// The pending row exists; mark it failed so it is not reconciled forever.
|
// The pending row exists; mark it failed so it is not reconciled forever.
|
||||||
_ = b.Store.FinishBuild(ctx, bld.ID, StatusFailed, "job creation failed: "+err.Error(), b.now())
|
_ = b.Store.FinishBuild(ctx, bld.ID, StatusFailed, "job creation failed: "+err.Error(), b.now())
|
||||||
|
// felis_image_build_failures_total (spec §23): this terminal-failure path
|
||||||
|
// records the build directly, not via finishAt, so it increments the counter
|
||||||
|
// itself. The FinishBuild error is deliberately ignored (the build is failed
|
||||||
|
// for the caller regardless), so the count tracks the failure event, not the
|
||||||
|
// store write.
|
||||||
|
metrics.ImageBuildFailuresTotal.Inc()
|
||||||
bld.Status = StatusFailed
|
bld.Status = StatusFailed
|
||||||
bld.Error = "job creation failed: " + err.Error()
|
bld.Error = "job creation failed: " + err.Error()
|
||||||
return bld, fmt.Errorf("build: create job: %w", err)
|
return bld, fmt.Errorf("build: create job: %w", err)
|
||||||
@@ -444,6 +452,16 @@ func (b *Builder) finishAt(ctx context.Context, bld *Build, status Status, msg s
|
|||||||
if err := b.Store.FinishBuild(ctx, bld.ID, status, msg, at); err != nil {
|
if err := b.Store.FinishBuild(ctx, bld.ID, status, msg, at); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if status == StatusFailed {
|
||||||
|
// felis_image_build_failures_total (spec §23) counts builds that reached a
|
||||||
|
// failed terminal state — a kaniko failure or a CRITICAL CVE surfaced by
|
||||||
|
// trivy's --exit-code 1, observed here as the Sync JobFailed/JobUnknown
|
||||||
|
// verdict. Cancellations (StatusCancelled) are deliberately not failures.
|
||||||
|
// Incremented only after the failed status is persisted, so the counter
|
||||||
|
// never runs ahead of the store. (Submit's job-creation path records its
|
||||||
|
// failure outside finishAt and increments there.)
|
||||||
|
metrics.ImageBuildFailuresTotal.Inc()
|
||||||
|
}
|
||||||
bld.Status = status
|
bld.Status = status
|
||||||
bld.Error = msg
|
bld.Error = msg
|
||||||
finished := at
|
finished := at
|
||||||
|
|||||||
@@ -5,6 +5,9 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"felis.lolicon.best/internal/metrics"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||||
)
|
)
|
||||||
|
|
||||||
var testNow = time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
var testNow = time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
@@ -356,6 +359,56 @@ func TestSyncAllAdvancesUnfinishedBuilds(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestImageBuildFailuresMetricCountsFailedBuilds asserts felis_image_build_failures_total
|
||||||
|
// (spec §23) advances exactly once per failed build from BOTH terminal-failure
|
||||||
|
// producers, and never on a successful build. There are two distinct Inc sites —
|
||||||
|
// finishAt (the Sync verdict) and Submit's job-creation bypass — so each is
|
||||||
|
// exercised separately; the success case is the negative control proving the
|
||||||
|
// StatusFailed guard discriminates rather than firing on every terminal write.
|
||||||
|
// Deltas are read around each action because the counter is a process-global
|
||||||
|
// singleton these package tests share (they run sequentially).
|
||||||
|
func TestImageBuildFailuresMetricCountsFailedBuilds(t *testing.T) {
|
||||||
|
read := func() float64 { return testutil.ToFloat64(metrics.ImageBuildFailuresTotal) }
|
||||||
|
|
||||||
|
t.Run("sync job-failed verdict increments via finishAt", func(t *testing.T) {
|
||||||
|
b, _, jb := newBuilder()
|
||||||
|
bld, _ := b.Submit(context.Background(), goodRequest())
|
||||||
|
before := read()
|
||||||
|
jb.phase = JobFailed
|
||||||
|
if _, err := b.Sync(context.Background(), bld.ID); err != nil {
|
||||||
|
t.Fatalf("Sync: %v", err)
|
||||||
|
}
|
||||||
|
if got := read() - before; got != 1 {
|
||||||
|
t.Errorf("failures delta = %v, want 1", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("submit job-create failure increments via bypass", func(t *testing.T) {
|
||||||
|
b, _, jb := newBuilder()
|
||||||
|
jb.createErr = errors.New("apiserver down")
|
||||||
|
before := read()
|
||||||
|
if _, err := b.Submit(context.Background(), goodRequest()); err == nil {
|
||||||
|
t.Fatal("expected Submit error when job creation fails")
|
||||||
|
}
|
||||||
|
if got := read() - before; got != 1 {
|
||||||
|
t.Errorf("failures delta = %v, want 1", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("successful build does not increment", func(t *testing.T) {
|
||||||
|
b, _, jb := newBuilder()
|
||||||
|
bld, _ := b.Submit(context.Background(), goodRequest())
|
||||||
|
before := read()
|
||||||
|
jb.phase = JobSucceeded
|
||||||
|
if _, err := b.Sync(context.Background(), bld.ID); err != nil {
|
||||||
|
t.Fatalf("Sync: %v", err)
|
||||||
|
}
|
||||||
|
if got := read() - before; got != 0 {
|
||||||
|
t.Errorf("failures delta on success = %v, want 0", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// ---- cancellation ----
|
// ---- cancellation ----
|
||||||
|
|
||||||
func TestCancelDeletesJobAndMarksCancelled(t *testing.T) {
|
func TestCancelDeletesJobAndMarksCancelled(t *testing.T) {
|
||||||
|
|||||||
Reference in new issue
Block a user