feat(breakglass): add halt-a-server op to the recovery console (§B4)
Add a root-gated "Halt a running server" operation to the break-glass console. The operator picks a server from the live fleet and the console flips that MinecraftServer CRD's spec.desiredState to Stopped via a spec-only merge patch, disjoint from the operator's status writes, so it cannot race or clobber reconciliation. It is the panel-independent emergency stop for when the box still has root plus a kubeconfig. System servers (login/lobby) are allowed but flagged: a system tag in the picker and an explicit WARNING in the post-exit summary, since halting login takes the shared auth front door down with no fallback. Audit is best-effort so a halt still works with the audit sink down. Already-stopped is a distinct no-op. The core (halt.go) is unit-tested against a real controller-runtime fake client that applies the patch.
This commit is contained in:
8 files changed
+649
-10
No files matched your search
+33
-8
@@ -146,13 +146,13 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := runBreakGlassTUI(ctx, repo, cfg.Database.URL, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, accountableOSUser(), adminExists)
|
res, err := runBreakGlassTUI(ctx, repo, cfg.Database.URL, cfg.Server.RootDomain, cfg.Auth.AdminHostname, cfg.Auth.PanelHostname, cfg.Auth.AccessJWTAud, cfg.K8s.Namespace, accountableOSUser(), adminExists)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "felis breakGlass: %v\n", err)
|
fmt.Fprintf(stderr, "felis breakGlass: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
if !res.provisioned && !res.edgeConfigured {
|
if !res.provisioned && !res.edgeConfigured && !res.halted {
|
||||||
fmt.Fprintln(stdout, "felis breakGlass: cancelled — no changes made.")
|
fmt.Fprintln(stdout, "felis breakGlass: cancelled — no changes made.")
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
@@ -198,6 +198,23 @@ func cmdBreakGlass(args []string, stdout, stderr io.Writer) int {
|
|||||||
fmt.Fprintln(stdout, "Then start the tunnel: cloudflared tunnel run")
|
fmt.Fprintln(stdout, "Then start the tunnel: cloudflared tunnel run")
|
||||||
fmt.Fprintln(stdout, "Verify the Access app actually guards the admin face before relying on it.")
|
fmt.Fprintln(stdout, "Verify the Access app actually guards the admin face before relying on it.")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if res.halted {
|
||||||
|
verb := "is now stopping"
|
||||||
|
if res.haltAlreadyStopped {
|
||||||
|
verb = "was already stopped"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stdout, "\nfelis breakGlass: server %q %s (namespace %s).\n", res.haltServer, verb, res.haltNamespace)
|
||||||
|
if res.haltSystemServer {
|
||||||
|
// login has no fallback (systemservers.go): stopping it takes the whole proxy
|
||||||
|
// front door down, so the summary says so explicitly rather than burying it.
|
||||||
|
fmt.Fprintf(stdout, "WARNING: %q is a system server — the shared front door is down until it is running again.\n", res.haltServer)
|
||||||
|
}
|
||||||
|
if res.haltAuditWarning != "" {
|
||||||
|
fmt.Fprintf(stdout, "WARNING: the accountability audit row was NOT written: %s\n", res.haltAuditWarning)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(stdout, "Restart it from the panel, or set the MinecraftServer's spec.desiredState back to Running.\n")
|
||||||
|
}
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -491,6 +508,14 @@ type breakGlassResult struct {
|
|||||||
storageMethod storageMethod
|
storageMethod storageMethod
|
||||||
storageDetail string
|
storageDetail string
|
||||||
|
|
||||||
|
// halt outcome (break-glass "halt a server" op #31)
|
||||||
|
halted bool
|
||||||
|
haltServer string
|
||||||
|
haltNamespace string
|
||||||
|
haltAlreadyStopped bool
|
||||||
|
haltSystemServer bool
|
||||||
|
haltAuditWarning string
|
||||||
|
|
||||||
// Cloudflare-specific edge detail (set only when connectMethod is Cloudflare)
|
// Cloudflare-specific edge detail (set only when connectMethod is Cloudflare)
|
||||||
edgeConfigured bool
|
edgeConfigured bool
|
||||||
edgeAud string
|
edgeAud string
|
||||||
@@ -518,16 +543,16 @@ const (
|
|||||||
cloudflareAPITokenDocsURL = "https://developers.cloudflare.com/fundamentals/api/how-to/account-owned-token-template/"
|
cloudflareAPITokenDocsURL = "https://developers.cloudflare.com/fundamentals/api/how-to/account-owned-token-template/"
|
||||||
)
|
)
|
||||||
|
|
||||||
func runBreakGlassTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, osUser string, adminExists bool) (breakGlassResult, error) {
|
func runBreakGlassTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
|
||||||
return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, osUser, adminExists, consoleModeBreakGlass)
|
return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeBreakGlass)
|
||||||
}
|
}
|
||||||
|
|
||||||
func runSetupTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, osUser string, adminExists bool) (breakGlassResult, error) {
|
func runSetupTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool) (breakGlassResult, error) {
|
||||||
return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, osUser, adminExists, consoleModeSetup)
|
return runConsoleTUI(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, consoleModeSetup)
|
||||||
}
|
}
|
||||||
|
|
||||||
func runConsoleTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, osUser string, adminExists bool, mode consoleMode) (breakGlassResult, error) {
|
func runConsoleTUI(ctx context.Context, s ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) (breakGlassResult, error) {
|
||||||
rm := newRootModel(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, osUser, adminExists, mode)
|
rm := newRootModel(ctx, s, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser, adminExists, mode)
|
||||||
final, err := tea.NewProgram(rm, tea.WithAltScreen()).Run()
|
final, err := tea.NewProgram(rm, tea.WithAltScreen()).Run()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return breakGlassResult{}, err
|
return breakGlassResult{}, err
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"felis.lolicon.best/internal/api"
|
||||||
|
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
||||||
|
"felis.lolicon.best/internal/naming"
|
||||||
|
|
||||||
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||||
|
"k8s.io/apimachinery/pkg/types"
|
||||||
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||||
|
)
|
||||||
|
|
||||||
|
// halt is the break-glass "stop a running server now" op (#31). Its authority is
|
||||||
|
// the same as the rest of the console: local root holding the cluster kubeconfig.
|
||||||
|
// The console does not stop the pod itself — it flips the MinecraftServer's desired
|
||||||
|
// state to Stopped and lets the operator reconcile that into a graceful shutdown, so
|
||||||
|
// a halt is exactly the CRD write the operator already knows how to honour.
|
||||||
|
//
|
||||||
|
// This file is the pure core — no bubbletea, no huh — so the whole thing is unit
|
||||||
|
// tested against a controller-runtime fake client. The TUI shell lives in
|
||||||
|
// tui_halt.go and only calls into here.
|
||||||
|
|
||||||
|
// haltableServer is a MinecraftServer projected down to what the halt picker shows:
|
||||||
|
// its name, its observed phase (or desired state before the operator has reconciled
|
||||||
|
// it), and whether it is a platform system server whose halt takes the front door
|
||||||
|
// down.
|
||||||
|
type haltableServer struct {
|
||||||
|
name string
|
||||||
|
phase string
|
||||||
|
system bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// haltOutcome is the durable result of a halt attempt, surfaced in the TUI card and
|
||||||
|
// re-printed to the normal screen after the alt-screen tears down.
|
||||||
|
type haltOutcome struct {
|
||||||
|
name string
|
||||||
|
namespace string
|
||||||
|
alreadyStopped bool
|
||||||
|
system bool // login/lobby — halting these takes the auth front door down
|
||||||
|
auditErr error // non-nil if the accountability row could not be written
|
||||||
|
}
|
||||||
|
|
||||||
|
// listServersForHalt lists the MinecraftServers an operator may halt in the given
|
||||||
|
// namespace, projecting only what the picker renders. The phase falls back to the
|
||||||
|
// desired state for a server the operator has not yet reconciled (empty status).
|
||||||
|
func listServersForHalt(ctx context.Context, cl client.Client, namespace string) ([]haltableServer, error) {
|
||||||
|
var list v1alpha1.MinecraftServerList
|
||||||
|
if err := cl.List(ctx, &list, client.InNamespace(namespace)); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := make([]haltableServer, 0, len(list.Items))
|
||||||
|
for i := range list.Items {
|
||||||
|
ms := &list.Items[i]
|
||||||
|
phase := string(ms.Status.Phase)
|
||||||
|
if phase == "" {
|
||||||
|
phase = string(ms.Spec.DesiredState) // not yet reconciled — show intent
|
||||||
|
}
|
||||||
|
out = append(out, haltableServer{
|
||||||
|
name: ms.Name,
|
||||||
|
phase: phase,
|
||||||
|
system: isSystemServer(ms.Name),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// haltServer flips one MinecraftServer's desiredState to Stopped with a spec-only
|
||||||
|
// merge patch. MergeFrom (not Update) is deliberate: the operator writes status on
|
||||||
|
// the same object continuously, and a full-object Update would race and clobber it,
|
||||||
|
// whereas a merge patch of spec.desiredState touches a disjoint field. Halting a
|
||||||
|
// server already Stopped is a no-op, reported via alreadyStopped so the console can
|
||||||
|
// say "already stopped" instead of claiming it just stopped it.
|
||||||
|
func haltServer(ctx context.Context, cl client.Client, namespace, name string) (haltOutcome, error) {
|
||||||
|
var ms v1alpha1.MinecraftServer
|
||||||
|
if err := cl.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &ms); err != nil {
|
||||||
|
if apierrors.IsNotFound(err) {
|
||||||
|
return haltOutcome{}, fmt.Errorf("no MinecraftServer %q in namespace %q", name, namespace)
|
||||||
|
}
|
||||||
|
return haltOutcome{}, fmt.Errorf("get server %q: %w", name, err)
|
||||||
|
}
|
||||||
|
out := haltOutcome{name: name, namespace: namespace, system: isSystemServer(name)}
|
||||||
|
if ms.Spec.DesiredState == v1alpha1.DesiredStopped {
|
||||||
|
out.alreadyStopped = true
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
patch := client.MergeFrom(ms.DeepCopy())
|
||||||
|
ms.Spec.DesiredState = v1alpha1.DesiredStopped
|
||||||
|
if err := cl.Patch(ctx, &ms, patch); err != nil {
|
||||||
|
return haltOutcome{}, fmt.Errorf("halt server %q: %w", name, err)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isSystemServer reports whether name is a platform-provisioned system server. The
|
||||||
|
// login gate has no fallback (systemservers.go), so halting it locks every player
|
||||||
|
// out of the whole proxy — the console warns when the target is one rather than
|
||||||
|
// forbidding it, since break-glass is deliberately full power.
|
||||||
|
func isSystemServer(name string) bool {
|
||||||
|
return name == naming.SystemLoginServer || name == naming.SystemLobbyServer
|
||||||
|
}
|
||||||
|
|
||||||
|
// performHalt runs haltServer and records a best-effort accountability audit row. It
|
||||||
|
// mirrors performBreakGlass: the halt succeeds even when the audit sink is unhappy
|
||||||
|
// (break-glass must work with logging down), and any audit error rides back in the
|
||||||
|
// outcome for the console to surface. accountable is the OS user who escalated to
|
||||||
|
// root — attribution, not proof (the root gate is the real authority).
|
||||||
|
func performHalt(ctx context.Context, cl client.Client, s ownerStore, namespace, name, accountable string) (haltOutcome, error) {
|
||||||
|
out, err := haltServer(ctx, cl, namespace, name)
|
||||||
|
if err != nil {
|
||||||
|
return haltOutcome{}, err
|
||||||
|
}
|
||||||
|
out.auditErr = auditHalt(ctx, s, out, accountable)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// auditHalt writes the halt accountability row under the break_glass.halt action,
|
||||||
|
// recording who halted what and whether it was a no-op or a system server.
|
||||||
|
func auditHalt(ctx context.Context, s ownerStore, out haltOutcome, accountable string) error {
|
||||||
|
blob, err := json.Marshal(map[string]any{
|
||||||
|
"server": out.name,
|
||||||
|
"namespace": out.namespace,
|
||||||
|
"os_user": accountable,
|
||||||
|
"already_stopped": out.alreadyStopped,
|
||||||
|
"system_server": out.system,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return s.Audit(ctx, api.AuditEntry{
|
||||||
|
Actor: accountable,
|
||||||
|
Source: "break-glass",
|
||||||
|
Action: "break_glass.halt",
|
||||||
|
Payload: blob,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,179 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
||||||
|
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/runtime"
|
||||||
|
"k8s.io/apimachinery/pkg/types"
|
||||||
|
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
|
||||||
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||||
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||||
|
)
|
||||||
|
|
||||||
|
const haltNS = "minecraft"
|
||||||
|
|
||||||
|
func haltScheme(t *testing.T) *runtime.Scheme {
|
||||||
|
t.Helper()
|
||||||
|
scheme := runtime.NewScheme()
|
||||||
|
if err := clientgoscheme.AddToScheme(scheme); err != nil {
|
||||||
|
t.Fatalf("clientgo scheme: %v", err)
|
||||||
|
}
|
||||||
|
if err := v1alpha1.AddToScheme(scheme); err != nil {
|
||||||
|
t.Fatalf("v1alpha1 scheme: %v", err)
|
||||||
|
}
|
||||||
|
return scheme
|
||||||
|
}
|
||||||
|
|
||||||
|
func mcServer(name string, desired v1alpha1.DesiredState, phase v1alpha1.Phase) *v1alpha1.MinecraftServer {
|
||||||
|
return &v1alpha1.MinecraftServer{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: haltNS},
|
||||||
|
Spec: v1alpha1.MinecraftServerSpec{DesiredState: desired},
|
||||||
|
Status: v1alpha1.MinecraftServerStatus{Phase: phase},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func haltClient(t *testing.T, objs ...client.Object) client.Client {
|
||||||
|
t.Helper()
|
||||||
|
return fake.NewClientBuilder().WithScheme(haltScheme(t)).WithObjects(objs...).Build()
|
||||||
|
}
|
||||||
|
|
||||||
|
// desiredStateOf reads a server's spec.desiredState straight back from the client,
|
||||||
|
// so the patch is verified by its actual persisted effect, not by "Patch was called".
|
||||||
|
func desiredStateOf(t *testing.T, cl client.Client, name string) v1alpha1.DesiredState {
|
||||||
|
t.Helper()
|
||||||
|
var ms v1alpha1.MinecraftServer
|
||||||
|
if err := cl.Get(context.Background(), types.NamespacedName{Namespace: haltNS, Name: name}, &ms); err != nil {
|
||||||
|
t.Fatalf("get %q back: %v", name, err)
|
||||||
|
}
|
||||||
|
return ms.Spec.DesiredState
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHaltServer(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
t.Run("running server is patched to Stopped", func(t *testing.T) {
|
||||||
|
cl := haltClient(t, mcServer("survival", v1alpha1.DesiredRunning, v1alpha1.PhaseRunning))
|
||||||
|
out, err := haltServer(ctx, cl, haltNS, "survival")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("haltServer: %v", err)
|
||||||
|
}
|
||||||
|
if out.alreadyStopped {
|
||||||
|
t.Error("alreadyStopped = true, want false for a running server")
|
||||||
|
}
|
||||||
|
if got := desiredStateOf(t, cl, "survival"); got != v1alpha1.DesiredStopped {
|
||||||
|
t.Errorf("desiredState after halt = %q, want Stopped", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("already-stopped server is a reported no-op", func(t *testing.T) {
|
||||||
|
cl := haltClient(t, mcServer("survival", v1alpha1.DesiredStopped, v1alpha1.PhaseStopped))
|
||||||
|
out, err := haltServer(ctx, cl, haltNS, "survival")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("haltServer: %v", err)
|
||||||
|
}
|
||||||
|
if !out.alreadyStopped {
|
||||||
|
t.Error("alreadyStopped = false, want true for an already-stopped server")
|
||||||
|
}
|
||||||
|
if got := desiredStateOf(t, cl, "survival"); got != v1alpha1.DesiredStopped {
|
||||||
|
t.Errorf("desiredState = %q, want it left Stopped", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("missing server is a clear error, not a patch", func(t *testing.T) {
|
||||||
|
cl := haltClient(t)
|
||||||
|
_, err := haltServer(ctx, cl, haltNS, "ghost")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("haltServer(ghost) = nil error, want not-found error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "ghost") {
|
||||||
|
t.Errorf("error %q does not name the missing server", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("halting login is allowed and flagged a system server", func(t *testing.T) {
|
||||||
|
cl := haltClient(t, mcServer("login", v1alpha1.DesiredRunning, v1alpha1.PhaseRunning))
|
||||||
|
out, err := haltServer(ctx, cl, haltNS, "login")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("haltServer(login): %v", err)
|
||||||
|
}
|
||||||
|
if !out.system {
|
||||||
|
t.Error("system = false, want true for the login front-door server")
|
||||||
|
}
|
||||||
|
if got := desiredStateOf(t, cl, "login"); got != v1alpha1.DesiredStopped {
|
||||||
|
t.Errorf("desiredState after halt = %q, want Stopped", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListServersForHalt(t *testing.T) {
|
||||||
|
cl := haltClient(t,
|
||||||
|
mcServer("survival", v1alpha1.DesiredRunning, ""), // no status yet → phase falls back to intent
|
||||||
|
mcServer("login", v1alpha1.DesiredRunning, ""),
|
||||||
|
)
|
||||||
|
got, err := listServersForHalt(context.Background(), cl, haltNS)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("listServersForHalt: %v", err)
|
||||||
|
}
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("listed %d servers, want 2", len(got))
|
||||||
|
}
|
||||||
|
by := map[string]haltableServer{}
|
||||||
|
for _, s := range got {
|
||||||
|
by[s.name] = s
|
||||||
|
}
|
||||||
|
if s := by["survival"]; s.system || s.phase != "Running" {
|
||||||
|
t.Errorf("survival projected %+v, want system=false phase=Running (desired-state fallback)", s)
|
||||||
|
}
|
||||||
|
if s := by["login"]; !s.system {
|
||||||
|
t.Errorf("login projected system=false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPerformHalt(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
t.Run("halts and records an accountability audit row", func(t *testing.T) {
|
||||||
|
cl := haltClient(t, mcServer("survival", v1alpha1.DesiredRunning, v1alpha1.PhaseRunning))
|
||||||
|
f := &fakeOwnerStore{}
|
||||||
|
out, err := performHalt(ctx, cl, f, haltNS, "survival", "deploybot")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("performHalt: %v", err)
|
||||||
|
}
|
||||||
|
if out.auditErr != nil {
|
||||||
|
t.Errorf("auditErr = %v, want nil", out.auditErr)
|
||||||
|
}
|
||||||
|
if got := desiredStateOf(t, cl, "survival"); got != v1alpha1.DesiredStopped {
|
||||||
|
t.Errorf("desiredState after performHalt = %q, want Stopped", got)
|
||||||
|
}
|
||||||
|
e, payload := auditOf(t, f)
|
||||||
|
if e.Action != "break_glass.halt" {
|
||||||
|
t.Errorf("audit action = %q, want break_glass.halt", e.Action)
|
||||||
|
}
|
||||||
|
if e.Actor != "deploybot" {
|
||||||
|
t.Errorf("audit actor = %q, want deploybot", e.Actor)
|
||||||
|
}
|
||||||
|
if payload["server"] != "survival" || payload["system_server"] != false {
|
||||||
|
t.Errorf("audit payload = %v, want server=survival system_server=false", payload)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("a failed audit sink does not fail the halt", func(t *testing.T) {
|
||||||
|
cl := haltClient(t, mcServer("survival", v1alpha1.DesiredRunning, v1alpha1.PhaseRunning))
|
||||||
|
f := &fakeOwnerStore{auditErr: context.DeadlineExceeded}
|
||||||
|
out, err := performHalt(ctx, cl, f, haltNS, "survival", "deploybot")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("performHalt returned error on audit failure, want halt to still succeed: %v", err)
|
||||||
|
}
|
||||||
|
if out.auditErr == nil {
|
||||||
|
t.Error("auditErr = nil, want the sink failure surfaced")
|
||||||
|
}
|
||||||
|
if got := desiredStateOf(t, cl, "survival"); got != v1alpha1.DesiredStopped {
|
||||||
|
t.Errorf("desiredState = %q, want Stopped even though the audit failed", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
+1
-1
@@ -102,7 +102,7 @@ func cmdSetup(args []string, stdout, stderr io.Writer) int {
|
|||||||
}
|
}
|
||||||
defer setup.drv.Close()
|
defer setup.drv.Close()
|
||||||
|
|
||||||
res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database.URL, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, accountableOSUser(), setup.adminExists)
|
res, err := runSetupTUI(ctx, setup.repo, setup.cfg.Database.URL, setup.cfg.Server.RootDomain, setup.cfg.Auth.AdminHostname, setup.cfg.Auth.PanelHostname, setup.cfg.Auth.AccessJWTAud, setup.cfg.K8s.Namespace, accountableOSUser(), setup.adminExists)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "felis setup: %v\n", err)
|
fmt.Fprintf(stderr, "felis setup: %v\n", err)
|
||||||
return 1
|
return 1
|
||||||
|
|||||||
@@ -0,0 +1,268 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/charmbracelet/bubbles/spinner"
|
||||||
|
tea "github.com/charmbracelet/bubbletea"
|
||||||
|
"github.com/charmbracelet/huh"
|
||||||
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||||
|
)
|
||||||
|
|
||||||
|
// haltModel is the break-glass "halt a server" screen. It mirrors ownerModel's
|
||||||
|
// shape (async load → huh pick → async work → outcome card) but carries no auth
|
||||||
|
// branch: the console's root gate is the authority, and the accountable OS user is
|
||||||
|
// already known. All decision logic lives in halt.go (unit-tested); this file is the
|
||||||
|
// untested terminal glue, like the other console screens.
|
||||||
|
type haltStep int
|
||||||
|
|
||||||
|
const (
|
||||||
|
haltLoading haltStep = iota // building the cluster client + listing servers
|
||||||
|
haltPick // choosing which server to halt
|
||||||
|
haltWorking // patching desiredState=Stopped
|
||||||
|
haltDone // outcome card, or the empty/error terminal note
|
||||||
|
)
|
||||||
|
|
||||||
|
// haltListMsg carries the built cluster client and haltable server list (or the
|
||||||
|
// failure of either) back from the async load.
|
||||||
|
type haltListMsg struct {
|
||||||
|
cl client.Client
|
||||||
|
servers []haltableServer
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type haltPerformedMsg struct {
|
||||||
|
outcome haltOutcome
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
// haltResultMsg is the terminal signal to the root: it records the outcome into
|
||||||
|
// breakGlassResult and quits, so cmdBreakGlass can re-print it after the alt-screen
|
||||||
|
// is torn down. done is false for a cancel or an empty fleet (no change made).
|
||||||
|
type haltResultMsg struct {
|
||||||
|
outcome haltOutcome
|
||||||
|
done bool
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
type haltModel struct {
|
||||||
|
ctx context.Context
|
||||||
|
store ownerStore
|
||||||
|
namespace string
|
||||||
|
osUser string
|
||||||
|
|
||||||
|
step haltStep
|
||||||
|
cl client.Client
|
||||||
|
sp spinner.Model
|
||||||
|
form *huh.Form
|
||||||
|
|
||||||
|
servers []haltableServer
|
||||||
|
pick string // huh-bound selected server name
|
||||||
|
outcome haltOutcome
|
||||||
|
loadErr error
|
||||||
|
empty bool
|
||||||
|
|
||||||
|
width, height int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newHaltModel(ctx context.Context, store ownerStore, namespace, osUser string) *haltModel {
|
||||||
|
sp := spinner.New()
|
||||||
|
sp.Spinner = spinner.Dot
|
||||||
|
sp.Style = tuiLabel
|
||||||
|
return &haltModel{ctx: ctx, store: store, namespace: namespace, osUser: osUser, sp: sp, step: haltLoading}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *haltModel) Init() tea.Cmd { return tea.Batch(m.sp.Tick, m.loadCmd()) }
|
||||||
|
|
||||||
|
// loadCmd builds the cluster client and lists haltable servers off the UI thread.
|
||||||
|
// The client build (no kubeconfig) and the list (no API) can each fail; either
|
||||||
|
// becomes the screen's terminal error rather than a panic.
|
||||||
|
func (m *haltModel) loadCmd() tea.Cmd {
|
||||||
|
return func() tea.Msg {
|
||||||
|
cl, err := buildSystemServerClient()
|
||||||
|
if err != nil {
|
||||||
|
return haltListMsg{err: fmt.Errorf("connect to cluster: %w", err)}
|
||||||
|
}
|
||||||
|
servers, err := listServersForHalt(m.ctx, cl, m.namespace)
|
||||||
|
if err != nil {
|
||||||
|
return haltListMsg{err: fmt.Errorf("list servers: %w", err)}
|
||||||
|
}
|
||||||
|
return haltListMsg{cl: cl, servers: servers}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *haltModel) setSize(w, h int) {
|
||||||
|
m.width, m.height = w, h
|
||||||
|
if m.form != nil {
|
||||||
|
m.form = m.form.WithWidth(w).WithHeight(h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *haltModel) sized(f *huh.Form) *huh.Form {
|
||||||
|
if m.width > 0 {
|
||||||
|
return f.WithWidth(m.width).WithHeight(m.height)
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *haltModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||||
|
switch msg := msg.(type) {
|
||||||
|
case haltListMsg:
|
||||||
|
if msg.err != nil {
|
||||||
|
m.loadErr = msg.err
|
||||||
|
m.step = haltDone
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
m.cl = msg.cl
|
||||||
|
m.servers = msg.servers
|
||||||
|
if len(m.servers) == 0 {
|
||||||
|
m.empty = true
|
||||||
|
m.step = haltDone
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
m.step = haltPick
|
||||||
|
m.form = m.sized(m.buildPickForm())
|
||||||
|
return m, m.form.Init()
|
||||||
|
|
||||||
|
case haltPerformedMsg:
|
||||||
|
m.step = haltDone
|
||||||
|
if msg.err != nil {
|
||||||
|
m.loadErr = msg.err
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
m.outcome = msg.outcome
|
||||||
|
return m, nil
|
||||||
|
|
||||||
|
case spinner.TickMsg:
|
||||||
|
if m.step == haltLoading || m.step == haltWorking {
|
||||||
|
var cmd tea.Cmd
|
||||||
|
m.sp, cmd = m.sp.Update(msg)
|
||||||
|
return m, cmd
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
|
||||||
|
case tea.KeyMsg:
|
||||||
|
switch m.step {
|
||||||
|
case haltDone:
|
||||||
|
switch msg.String() {
|
||||||
|
case "ctrl+c", "esc", "enter":
|
||||||
|
return m, m.exitCmd()
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
case haltLoading, haltWorking:
|
||||||
|
if msg.String() == "ctrl+c" {
|
||||||
|
return m, tea.Quit
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
case haltPick:
|
||||||
|
switch msg.String() {
|
||||||
|
case "ctrl+c", "esc":
|
||||||
|
return m, tea.Quit
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if m.step == haltPick && m.form != nil {
|
||||||
|
form, cmd := m.form.Update(msg)
|
||||||
|
if f, ok := form.(*huh.Form); ok {
|
||||||
|
m.form = f
|
||||||
|
}
|
||||||
|
switch m.form.State {
|
||||||
|
case huh.StateCompleted:
|
||||||
|
return m.onPicked()
|
||||||
|
case huh.StateAborted:
|
||||||
|
return m, tea.Quit
|
||||||
|
}
|
||||||
|
return m, cmd
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *haltModel) onPicked() (tea.Model, tea.Cmd) {
|
||||||
|
name := strings.TrimSpace(m.pick)
|
||||||
|
m.step = haltWorking
|
||||||
|
cl, ns, store, osUser := m.cl, m.namespace, m.store, m.osUser
|
||||||
|
return m, tea.Batch(m.sp.Tick, func() tea.Msg {
|
||||||
|
out, err := performHalt(m.ctx, cl, store, ns, name, osUser)
|
||||||
|
return haltPerformedMsg{outcome: out, err: err}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// exitCmd hands the outcome to the root: a load/perform error routes through the
|
||||||
|
// root's error path, a real halt records the durable summary, and an empty fleet is
|
||||||
|
// a benign cancel.
|
||||||
|
func (m *haltModel) exitCmd() tea.Cmd {
|
||||||
|
out, done, err := m.outcome, !m.empty && m.loadErr == nil, m.loadErr
|
||||||
|
return func() tea.Msg { return haltResultMsg{outcome: out, done: done, err: err} }
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *haltModel) buildPickForm() *huh.Form {
|
||||||
|
opts := make([]huh.Option[string], 0, len(m.servers))
|
||||||
|
for _, s := range m.servers {
|
||||||
|
label := fmt.Sprintf("%s (%s)", s.name, s.phase)
|
||||||
|
if s.system {
|
||||||
|
label += " ⚠ system — front door"
|
||||||
|
}
|
||||||
|
opts = append(opts, huh.NewOption(label, s.name))
|
||||||
|
}
|
||||||
|
return m.sized(newFelisForm(huh.NewGroup(
|
||||||
|
huh.NewSelect[string]().
|
||||||
|
Title("Halt a server").
|
||||||
|
Description("Sets desiredState=Stopped; the operator reconciles a graceful shutdown.").
|
||||||
|
Value(&m.pick).
|
||||||
|
Options(opts...),
|
||||||
|
huh.NewNote().Description(
|
||||||
|
"Halting a ⚠ system server stops shared infrastructure: halting login takes "+
|
||||||
|
"the whole auth front door down (it has no fallback)."),
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *haltModel) View() string {
|
||||||
|
switch m.step {
|
||||||
|
case haltLoading:
|
||||||
|
return " " + m.sp.View() + " " + tuiHint.Render("Connecting to the cluster…") + "\n"
|
||||||
|
case haltWorking:
|
||||||
|
return " " + m.sp.View() + " " + tuiHint.Render("Halting "+strings.TrimSpace(m.pick)+"…") + "\n"
|
||||||
|
case haltDone:
|
||||||
|
return m.doneView()
|
||||||
|
default:
|
||||||
|
if m.form == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return m.form.View()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *haltModel) doneView() string {
|
||||||
|
var b strings.Builder
|
||||||
|
switch {
|
||||||
|
case m.loadErr != nil:
|
||||||
|
b.WriteString(tuiWarn.Render("Halt failed.") + "\n\n")
|
||||||
|
b.WriteString(tuiCardStyle.Render(m.loadErr.Error()) + "\n\n")
|
||||||
|
case m.empty:
|
||||||
|
b.WriteString(tuiHint.Render("No servers to halt in namespace "+m.namespace+".") + "\n\n")
|
||||||
|
default:
|
||||||
|
b.WriteString(tuiSuccessBanner(haltHeadline(m.outcome)) + "\n\n")
|
||||||
|
var box strings.Builder
|
||||||
|
box.WriteString(tuiLabel.Render("server ") + m.outcome.name + "\n")
|
||||||
|
box.WriteString(tuiLabel.Render("namespace ") + m.outcome.namespace)
|
||||||
|
if m.outcome.system {
|
||||||
|
box.WriteString("\n\n" + tuiWarn.Render("This is a system server — the shared front door is now going down."))
|
||||||
|
}
|
||||||
|
if m.outcome.auditErr != nil {
|
||||||
|
box.WriteString("\n\n" + tuiWarn.Render("Audit warning: "+m.outcome.auditErr.Error()))
|
||||||
|
}
|
||||||
|
b.WriteString(tuiCardStyle.Render(box.String()) + "\n\n")
|
||||||
|
}
|
||||||
|
b.WriteString(tuiAction("enter", "continue"))
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func haltHeadline(out haltOutcome) string {
|
||||||
|
if out.alreadyStopped {
|
||||||
|
return out.name + " was already stopped."
|
||||||
|
}
|
||||||
|
return out.name + " is stopping."
|
||||||
|
}
|
||||||
@@ -14,6 +14,7 @@ type bgOperation int
|
|||||||
const (
|
const (
|
||||||
bgProvisionOwner bgOperation = iota
|
bgProvisionOwner bgOperation = iota
|
||||||
bgAddOperator
|
bgAddOperator
|
||||||
|
bgHaltServer
|
||||||
)
|
)
|
||||||
|
|
||||||
// menuChoiceMsg is emitted to the root once the operator picks an operation. The
|
// menuChoiceMsg is emitted to the root once the operator picks an operation. The
|
||||||
@@ -50,6 +51,7 @@ func (m *menuModel) build() *huh.Form {
|
|||||||
// recovery flow, and landing on it keeps that path a single Enter.
|
// recovery flow, and landing on it keeps that path a single Enter.
|
||||||
huh.NewOption("Provision or reset the Owner account", bgProvisionOwner),
|
huh.NewOption("Provision or reset the Owner account", bgProvisionOwner),
|
||||||
huh.NewOption("Add an Operator account", bgAddOperator),
|
huh.NewOption("Add an Operator account", bgAddOperator),
|
||||||
|
huh.NewOption("Halt a running server", bgHaltServer),
|
||||||
),
|
),
|
||||||
// A dim footnote spelling out the one behavioural difference that matters:
|
// A dim footnote spelling out the one behavioural difference that matters:
|
||||||
// Owner-reset re-enables local-password login, operator-add never touches the
|
// Owner-reset re-enables local-password login, operator-add never touches the
|
||||||
|
|||||||
+26
-1
@@ -139,10 +139,11 @@ type rootModel struct {
|
|||||||
adminHost string
|
adminHost string
|
||||||
panelHost string
|
panelHost string
|
||||||
accessAud string
|
accessAud string
|
||||||
|
namespace string // minecraft workload namespace (cfg.K8s.Namespace); target of the halt op
|
||||||
adminExists bool
|
adminExists bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func newRootModel(ctx context.Context, store ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, osUser string, adminExists bool, mode consoleMode) *rootModel {
|
func newRootModel(ctx context.Context, store ownerStore, dbURL, rootDomain, adminHostname, panelHostname, accessAud, namespace, osUser string, adminExists bool, mode consoleMode) *rootModel {
|
||||||
rm := &rootModel{
|
rm := &rootModel{
|
||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
reviewing: -1,
|
reviewing: -1,
|
||||||
@@ -153,6 +154,7 @@ func newRootModel(ctx context.Context, store ownerStore, dbURL, rootDomain, admi
|
|||||||
adminHost: adminHostname,
|
adminHost: adminHostname,
|
||||||
panelHost: panelHostname,
|
panelHost: panelHostname,
|
||||||
accessAud: accessAud,
|
accessAud: accessAud,
|
||||||
|
namespace: namespace,
|
||||||
adminExists: adminExists,
|
adminExists: adminExists,
|
||||||
mode: mode,
|
mode: mode,
|
||||||
result: breakGlassResult{
|
result: breakGlassResult{
|
||||||
@@ -219,10 +221,33 @@ func (m *rootModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
switch msg.op {
|
switch msg.op {
|
||||||
case bgAddOperator:
|
case bgAddOperator:
|
||||||
return m.adopt(newOperatorModel(m.ctx, m.store, m.osUser))
|
return m.adopt(newOperatorModel(m.ctx, m.store, m.osUser))
|
||||||
|
case bgHaltServer:
|
||||||
|
return m.adopt(newHaltModel(m.ctx, m.store, m.namespace, m.osUser))
|
||||||
default:
|
default:
|
||||||
return m.adopt(newOwnerModel(m.ctx, m.store, m.osUser, m.adminExists))
|
return m.adopt(newOwnerModel(m.ctx, m.store, m.osUser, m.adminExists))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case haltResultMsg:
|
||||||
|
// Halt is terminal in break-glass: record the durable summary (so cmdBreakGlass
|
||||||
|
// can re-print it past the alt-screen teardown) and quit. A load/perform failure
|
||||||
|
// routes through the root's error path; an empty fleet or cancel leaves halted
|
||||||
|
// false, so the summary reports "no changes made".
|
||||||
|
if msg.err != nil {
|
||||||
|
m.err = msg.err
|
||||||
|
return m, tea.Quit
|
||||||
|
}
|
||||||
|
if msg.done {
|
||||||
|
m.result.halted = true
|
||||||
|
m.result.haltServer = msg.outcome.name
|
||||||
|
m.result.haltNamespace = msg.outcome.namespace
|
||||||
|
m.result.haltAlreadyStopped = msg.outcome.alreadyStopped
|
||||||
|
m.result.haltSystemServer = msg.outcome.system
|
||||||
|
if msg.outcome.auditErr != nil {
|
||||||
|
m.result.haltAuditWarning = msg.outcome.auditErr.Error()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m, tea.Quit
|
||||||
|
|
||||||
case ownerResultMsg:
|
case ownerResultMsg:
|
||||||
if msg.err != nil {
|
if msg.err != nil {
|
||||||
m.err = msg.err
|
m.err = msg.err
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ func newTestRoot(adminExists bool, mode consoleMode, accessAud string) *rootMode
|
|||||||
"admin.felis.example.com",
|
"admin.felis.example.com",
|
||||||
"panel.felis.example.com",
|
"panel.felis.example.com",
|
||||||
accessAud,
|
accessAud,
|
||||||
|
"minecraft",
|
||||||
"root",
|
"root",
|
||||||
adminExists,
|
adminExists,
|
||||||
mode,
|
mode,
|
||||||
|
|||||||
Reference in new issue
Block a user