fix(api): wake 与回档/备份/改文件按服务器互斥

This commit is contained in:
Lemon-miaow committed 2026-09-24 14:39:02 +08:00
1 parent 7819e5de50
commit abfe60d62d
27 files changed
+1300 -36

No files matched your search

+3
View File
@@ -107,6 +107,9 @@ func cmdOperator(args []string, _, stderr io.Writer) int {
// injects into user servers. The Deployment passes it as FELIS_IMAGE (see // injects into user servers. The Deployment passes it as FELIS_IMAGE (see
// platform.OperatorDeployment); absent, that injection is simply skipped. // platform.OperatorDeployment); absent, that injection is simply skipped.
FelisImage: os.Getenv("FELIS_IMAGE"), FelisImage: os.Getenv("FELIS_IMAGE"),
// Uncached: the maintenance-lock check lists Jobs only when a server is
// about to start, which does not justify a namespace-wide Job informer.
Jobs: mgr.GetAPIReader(),
} }
if err := r.SetupWithManager(mgr); err != nil { if err := r.SetupWithManager(mgr); err != nil {
fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err) fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err)
+14 -4
View File
@@ -741,6 +741,11 @@ paths:
$ref: '#/components/responses/Forbidden' $ref: '#/components/responses/Forbidden'
'404': '404':
$ref: '#/components/responses/NotFound' $ref: '#/components/responses/NotFound'
'409':
description: A restore, backup or file write holds the server's world volume (maintenance_in_progress); nothing was started.
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'429': '429':
description: Wake cooldown is still active for this server. description: Wake cooldown is still active for this server.
content: content:
@@ -1193,7 +1198,7 @@ paths:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'409': '409':
description: Server is not stopped (its world PVC is still mounted). description: Server is not stopped (not_stopped), or a restore, backup or file write already holds its world volume (maintenance_in_progress).
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
@@ -1228,6 +1233,11 @@ paths:
$ref: '#/components/responses/Forbidden' $ref: '#/components/responses/Forbidden'
'404': '404':
$ref: '#/components/responses/NotFound' $ref: '#/components/responses/NotFound'
'409':
description: A restore, backup or file write holds the server's world volume (maintenance_in_progress); nothing was started.
content:
application/json:
schema: { $ref: '#/components/schemas/Error' }
'429': '429':
description: Wake cooldown is still active. description: Wake cooldown is still active.
content: content:
@@ -2828,7 +2838,7 @@ paths:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'409': '409':
description: Submission has already been reviewed. description: Server is not stopped (not_stopped), or a restore, backup or file write already holds its world volume (maintenance_in_progress).
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
@@ -2873,7 +2883,7 @@ paths:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'409': '409':
description: Server is not stopped (its world PVC is still mounted). description: Server is not stopped (not_stopped), or a restore, backup or file write already holds its world volume (maintenance_in_progress).
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
@@ -3122,7 +3132,7 @@ paths:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
'409': '409':
description: Server is not stopped (its world PVC is still mounted). description: Server is not stopped (not_stopped), or a restore, backup or file write already holds its world volume (maintenance_in_progress).
content: content:
application/json: application/json:
schema: { $ref: '#/components/schemas/Error' } schema: { $ref: '#/components/schemas/Error' }
+47 -2
View File
@@ -185,6 +185,7 @@ per-server cooldown → global running cap**. Map the API result:
| HTTP | Code | Cause | Fix | | HTTP | Code | Cause | Fix |
|---|---|---|---| |---|---|---|---|
| `403` | `forbidden` | `autostartPolicy=allowlist` and UUID not allowlisted, or `ownerOnly` and caller is not owner | Add the UUID / claim the server / set `autostartPolicy=public` | | `403` | `forbidden` | `autostartPolicy=allowlist` and UUID not allowlisted, or `ownerOnly` and caller is not owner | Add the UUID / claim the server / set `autostartPolicy=public` |
| `409` | `maintenance_in_progress` | A restore, backup or file write holds the server's world volume (§3b) | Wait for the Job to finish |
| `429` | (cooldown) | Wake retried within the 30s per-server `WakeCooldown` | Wait out the cooldown | | `429` | (cooldown) | Wake retried within the 30s per-server `WakeCooldown` | Wait out the cooldown |
| `503` | `at_capacity` | Global `MaxRunningServers` cap reached | Stop another server or raise the cap | | `503` | `at_capacity` | Global `MaxRunningServers` cap reached | Stop another server or raise the cap |
@@ -194,11 +195,54 @@ gate; the proxy polls `GET /api/v1/internal/servers/{name}/status` every ~2s and
teleports when `ready=true`. teleports when `ready=true`.
The Velocity-side consumption of these codes (`403` → "You're not allowed to The Velocity-side consumption of these codes (`403` → "You're not allowed to
start «server»"; `429` → re-queue; other → "Couldn't start … Try again start «server»"; `409 maintenance_in_progress` → "«server» is under
maintenance", not queued; `429` → re-queue; other → "Couldn't start … Try again
shortly.") lives in the Java plugin and is **[CODE-ONLY]** — the codes it reacts shortly.") lives in the Java plugin and is **[CODE-ONLY]** — the codes it reacts
to are produced by the Go-tested `authorizeWakeByUUID` / cooldown limiter, so to are produced by the Go-tested `authorizeWakeByUUID` / cooldown limiter, so
grade the two halves separately. grade the two halves separately.
### 3b. Wake, restore, backup or file save refused with `maintenance_in_progress`
A server's world volume is ReadWriteOnce, and on a single node RWO lets a game
pod and a restore Job mount it side by side. So felis-api serialises them per
server: a restore, a backup, or a file write takes the world, and until its Job
finishes every wake (panel or join) and every other world operation on that
server gets `409 maintenance_in_progress`. File reads and listings never hold
it. The operator applies the same rule when `desiredState` is flipped to
`Running` by anything other than felis-api: the StatefulSet is not scaled up,
and the `Ready` condition reads `MaintenanceInProgress` until the Job ends.
What holds the world, in order:
1. An unfinished Job labelled `felis.lolicon.best/server=<name>` with
`app.kubernetes.io/managed-by` `felis-restore`, `felis-backup`, or
`felis-files` plus `felis.lolicon.best/files-mode=write`:
```sh
kubectl -n minecraft get jobs -l felis.lolicon.best/server=<name>
```
A Job that is genuinely wedged is ended by its own `activeDeadlineSeconds`;
deleting it by hand releases the world at once (`kubectl -n minecraft delete
job <job>`), at the cost of whatever it was writing.
2. The admission lock `felis.lolicon.best/maintenance=<kind>@<RFC3339>` on the
MinecraftServer. felis-api sets it for the milliseconds between admitting an
operation and creating its Job; it holds for at most two minutes if felis-api
died in between, and the next wake clears a stale one. To drop it by hand:
```sh
kubectl -n minecraft annotate minecraftserver <name> felis.lolicon.best/maintenance-
```
A restore, backup or file write refused with `409 not_stopped` although the
panel shows `Stopped` means the game pod is still terminating (its preStop save
can take a while); retry once `kubectl -n minecraft get pods -l
felis.lolicon.best/server=<name>` shows nothing.
[GO-TESTED: `internal/maintenance`, `k8scluster_maintenance_test.go`,
`handlers_maintenance_test.go`, operator `maintenance_test.go`.]
--- ---
## 4. Routing is disabled even though servers are up (online-mode coupling) ## 4. Routing is disabled even though servers are up (online-mode coupling)
@@ -947,7 +991,8 @@ installer built — only hand-built tags need a manual re-mirror.
| RCON secret/auth/port errors | §1b, §1c | | RCON secret/auth/port errors | §1b, §1c |
| Phase `Failed` | §2 | | Phase `Failed` | §2 |
| Players land in lobby / wrong place | §3, §4 | | Players land in lobby / wrong place | §3, §4 |
| Wake refused / rate-limited (403/429/503) | §3a | | Wake refused / rate-limited (403/409/429/503) | §3a |
| `maintenance_in_progress`; server won't start after a restore | §3b |
| Routing disabled, offline-mode | §4 | | Routing disabled, offline-mode | §4 |
| Panel 401/403; fails-closed; audience error | §5 | | Panel 401/403; fails-closed; audience error | §5 |
| Local password login rejected | §5c | | Local password login rejected | §5c |
+22 -1
View File
@@ -1454,12 +1454,19 @@ type fakeCluster struct {
noWorld map[string]bool // server names modeled WITHOUT a world volume (never started / reaped) noWorld map[string]bool // server names modeled WITHOUT a world volume (never started / reaped)
createErr error createErr error
pingErr error pingErr error
// maintErr / wakeErr: what AcquireMaintenance / SetDesiredState(Running)
// return for a server (the world-volume lock, internal/maintenance).
maintErr map[string]error
wakeErr map[string]error
acquired []string // "name:kind" per admitted AcquireMaintenance
released []string // names per ReleaseMaintenance
} }
func newFakeCluster() *fakeCluster { func newFakeCluster() *fakeCluster {
return &fakeCluster{byName: map[string]*ServerInfo{}, bySub: map[string]*ServerInfo{}, return &fakeCluster{byName: map[string]*ServerInfo{}, bySub: map[string]*ServerInfo{},
desired: map[string]v1alpha1.DesiredState{}, created: map[string]CreateServerInput{}, desired: map[string]v1alpha1.DesiredState{}, created: map[string]CreateServerInput{},
patched: map[string]ServerSpecPatch{}, noWorld: map[string]bool{}} patched: map[string]ServerSpecPatch{}, noWorld: map[string]bool{},
maintErr: map[string]error{}, wakeErr: map[string]error{}}
} }
func (c *fakeCluster) GetServer(_ context.Context, n string) (*ServerInfo, error) { func (c *fakeCluster) GetServer(_ context.Context, n string) (*ServerInfo, error) {
if s, ok := c.byName[n]; ok { if s, ok := c.byName[n]; ok {
@@ -1483,9 +1490,23 @@ func (c *fakeCluster) WorldVolumeExists(_ context.Context, n string) (bool, erro
} }
func (c *fakeCluster) SetDesiredState(_ context.Context, n string, s v1alpha1.DesiredState) error { func (c *fakeCluster) SetDesiredState(_ context.Context, n string, s v1alpha1.DesiredState) error {
if err := c.wakeErr[n]; err != nil && s == v1alpha1.DesiredRunning {
return err
}
c.desired[n] = s c.desired[n] = s
return nil return nil
} }
func (c *fakeCluster) AcquireMaintenance(_ context.Context, n, kind string) error {
if err := c.maintErr[n]; err != nil {
return err
}
c.acquired = append(c.acquired, n+":"+kind)
return nil
}
func (c *fakeCluster) ReleaseMaintenance(_ context.Context, n string) error {
c.released = append(c.released, n)
return nil
}
func (c *fakeCluster) CreateServer(_ context.Context, in CreateServerInput) error { func (c *fakeCluster) CreateServer(_ context.Context, in CreateServerInput) error {
if c.createErr != nil { if c.createErr != nil {
return c.createErr return c.createErr
+11 -1
View File
@@ -94,8 +94,18 @@ type Cluster interface {
// velocity registration pull (spec §7 GET /servers). // velocity registration pull (spec §7 GET /servers).
ListServers(ctx context.Context) ([]ServerInfo, error) ListServers(ctx context.Context) ([]ServerInfo, error)
// SetDesiredState flips spec.desiredState — the only write the API performs // SetDesiredState flips spec.desiredState — the only write the API performs
// against the CRD (spec §9.1). It is idempotent. // against the CRD (spec §9.1). It is idempotent. Flipping to Running returns a
// *MaintenanceBusyError (errors.Is ErrMaintenanceInProgress) while a restore,
// backup or file write holds the world volume.
SetDesiredState(ctx context.Context, name string, state v1alpha1.DesiredState) error SetDesiredState(ctx context.Context, name string, state v1alpha1.DesiredState) error
// AcquireMaintenance admits one world-volume operation (internal/maintenance
// kind): ErrNotStopped unless the server is fully stopped, a
// *MaintenanceBusyError while another operation holds the volume. The check
// and the lock are one atomic write against a concurrent wake.
AcquireMaintenance(ctx context.Context, name, kind string) error
// ReleaseMaintenance drops the admission lock once the operation's Job exists
// (or could not be created). It is idempotent.
ReleaseMaintenance(ctx context.Context, name string) error
// CreateServer creates a MinecraftServer CRD from the validated form (spec // CreateServer creates a MinecraftServer CRD from the validated form (spec
// §15). It returns ErrConflict if a server of that name already exists. // §15). It returns ErrConflict if a server of that name already exists.
CreateServer(ctx context.Context, in CreateServerInput) error CreateServer(ctx context.Context, in CreateServerInput) error
+18
View File
@@ -82,8 +82,26 @@ var (
// sentinels so the handler answers 429 (a transient "too busy, retry" — the cap self-clears // sentinels so the handler answers 429 (a transient "too busy, retry" — the cap self-clears
// as challenges expire), never a 400 that invites an immediate retry. // as challenges expire), never a 400 that invites an immediate retry.
ErrTooManyDiscoverableChallenges = errors.New("too many discoverable login challenges in flight") ErrTooManyDiscoverableChallenges = errors.New("too many discoverable login challenges in flight")
// ErrNotStopped means a world-volume operation was refused because the server is
// not fully stopped: desiredState is not Stopped, or its pod is still shutting
// down (phase Stopping) and holds the volume while it saves.
ErrNotStopped = errors.New("server is not stopped")
// ErrMaintenanceInProgress means another operation holds the server's world
// volume (internal/maintenance). Cluster methods return it wrapped in a
// *MaintenanceBusyError that names the holder.
ErrMaintenanceInProgress = errors.New("world maintenance in progress")
) )
// MaintenanceBusyError names what holds a server's world volume. errors.Is
// matches it against ErrMaintenanceInProgress.
type MaintenanceBusyError struct{ Kind string }
func (e *MaintenanceBusyError) Error() string {
return "world maintenance in progress: " + e.Kind
}
func (e *MaintenanceBusyError) Is(target error) bool { return target == ErrMaintenanceInProgress }
// apiError is a handler-level error carrying an HTTP status and a stable, // apiError is a handler-level error carrying an HTTP status and a stable,
// machine-readable code. The error envelope matches the platform convention: // machine-readable code. The error envelope matches the platform convention:
// //
+34 -8
View File
@@ -6,6 +6,7 @@ import (
"strings" "strings"
"felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/maintenance"
"felis.lolicon.best/internal/naming" "felis.lolicon.best/internal/naming"
) )
@@ -69,7 +70,10 @@ func (a *API) handleListBackups(w http.ResponseWriter, r *http.Request) {
// re-claims a released server could otherwise resurrect user A's world (the // re-claims a released server could otherwise resurrect user A's world (the
// backup still carries former_owner=A), a data leak. Admin skips this check. // backup still carries former_owner=A), a data leak. Admin skips this check.
// ⑦ stopped gate: the world PVC must be free, so restore is refused unless the // ⑦ stopped gate: the world PVC must be free, so restore is refused unless the
// server is fully stopped. // server is fully stopped. The world-volume lock (internal/maintenance) then
// makes that atomic against a wake and refuses a second restore, backup or
// file write on the same world with 409 maintenance_in_progress until the
// restore Job finishes.
// ⑧ hand off to the Restorer. Restore is asynchronous (a restore Job, like an // ⑧ hand off to the Restorer. Restore is asynchronous (a restore Job, like an
// image build Job), so success means "enqueued" and the handler answers 202. // image build Job), so success means "enqueued" and the handler answers 202.
// //
@@ -152,7 +156,9 @@ func (a *API) handleRestoreBackup(w http.ResponseWriter, r *http.Request) {
// Refuse unless the server is fully stopped — Ready means it is up, and any // Refuse unless the server is fully stopped — Ready means it is up, and any
// desiredState other than Stopped means it is up or coming up and still owns the // desiredState other than Stopped means it is up or coming up and still owns the
// RWO volume (spec §141 readiness is an RCON probe; DesiredStopped is the // RWO volume (spec §141 readiness is an RCON probe; DesiredStopped is the
// intent). This yields a specific 409 instead of a restore Job that cannot mount. // intent). This is the early, readable refusal from a snapshot; acquireWorld
// below is the atomic one (RWO is per node, so on a single node a restore Job
// WOULD mount beside a running server).
info, err := a.Cluster.GetServer(r.Context(), name) info, err := a.Cluster.GetServer(r.Context(), name)
if err != nil { if err != nil {
a.writeLookupError(w, r, err) a.writeLookupError(w, r, err)
@@ -186,6 +192,16 @@ func (a *API) handleRestoreBackup(w http.ResponseWriter, r *http.Request) {
return return
} }
// World-volume lock (internal/maintenance). The stopped gate above reads a
// snapshot; this is the atomic check, and it keeps a wake — the owner's, or a
// player's join through velocity — from booting the server on a half-extracted
// world until the restore Job has finished.
release, ok := a.acquireWorld(w, r, name, maintenance.KindRestore, "stop the server before restoring a backup")
if !ok {
return
}
defer release()
if err := a.Restorer.Restore(r.Context(), name, backup.BackupRef); err != nil { if err := a.Restorer.Restore(r.Context(), name, backup.BackupRef); err != nil {
// ErrNotFound (server vanished from the execution backend) → 404; else 500. // ErrNotFound (server vanished from the execution backend) → 404; else 500.
a.writeLookupError(w, r, err) a.writeLookupError(w, r, err)
@@ -212,9 +228,10 @@ func (a *API) handleRestoreBackup(w http.ResponseWriter, r *http.Request) {
// ② ServerByName — an unknown server is 404 // ② ServerByName — an unknown server is 404
// ③ owner-or-admin, else 403 (an unowned server passes only for admin, so a // ③ owner-or-admin, else 403 (an unowned server passes only for admin, so a
// released world can still be snapshotted by an operator before disposal) // released world can still be snapshotted by an operator before disposal)
// ④ stopped gate: the world PVC is RWO and held by a running server, so a backup // ④ stopped gate: refuse unless the server is fully stopped, so the archive is
// Job cannot double-mount it — refuse unless the server is fully stopped. This // quiescent and non-torn. The world-volume lock (internal/maintenance) keeps it
// also guarantees a quiescent, non-torn archive. // that way until the backup Job finishes: a wake meanwhile, or a second
// restore/backup/file write, gets 409 maintenance_in_progress.
// ⑤ hand off to the Backuper. Backup is asynchronous (a backup Job), so success // ⑤ hand off to the Backuper. Backup is asynchronous (a backup Job), so success
// means "enqueued" and the handler answers 202. // means "enqueued" and the handler answers 202.
// //
@@ -294,9 +311,10 @@ func (a *API) handleInternalBackup(w http.ResponseWriter, r *http.Request) {
// (Principal vs trusted service token) and the audit actor/source — keeping the // (Principal vs trusted service token) and the audit actor/source — keeping the
// security-critical stopped-gate single-sourced so the two faces cannot diverge. // security-critical stopped-gate single-sourced so the two faces cannot diverge.
func (a *API) enqueueBackup(w http.ResponseWriter, r *http.Request, name string, rec *ServerRecord, actor, source string) { func (a *API) enqueueBackup(w http.ResponseWriter, r *http.Request, name string, rec *ServerRecord, actor, source string) {
// Stopped gate: the world PVC is RWO and held by a running server, so a backup // Stopped gate (mirrors the restore gate): Ready means it is up; any
// Job cannot double-mount it (mirrors the restore gate). Ready means it is up; // desiredState other than Stopped means it is up or coming up. RWO is per node,
// any desiredState other than Stopped means it owns the RWO volume. // so on a single node the Job WOULD mount beside a live server and archive a
// torn world; acquireWorld below makes this check atomic.
info, err := a.Cluster.GetServer(r.Context(), name) info, err := a.Cluster.GetServer(r.Context(), name)
if err != nil { if err != nil {
a.writeLookupError(w, r, err) a.writeLookupError(w, r, err)
@@ -329,6 +347,14 @@ func (a *API) enqueueBackup(w http.ResponseWriter, r *http.Request, name string,
return return
} }
// World-volume lock (internal/maintenance): a server woken mid-backup would
// leave a torn archive that a later restore makes permanent.
release, ok := a.acquireWorld(w, r, name, maintenance.KindBackup, "stop the server before backing up its world")
if !ok {
return
}
defer release()
if err := a.Backuper.Backup(r.Context(), name, rec.OwnerID); err != nil { if err := a.Backuper.Backup(r.Context(), name, rec.OwnerID); err != nil {
a.writeLookupError(w, r, err) a.writeLookupError(w, r, err)
return return
+10
View File
@@ -7,6 +7,7 @@ import (
"felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/fileedit" "felis.lolicon.best/internal/fileedit"
"felis.lolicon.best/internal/maintenance"
"felis.lolicon.best/internal/naming" "felis.lolicon.best/internal/naming"
) )
@@ -149,6 +150,15 @@ func (a *API) handleWriteFile(w http.ResponseWriter, r *http.Request) {
return return
} }
// A write holds the world volume for its Job's lifetime (internal/maintenance);
// reads and listings do not, since a read-only mount cannot hurt a server
// starting beside it.
release, ok := a.acquireWorld(w, r, name, maintenance.KindFileWrite, "stop the server before editing its files")
if !ok {
return
}
defer release()
if err := a.Files.Write(r.Context(), name, path, *body.Content); err != nil { if err := a.Files.Write(r.Context(), name, path, *body.Content); err != nil {
writeFileEditError(w, r, err) writeFileEditError(w, r, err)
return return
+3 -3
View File
@@ -63,9 +63,9 @@ func mkFiles() (*API, *fakeRepo, *fakeCluster, *fakeFileEditor) {
} }
// TestFileEditorStoppedGate is the gate this whole subsystem hinges on. The world // TestFileEditorStoppedGate is the gate this whole subsystem hinges on. The world
// PVC is ReadWriteOnce, so a running server holds it and a file Job physically // PVC is ReadWriteOnce, but RWO is per node: on a single node a file Job mounts it
// cannot mount it — an ungated request would not fail cleanly, it would hang // right beside a running server, and a write lands under a live world that the
// waiting for a Pod that can never be scheduled. Every one of the three routes // server's next save overwrites or tears. Every one of the three routes
// must therefore refuse a non-stopped server with 409 not_stopped BEFORE reaching // must therefore refuse a non-stopped server with 409 not_stopped BEFORE reaching
// the executor, which is why each asserts calls == 0 as well as the status. // the executor, which is why each asserts calls == 0 as well as the status.
func TestFileEditorStoppedGate(t *testing.T) { func TestFileEditorStoppedGate(t *testing.T) {
+5 -1
View File
@@ -153,8 +153,10 @@ func (a *API) handleInternalWake(w http.ResponseWriter, r *http.Request) {
return return
} }
// A 409 maintenance_in_progress tells velocity nothing is coming up until the
// restore/backup/file write finishes, so it does not enqueue the player.
if err := a.Cluster.SetDesiredState(r.Context(), name, v1alpha1.DesiredRunning); err != nil { if err := a.Cluster.SetDesiredState(r.Context(), name, v1alpha1.DesiredRunning); err != nil {
writeError(w, r, err) a.writeLookupError(w, r, err)
return return
} }
// Consume the shared per-server cooldown only after the wake flips, so a join // Consume the shared per-server cooldown only after the wake flips, so a join
@@ -365,6 +367,8 @@ func (a *API) writeLookupError(w http.ResponseWriter, r *http.Request, err error
writeError(w, r, newError(http.StatusNotFound, "not_found", "not found")) writeError(w, r, newError(http.StatusNotFound, "not_found", "not found"))
case errors.Is(err, ErrConflict): case errors.Is(err, ErrConflict):
writeError(w, r, newError(http.StatusConflict, "conflict", "conflict")) writeError(w, r, newError(http.StatusConflict, "conflict", "conflict"))
case errors.Is(err, ErrMaintenanceInProgress), errors.Is(err, ErrNotStopped):
writeError(w, r, maintenanceError(err, "stop the server completely first"))
default: default:
writeError(w, r, err) writeError(w, r, err)
} }
+176
View File
@@ -0,0 +1,176 @@
package api
import (
"errors"
"fmt"
"net/http"
"net/http/httptest"
"slices"
"testing"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/maintenance"
)
// The world-volume lock (internal/maintenance) from the handlers' side: a wake
// refused because a restore/backup/file write holds the world, and those three
// operations refused while another one does. The lock itself (atomicity, stale
// locks, Job-backed holds) is K8sCluster's and is tested in k8scluster_test.
func TestWakeRefusedDuringMaintenance(t *testing.T) {
busy := &MaintenanceBusyError{Kind: maintenance.KindRestore}
t.Run("external wake -> 409 maintenance_in_progress, cooldown kept", func(t *testing.T) {
repo := newFakeRepo()
cl := newFakeCluster()
cl.byName["survival"] = &ServerInfo{Name: "survival", AutostartPolicy: "public"}
cl.wakeErr["survival"] = busy
api := newTestAPI(repo, cl)
api.External = staticExternal{p: &Principal{UserID: "u1", Role: "user"}}
w := do(api.ExternalHandler(), "POST", "/api/v1/servers/survival/wake", "", nil)
if w.Code != http.StatusConflict || decodeErr(t, w) != "maintenance_in_progress" {
t.Fatalf("code = %d body %s, want 409 maintenance_in_progress", w.Code, w.Body.String())
}
if _, set := cl.desired["survival"]; set {
t.Fatal("a refused wake must not flip desiredState")
}
// The refusal did not burn the cooldown: once the restore is done the very
// next wake goes through.
delete(cl.wakeErr, "survival")
if w := do(api.ExternalHandler(), "POST", "/api/v1/servers/survival/wake", "", nil); w.Code != http.StatusAccepted {
t.Fatalf("wake after maintenance: code = %d body %s", w.Code, w.Body.String())
}
})
t.Run("internal wake (velocity) -> 409 maintenance_in_progress, cooldown kept", func(t *testing.T) {
api, cl := newInternalWakeAPI("public")
cl.wakeErr["survival"] = busy
body := `{"mc_uuid":"` + wakeUUID + `"}`
w := internalWake(api, body)
if w.Code != http.StatusConflict || decodeErr(t, w) != "maintenance_in_progress" {
t.Fatalf("code = %d body %s, want 409 maintenance_in_progress", w.Code, w.Body.String())
}
delete(cl.wakeErr, "survival")
if w := internalWake(api, body); w.Code != http.StatusAccepted {
t.Fatalf("wake after maintenance: code = %d body %s", w.Code, w.Body.String())
}
})
}
// maintenanceOp is one world-volume operation as the external face serves it.
type maintenanceOp struct {
name string
kind string
method string
path string
body string
calls func() int
}
func maintenanceOps() (*API, *fakeCluster, []maintenanceOp) {
repo := newFakeRepo()
repo.byName["survival"] = &ServerRecord{Name: "survival", OwnerID: "owner1"}
repo.backups = []fakeBackup{{view: BackupView{ID: "bk1", ServerName: "survival",
FormerOwner: "owner1", Status: "present"}, ref: "world-archive-ref"}}
cl := newFakeCluster()
cl.byName["survival"] = &ServerInfo{Name: "survival", Phase: "Stopped",
DesiredState: string(v1alpha1.DesiredStopped)}
restorer, backuper, files := &fakeRestorer{}, &fakeBackuper{}, &fakeFileEditor{}
api := newTestAPI(repo, cl)
api.Restorer, api.Backuper, api.Files = restorer, backuper, files
api.External = staticExternal{p: &Principal{UserID: "owner1", Email: "[email protected]", Role: "user"}}
return api, cl, []maintenanceOp{
{"restore", maintenance.KindRestore, "POST", "/api/v1/servers/survival/restore-backup", "",
func() int { return restorer.calls }},
{"backup", maintenance.KindBackup, "POST", "/api/v1/servers/survival/backup", "",
func() int { return backuper.calls }},
{"file write", maintenance.KindFileWrite, "PUT", "/api/v1/servers/survival/file?path=server.properties",
`{"content":"aGk="}`, func() int { return files.calls }},
}
}
func (op maintenanceOp) do(api *API) *httptest.ResponseRecorder {
var hdr map[string]string
if op.body != "" {
hdr = jsonHeader
}
return do(api.ExternalHandler(), op.method, op.path, op.body, hdr)
}
func TestMaintenanceOpsTakeAndReleaseTheLock(t *testing.T) {
_, _, ops := maintenanceOps()
for i := range ops {
t.Run(ops[i].name, func(t *testing.T) {
api, cl, ops := maintenanceOps()
op := ops[i]
if w := op.do(api); w.Code/100 != 2 {
t.Fatalf("code = %d body %s", w.Code, w.Body.String())
}
if op.calls() != 1 {
t.Fatalf("executor calls = %d, want 1", op.calls())
}
if want := []string{"survival:" + op.kind}; !slices.Equal(cl.acquired, want) {
t.Fatalf("acquired = %v, want %v", cl.acquired, want)
}
if want := []string{"survival"}; !slices.Equal(cl.released, want) {
t.Fatalf("released = %v, want %v (the Job is the lock from here on)", cl.released, want)
}
})
}
}
func TestMaintenanceOpsRefusedWhileHeld(t *testing.T) {
refusals := []struct {
name string
err error
code string
}{
{"another holder", &MaintenanceBusyError{Kind: maintenance.KindBackup}, "maintenance_in_progress"},
// The snapshot said Stopped but the atomic re-check found it waking: the
// wake won the race.
{"server not stopped", fmt.Errorf("wrapped: %w", ErrNotStopped), "not_stopped"},
}
_, _, ops := maintenanceOps()
for i := range ops {
for _, rf := range refusals {
t.Run(ops[i].name+" / "+rf.name, func(t *testing.T) {
api, cl, ops := maintenanceOps()
op := ops[i]
cl.maintErr["survival"] = rf.err
w := op.do(api)
if w.Code != http.StatusConflict || decodeErr(t, w) != rf.code {
t.Fatalf("code = %d body %s, want 409 %s", w.Code, w.Body.String(), rf.code)
}
if op.calls() != 0 {
t.Fatal("a refused operation must not reach its executor")
}
if len(cl.released) != 0 {
t.Fatalf("released %v a lock that was never taken", cl.released)
}
})
}
}
}
func TestMaintenanceErrorMapping(t *testing.T) {
for _, tc := range []struct {
err error
code string
}{
{&MaintenanceBusyError{Kind: maintenance.KindFileWrite}, "maintenance_in_progress"},
{fmt.Errorf("x: %w", ErrMaintenanceInProgress), "maintenance_in_progress"},
{ErrNotStopped, "not_stopped"},
} {
var ae *apiError
if !errors.As(maintenanceError(tc.err, "stop it"), &ae) || ae.code != tc.code || ae.status != http.StatusConflict {
t.Errorf("%v -> %+v, want 409 %s", tc.err, ae, tc.code)
}
}
other := errors.New("boom")
if got := maintenanceError(other, "stop it"); got != other {
t.Errorf("unrelated error rewritten to %v", got)
}
}
+3 -1
View File
@@ -56,8 +56,10 @@ func (a *API) handleWake(w http.ResponseWriter, r *http.Request) {
return return
} }
// Refused with 409 maintenance_in_progress while a restore, backup or file
// write holds the world volume: starting on a half-written world corrupts it.
if err := a.Cluster.SetDesiredState(r.Context(), name, v1alpha1.DesiredRunning); err != nil { if err := a.Cluster.SetDesiredState(r.Context(), name, v1alpha1.DesiredRunning); err != nil {
writeError(w, r, err) a.writeLookupError(w, r, err)
return return
} }
// The wake actually flipped, so consume the per-server cooldown only now: a 503 // The wake actually flipped, so consume the per-server cooldown only now: a 503
+148 -5
View File
@@ -2,13 +2,18 @@ package api
import ( import (
"context" "context"
"errors"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/maintenance"
"felis.lolicon.best/internal/naming" "felis.lolicon.best/internal/naming"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
"sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client"
) )
@@ -20,6 +25,8 @@ import (
type K8sCluster struct { type K8sCluster struct {
c client.Client c client.Client
namespace string namespace string
// now is injectable for the maintenance-lock tests; nil means time.Now.
now func() time.Time
} }
// NewK8sCluster builds a Cluster over c, scoped to namespace. // NewK8sCluster builds a Cluster over c, scoped to namespace.
@@ -142,13 +149,15 @@ func (k *K8sCluster) CreateServer(ctx context.Context, in CreateServerInput) err
} }
// SetDesiredState patches spec.desiredState with a merge patch so concurrent // SetDesiredState patches spec.desiredState with a merge patch so concurrent
// status writes by the operator are never clobbered (spec §9.1). // status writes by the operator are never clobbered (spec §9.1). A stop always
// goes through. A start goes through start, which refuses while a maintenance
// operation holds the world volume.
func (k *K8sCluster) SetDesiredState(ctx context.Context, name string, state v1alpha1.DesiredState) error { func (k *K8sCluster) SetDesiredState(ctx context.Context, name string, state v1alpha1.DesiredState) error {
if state == v1alpha1.DesiredRunning {
return k.start(ctx, name)
}
var ms v1alpha1.MinecraftServer var ms v1alpha1.MinecraftServer
if err := k.c.Get(ctx, types.NamespacedName{Namespace: k.namespace, Name: name}, &ms); err != nil { if err := k.getServer(ctx, name, &ms); err != nil {
if apierrors.IsNotFound(err) {
return ErrNotFound
}
return err return err
} }
patch := client.MergeFrom(ms.DeepCopy()) patch := client.MergeFrom(ms.DeepCopy())
@@ -156,6 +165,140 @@ func (k *K8sCluster) SetDesiredState(ctx context.Context, name string, state v1a
return k.c.Patch(ctx, &ms, patch) return k.c.Patch(ctx, &ms, patch)
} }
// start flips desiredState to Running unless a restore, backup or file write
// holds the world volume (internal/maintenance), in which case it returns a
// *MaintenanceBusyError. The patch carries the resourceVersion it checked
// against, the same as AcquireMaintenance's: whichever of a racing wake and
// admission writes second gets a conflict, re-reads, and sees the other.
func (k *K8sCluster) start(ctx context.Context, name string) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
var ms v1alpha1.MinecraftServer
if err := k.getServer(ctx, name, &ms); err != nil {
return err
}
kind, held, err := k.maintenanceHolder(ctx, &ms)
if err != nil {
return err
}
if held {
return &MaintenanceBusyError{Kind: kind}
}
patch := client.MergeFromWithOptions(ms.DeepCopy(), client.MergeFromWithOptimisticLock{})
ms.Spec.DesiredState = v1alpha1.DesiredRunning
// A lock still on the object here no longer holds anything (Holder said
// so): drop it in the same write.
delete(ms.Annotations, maintenance.Annotation)
return k.c.Patch(ctx, &ms, patch)
})
}
// AcquireMaintenance admits one world-volume operation of the given kind: the
// server must be fully stopped (desiredState Stopped, phase Stopped, and no game
// pod left, so a pod still saving on its way down is waited out) and nothing else
// may hold the volume. Admission writes the maintenance lock under the resourceVersion it
// checked; the caller creates its Job and then calls ReleaseMaintenance, after
// which the Job itself is the lock.
func (k *K8sCluster) AcquireMaintenance(ctx context.Context, name, kind string) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
var ms v1alpha1.MinecraftServer
if err := k.getServer(ctx, name, &ms); err != nil {
return err
}
desired := ms.Spec.DesiredState
if desired == "" {
desired = v1alpha1.DesiredStopped
}
if desired != v1alpha1.DesiredStopped || ms.Status.Ready || ms.Status.Phase != v1alpha1.PhaseStopped {
return ErrNotStopped
}
if up, err := k.gamePodExists(ctx, name); err != nil {
return err
} else if up {
return ErrNotStopped
}
holder, held, err := k.maintenanceHolder(ctx, &ms)
if err != nil {
return err
}
if held {
return &MaintenanceBusyError{Kind: holder}
}
patch := client.MergeFromWithOptions(ms.DeepCopy(), client.MergeFromWithOptimisticLock{})
if ms.Annotations == nil {
ms.Annotations = map[string]string{}
}
ms.Annotations[maintenance.Annotation] = maintenance.LockValue(kind, k.clock())
return k.c.Patch(ctx, &ms, patch)
})
}
// ReleaseMaintenance drops the admission lock. It is called once the Job exists
// (or failed to be created); a lock that is never released stops holding after
// maintenance.Grace on its own.
func (k *K8sCluster) ReleaseMaintenance(ctx context.Context, name string) error {
var ms v1alpha1.MinecraftServer
if err := k.getServer(ctx, name, &ms); err != nil {
if errors.Is(err, ErrNotFound) {
return nil
}
return err
}
if _, ok := ms.Annotations[maintenance.Annotation]; !ok {
return nil
}
patch := client.MergeFrom(ms.DeepCopy())
delete(ms.Annotations, maintenance.Annotation)
return k.c.Patch(ctx, &ms, patch)
}
// maintenanceHolder reads what holds ms's world volume right now. The Jobs are
// listed through the same direct client as the object, so a Job created before
// the lock was released is always visible here.
func (k *K8sCluster) maintenanceHolder(ctx context.Context, ms *v1alpha1.MinecraftServer) (string, bool, error) {
var jobs batchv1.JobList
if err := k.c.List(ctx, &jobs, client.InNamespace(k.namespace),
client.MatchingLabels{maintenance.LabelServer: ms.Name}); err != nil {
return "", false, err
}
kind, held := maintenance.Holder(ms.Name, ms.Annotations, jobs.Items, k.clock())
return kind, held, nil
}
// gamePodComponent is the operator's component label value on a game server's
// pod (internal/operator.ComponentValue; k8scluster_test pins the two).
const gamePodComponent = "server"
// gamePodExists reports whether the server's game pod still exists, terminating
// or not. Phase Stopped is the operator's reading of the StatefulSet's replica
// counts; the pod object itself is the ground truth for "is anything of the
// server still running its preStop save against the volume".
func (k *K8sCluster) gamePodExists(ctx context.Context, name string) (bool, error) {
var pods corev1.PodList
if err := k.c.List(ctx, &pods, client.InNamespace(k.namespace), client.MatchingLabels{
v1alpha1.LabelServer: name, v1alpha1.LabelComponent: gamePodComponent,
}); err != nil {
return false, err
}
return len(pods.Items) > 0, nil
}
func (k *K8sCluster) getServer(ctx context.Context, name string, ms *v1alpha1.MinecraftServer) error {
if err := k.c.Get(ctx, types.NamespacedName{Namespace: k.namespace, Name: name}, ms); err != nil {
if apierrors.IsNotFound(err) {
return ErrNotFound
}
return err
}
return nil
}
func (k *K8sCluster) clock() time.Time {
if k.now != nil {
return k.now()
}
return time.Now()
}
// PatchServerSpec applies the admin-tier spec mutation (spec §7) with the same // PatchServerSpec applies the admin-tier spec mutation (spec §7) with the same
// merge-patch discipline as SetDesiredState: read, copy, mutate only the fields // merge-patch discipline as SetDesiredState: read, copy, mutate only the fields
// the admin set, patch — so an operator status write racing in parallel survives. // the admin set, patch — so an operator status write racing in parallel survives.
+237
View File
@@ -0,0 +1,237 @@
package api
import (
"context"
"errors"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/maintenance"
"felis.lolicon.best/internal/operator"
"felis.lolicon.best/internal/restore"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
// The world-volume lock against a fake API server. The fake client honours
// resourceVersion on an optimistic-lock patch, so the atomic half is exercised
// for real; what it cannot model is two felis-api replicas racing, which the
// resourceVersion check is precisely the defence against.
var lockNow = time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
func stoppedServer() *v1alpha1.MinecraftServer {
return &v1alpha1.MinecraftServer{
ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredStopped},
Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStopped},
}
}
func lockCluster(t *testing.T, objs ...client.Object) (*K8sCluster, client.Client) {
t.Helper()
scheme := runtime.NewScheme()
if err := clientgoscheme.AddToScheme(scheme); err != nil {
t.Fatalf("scheme: %v", err)
}
if err := v1alpha1.AddToScheme(scheme); err != nil {
t.Fatalf("scheme: %v", err)
}
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).
WithStatusSubresource(&v1alpha1.MinecraftServer{}).Build()
k := NewK8sCluster(c, "minecraft")
k.now = func() time.Time { return lockNow }
return k, c
}
func runningRestore(t *testing.T) *batchv1.Job {
t.Helper()
j, err := restore.RestoreJob(restore.JobParams{
Server: "survival", WorldPVC: "world-survival-0", BackupPVC: "felis-backups",
BackupRef: "/backups/survival/a.tar.gz", ArchiveStore: "tarLocal",
Namespace: "minecraft", ServiceAccount: "felis-restore", Image: "felis:1",
BackupRoot: "/backups", WorldsRoot: "/world", Deadline: time.Minute,
CPULimit: "1", MemLimit: "1Gi", TTLAfterFinished: time.Minute,
})
if err != nil {
t.Fatalf("RestoreJob: %v", err)
}
return j
}
func annotations(t *testing.T, c client.Client) (map[string]string, v1alpha1.DesiredState) {
t.Helper()
var ms v1alpha1.MinecraftServer
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &ms); err != nil {
t.Fatalf("get: %v", err)
}
return ms.Annotations, ms.Spec.DesiredState
}
func TestAcquireMaintenance(t *testing.T) {
ctx := context.Background()
t.Run("stopped and free -> lock written", func(t *testing.T) {
k, c := lockCluster(t, stoppedServer())
if err := k.AcquireMaintenance(ctx, "survival", maintenance.KindRestore); err != nil {
t.Fatalf("AcquireMaintenance: %v", err)
}
ann, _ := annotations(t, c)
if got, want := ann[maintenance.Annotation], maintenance.LockValue(maintenance.KindRestore, lockNow); got != want {
t.Fatalf("lock = %q, want %q", got, want)
}
// A second admission while the first holds (its Job not yet created) is refused.
var busy *MaintenanceBusyError
if err := k.AcquireMaintenance(ctx, "survival", maintenance.KindBackup); !errors.As(err, &busy) || busy.Kind != maintenance.KindRestore {
t.Fatalf("second admission: %v, want busy(restore)", err)
}
if err := k.ReleaseMaintenance(ctx, "survival"); err != nil {
t.Fatalf("ReleaseMaintenance: %v", err)
}
if ann, _ := annotations(t, c); ann[maintenance.Annotation] != "" {
t.Fatalf("lock survived release: %q", ann[maintenance.Annotation])
}
})
notStopped := []struct {
name string
mut func(*v1alpha1.MinecraftServer)
}{
{"desired Running", func(ms *v1alpha1.MinecraftServer) { ms.Spec.DesiredState = v1alpha1.DesiredRunning }},
{"still Stopping", func(ms *v1alpha1.MinecraftServer) { ms.Status.Phase = v1alpha1.PhaseStopping }},
{"Ready", func(ms *v1alpha1.MinecraftServer) { ms.Status.Ready = true }},
{"never reconciled", func(ms *v1alpha1.MinecraftServer) { ms.Status.Phase = "" }},
}
for _, tc := range notStopped {
t.Run(tc.name+" -> ErrNotStopped", func(t *testing.T) {
ms := stoppedServer()
tc.mut(ms)
k, c := lockCluster(t, ms)
if err := k.AcquireMaintenance(ctx, "survival", maintenance.KindRestore); !errors.Is(err, ErrNotStopped) {
t.Fatalf("err = %v, want ErrNotStopped", err)
}
if ann, _ := annotations(t, c); ann[maintenance.Annotation] != "" {
t.Fatal("a refused admission wrote the lock")
}
})
}
t.Run("game pod still terminating -> ErrNotStopped", func(t *testing.T) {
pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "survival-0", Namespace: "minecraft",
Labels: map[string]string{v1alpha1.LabelServer: "survival", v1alpha1.LabelComponent: gamePodComponent}}}
k, _ := lockCluster(t, stoppedServer(), pod)
if err := k.AcquireMaintenance(ctx, "survival", maintenance.KindBackup); !errors.Is(err, ErrNotStopped) {
t.Fatalf("err = %v, want ErrNotStopped", err)
}
})
t.Run("a Job's own pod is not the game pod", func(t *testing.T) {
pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "files-x", Namespace: "minecraft",
Labels: map[string]string{v1alpha1.LabelServer: "survival"}}}
k, _ := lockCluster(t, stoppedServer(), pod)
if err := k.AcquireMaintenance(ctx, "survival", maintenance.KindBackup); err != nil {
t.Fatalf("err = %v", err)
}
})
t.Run("running restore Job -> busy", func(t *testing.T) {
k, _ := lockCluster(t, stoppedServer(), runningRestore(t))
var busy *MaintenanceBusyError
if err := k.AcquireMaintenance(ctx, "survival", maintenance.KindFileWrite); !errors.As(err, &busy) || busy.Kind != maintenance.KindRestore {
t.Fatalf("err = %v, want busy(restore)", err)
}
})
t.Run("unknown server -> ErrNotFound", func(t *testing.T) {
k, _ := lockCluster(t)
if err := k.AcquireMaintenance(ctx, "survival", maintenance.KindRestore); !errors.Is(err, ErrNotFound) {
t.Fatalf("err = %v, want ErrNotFound", err)
}
if err := k.ReleaseMaintenance(ctx, "survival"); err != nil {
t.Fatalf("release on a deleted server: %v", err)
}
})
}
func TestStartRespectsMaintenance(t *testing.T) {
ctx := context.Background()
t.Run("fresh lock -> busy, desiredState untouched", func(t *testing.T) {
ms := stoppedServer()
ms.Annotations = map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindBackup, lockNow.Add(-10*time.Second))}
k, c := lockCluster(t, ms)
var busy *MaintenanceBusyError
if err := k.SetDesiredState(ctx, "survival", v1alpha1.DesiredRunning); !errors.As(err, &busy) || busy.Kind != maintenance.KindBackup {
t.Fatalf("err = %v, want busy(backup)", err)
}
if !errors.Is(&MaintenanceBusyError{}, ErrMaintenanceInProgress) {
t.Fatal("MaintenanceBusyError must match ErrMaintenanceInProgress")
}
if _, desired := annotations(t, c); desired != v1alpha1.DesiredStopped {
t.Fatalf("desiredState = %q, want Stopped", desired)
}
})
t.Run("running restore Job -> busy", func(t *testing.T) {
k, _ := lockCluster(t, stoppedServer(), runningRestore(t))
if err := k.SetDesiredState(ctx, "survival", v1alpha1.DesiredRunning); !errors.Is(err, ErrMaintenanceInProgress) {
t.Fatalf("err = %v, want maintenance in progress", err)
}
})
t.Run("finished restore Job -> starts", func(t *testing.T) {
j := runningRestore(t)
j.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
k, c := lockCluster(t, stoppedServer(), j)
if err := k.SetDesiredState(ctx, "survival", v1alpha1.DesiredRunning); err != nil {
t.Fatalf("err = %v", err)
}
if _, desired := annotations(t, c); desired != v1alpha1.DesiredRunning {
t.Fatalf("desiredState = %q, want Running", desired)
}
})
t.Run("stale lock -> starts and the lock is dropped", func(t *testing.T) {
ms := stoppedServer()
ms.Annotations = map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindRestore, lockNow.Add(-maintenance.Grace-time.Second))}
k, c := lockCluster(t, ms)
if err := k.SetDesiredState(ctx, "survival", v1alpha1.DesiredRunning); err != nil {
t.Fatalf("err = %v", err)
}
ann, desired := annotations(t, c)
if desired != v1alpha1.DesiredRunning {
t.Fatalf("desiredState = %q, want Running", desired)
}
if _, ok := ann[maintenance.Annotation]; ok {
t.Fatal("the stale lock was left behind")
}
})
t.Run("stop ignores the lock", func(t *testing.T) {
ms := stoppedServer()
ms.Spec.DesiredState = v1alpha1.DesiredRunning
ms.Annotations = map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindRestore, lockNow)}
k, c := lockCluster(t, ms)
if err := k.SetDesiredState(ctx, "survival", v1alpha1.DesiredStopped); err != nil {
t.Fatalf("err = %v", err)
}
if _, desired := annotations(t, c); desired != v1alpha1.DesiredStopped {
t.Fatalf("desiredState = %q, want Stopped", desired)
}
})
}
// gamePodComponent is a copy of the operator's pod label value; a drift would
// let a restore start beside a terminating server.
func TestGamePodComponentMatchesOperator(t *testing.T) {
if gamePodComponent != operator.ComponentValue {
t.Fatalf("gamePodComponent = %q, operator labels its pods %q", gamePodComponent, operator.ComponentValue)
}
}
+64
View File
@@ -0,0 +1,64 @@
package api
import (
"context"
"errors"
"log"
"net/http"
"time"
"felis.lolicon.best/internal/maintenance"
)
// maintenanceError maps the world-volume lock's refusals onto their 409s:
// maintenance_in_progress while a restore, backup or file write holds the volume,
// not_stopped (with the caller's wording) while the server is not fully down.
// Anything else passes through unchanged.
func maintenanceError(err error, notStopped string) error {
var busy *MaintenanceBusyError
switch {
case errors.As(err, &busy):
return newError(http.StatusConflict, "maintenance_in_progress",
"%s is running on this server's world; retry once it finishes", maintenanceLabel(busy.Kind))
case errors.Is(err, ErrMaintenanceInProgress):
return newError(http.StatusConflict, "maintenance_in_progress",
"another operation is running on this server's world; retry once it finishes")
case errors.Is(err, ErrNotStopped):
return newError(http.StatusConflict, "not_stopped", "%s", notStopped)
}
return err
}
func maintenanceLabel(kind string) string {
switch kind {
case maintenance.KindRestore:
return "a restore"
case maintenance.KindBackup:
return "a backup"
case maintenance.KindFileWrite:
return "a file write"
}
return "another operation"
}
// acquireWorld admits one world-volume operation of the given kind
// (internal/maintenance) and returns the release to run once its Job exists, or
// false after writing the refusal. notStopped is the not_stopped wording the
// calling face uses.
//
// The release runs on a context detached from the request: a client that hangs
// up the moment its 202 is written must not leave the lock behind to refuse the
// owner's next wake for maintenance.Grace.
func (a *API) acquireWorld(w http.ResponseWriter, r *http.Request, name, kind, notStopped string) (func(), bool) {
if err := a.Cluster.AcquireMaintenance(r.Context(), name, kind); err != nil {
a.writeLookupError(w, r, maintenanceError(err, notStopped))
return nil, false
}
return func() {
ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 10*time.Second)
defer cancel()
if err := a.Cluster.ReleaseMaintenance(ctx, name); err != nil {
log.Printf("api: release the maintenance lock on %s: %v (it lapses after %s)", name, err, maintenance.Grace)
}
}, true
}
+5 -1
View File
@@ -16,12 +16,15 @@ import (
// way. LabelOpID is the one addition: it is how felis-api finds THIS invocation's // way. LabelOpID is the one addition: it is how felis-api finds THIS invocation's
// Pod among any others, which matters here in a way it does not for restore — // Pod among any others, which matters here in a way it does not for restore —
// file operations are interactive and repeated, so several may be in flight or // file operations are interactive and repeated, so several may be in flight or
// lingering inside their TTL at once. // lingering inside their TTL at once. LabelMode carries the operation so the
// world-volume lock (internal/maintenance) can tell a write, which holds the
// volume, from a read or listing, which does not.
const ( const (
LabelManagedBy = "app.kubernetes.io/managed-by" LabelManagedBy = "app.kubernetes.io/managed-by"
LabelComponent = "app.kubernetes.io/component" LabelComponent = "app.kubernetes.io/component"
LabelServer = "felis.lolicon.best/server" LabelServer = "felis.lolicon.best/server"
LabelOpID = "felis.lolicon.best/files-op" LabelOpID = "felis.lolicon.best/files-op"
LabelMode = "felis.lolicon.best/files-mode"
managedByValue = "felis-files" managedByValue = "felis-files"
componentValue = "world-files" componentValue = "world-files"
@@ -80,6 +83,7 @@ func filesLabels(p JobParams) map[string]string {
LabelComponent: componentValue, LabelComponent: componentValue,
LabelServer: p.Server, LabelServer: p.Server,
LabelOpID: p.OpID, LabelOpID: p.OpID,
LabelMode: p.Op,
} }
} }
+141
View File
@@ -0,0 +1,141 @@
// Package maintenance is the per-server mutual exclusion between a game server
// and the Jobs that write or snapshot its world volume (restore, backup, file
// write). The world PVC is ReadWriteOnce, and RWO is exclusive per NODE: on a
// single-node cluster the game pod and a restore pod mount it side by side, so
// the access mode alone guards nothing. A server woken mid-restore boots on a
// half-extracted world and the restore then prunes what it wrote; a server woken
// mid-backup produces a torn archive that a later restore makes permanent.
//
// Two signals mark a volume as held:
//
// - an unfinished maintenance Job labelled for the server. Once the Job exists
// it IS the lock, for as long as it runs, however long that is.
// - the Annotation on the MinecraftServer, written by felis-api with an
// optimistic-lock patch before it creates the Job and removed right after.
// It bridges the gap between "admitted" and "the Job is visible", and it is
// what serialises admission against a wake: both write the same object under
// its resourceVersion, so one of two racing writers always loses with a
// conflict and re-checks.
//
// A lock older than Grace with no Job behind it is stale (felis-api died between
// the two writes) and holds nothing.
//
// File reads and listings are not holders. They mount the volume read-only for a
// second or two, and a server starting beside one cannot hurt either side, so
// nobody waits for them.
package maintenance
import (
"strings"
"time"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
)
const (
// Annotation is the admission lock on a MinecraftServer. Its value is
// "<kind>@<RFC3339 time>" (LockValue).
Annotation = "felis.lolicon.best/maintenance"
// Grace is how long a lock counts as held when no Job backs it. felis-api
// creates the Job within milliseconds of taking the lock and then drops it, so
// a lock this old means the process died in between.
Grace = 2 * time.Minute
// LabelServer / LabelManagedBy are the labels every maintenance executor puts
// on its Job (internal/restore, internal/backupjob, internal/fileedit keep
// their own copies; maintenance_test pins them against these).
LabelServer = "felis.lolicon.best/server"
LabelManagedBy = "app.kubernetes.io/managed-by"
// LabelFilesMode is the file-editor operation (list, read, write) a files Job
// performs. Only write holds the volume.
LabelFilesMode = "felis.lolicon.best/files-mode"
)
// Kinds of holder.
const (
KindRestore = "restore"
KindBackup = "backup"
KindFileWrite = "file-write"
)
// FilesModeWrite is the LabelFilesMode value of a file write.
const FilesModeWrite = "write"
// JobKind names the holder a Job represents, or reports false for a Job that
// holds nothing (a file read, a build, anything else in the namespace). A files
// Job without LabelFilesMode predates the label and is counted as a write: it can
// only be an old Job still inside its TTL, and over-counting it for that window
// is the safe side.
func JobKind(j *batchv1.Job) (string, bool) {
switch j.Labels[LabelManagedBy] {
case "felis-restore":
return KindRestore, true
case "felis-backup":
return KindBackup, true
case "felis-files":
mode, ok := j.Labels[LabelFilesMode]
if !ok || mode == FilesModeWrite {
return KindFileWrite, true
}
}
return "", false
}
// JobFinished reports whether a Job has reached a terminal condition. The
// success/failure-target conditions count as terminal: the Job controller sets
// them the moment the outcome is decided, before it finishes tearing the pods
// down, and waiting for Complete would keep a wake refused for no reason.
func JobFinished(j *batchv1.Job) bool {
for _, c := range j.Status.Conditions {
if c.Status != corev1.ConditionTrue {
continue
}
switch c.Type {
case batchv1.JobComplete, batchv1.JobFailed, batchv1.JobSuccessCriteriaMet, batchv1.JobFailureTarget:
return true
}
}
return false
}
// LockValue renders the Annotation value for a holder admitted at `at`.
func LockValue(kind string, at time.Time) string {
return kind + "@" + at.UTC().Format(time.RFC3339)
}
// parseLock splits a lock value. A value that does not parse is stale: a lock
// nobody can date must not be able to hold a server down forever.
func parseLock(v string) (string, time.Time, bool) {
kind, stamp, ok := strings.Cut(v, "@")
if !ok || kind == "" {
return "", time.Time{}, false
}
at, err := time.Parse(time.RFC3339, stamp)
if err != nil {
return "", time.Time{}, false
}
return kind, at, true
}
// Holder reports what, if anything, holds the server's world volume at `now`:
// the first unfinished maintenance Job among jobs, else a lock in annotations
// younger than Grace. jobs may contain unrelated Jobs; only the server's own
// holders count.
func Holder(server string, annotations map[string]string, jobs []batchv1.Job, now time.Time) (string, bool) {
for i := range jobs {
j := &jobs[i]
if j.Labels[LabelServer] != server || JobFinished(j) {
continue
}
if kind, ok := JobKind(j); ok {
return kind, true
}
}
if v, ok := annotations[Annotation]; ok {
if kind, at, ok := parseLock(v); ok && now.Sub(at) < Grace && at.Sub(now) < Grace {
return kind, true
}
}
return "", false
}
+150
View File
@@ -0,0 +1,150 @@
package maintenance
import (
"testing"
"time"
"felis.lolicon.best/internal/backupjob"
"felis.lolicon.best/internal/fileedit"
"felis.lolicon.best/internal/restore"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
)
var now = time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
func restoreJob(t *testing.T, server string) batchv1.Job {
t.Helper()
j, err := restore.RestoreJob(restore.JobParams{
Server: server, WorldPVC: "world-" + server + "-0", BackupPVC: "felis-backups",
BackupRef: "/backups/" + server + "/a.tar.gz", ArchiveStore: "tarLocal",
Namespace: "minecraft", ServiceAccount: "felis-restore", Image: "felis:1",
BackupRoot: "/backups", WorldsRoot: "/world", Deadline: time.Minute,
CPULimit: "1", MemLimit: "1Gi", TTLAfterFinished: time.Minute,
})
if err != nil {
t.Fatalf("RestoreJob: %v", err)
}
return *j
}
func backupJob(t *testing.T, server string) batchv1.Job {
t.Helper()
j, err := backupjob.BackupJob(backupjob.JobParams{
Server: server, FormerOwner: "u", WorldPVC: "world-" + server + "-0", BackupPVC: "felis-backups",
Namespace: "minecraft", ServiceAccount: "felis-restore", Image: "felis:1",
ConfigSecret: "felis-config", ConfigMount: "/etc/felis", BackupRoot: "/backups",
WorldsRoot: "/world", Deadline: time.Minute, CPULimit: "1", MemLimit: "1Gi",
TTLAfterFinished: time.Minute,
})
if err != nil {
t.Fatalf("BackupJob: %v", err)
}
return *j
}
func filesJob(t *testing.T, server, op string) batchv1.Job {
t.Helper()
p := fileedit.JobParams{
Server: server, OpID: "0011223344556677", Op: op, Path: "server.properties",
WorldPVC: "world-" + server + "-0", Namespace: "minecraft", ServiceAccount: "felis-restore",
Image: "felis:1", WorldsRoot: "/data", Deadline: time.Minute, CPULimit: "500m",
MemLimit: "256Mi", TTLAfterFinished: time.Minute,
}
if op == fileedit.OpWrite {
p.Content = []byte("motd=hi\n")
}
j, err := fileedit.FilesJob(p)
if err != nil {
t.Fatalf("FilesJob: %v", err)
}
return *j
}
func finished(j batchv1.Job, cond batchv1.JobConditionType) batchv1.Job {
j.Status.Conditions = append(j.Status.Conditions, batchv1.JobCondition{Type: cond, Status: corev1.ConditionTrue})
return j
}
// The executors keep their own label copies (they must not import each other or
// this package's callers); this pins what they actually render against JobKind.
func TestJobKindMatchesTheExecutors(t *testing.T) {
for _, tc := range []struct {
name string
job batchv1.Job
kind string
ok bool
}{
{"restore", restoreJob(t, "survival"), KindRestore, true},
{"backup", backupJob(t, "survival"), KindBackup, true},
{"file write", filesJob(t, "survival", fileedit.OpWrite), KindFileWrite, true},
{"file read", filesJob(t, "survival", fileedit.OpRead), "", false},
{"file list", filesJob(t, "survival", fileedit.OpList), "", false},
} {
kind, ok := JobKind(&tc.job)
if kind != tc.kind || ok != tc.ok {
t.Errorf("%s: JobKind = %q, %v; want %q, %v", tc.name, kind, ok, tc.kind, tc.ok)
}
if tc.job.Labels[LabelServer] != "survival" {
t.Errorf("%s: server label = %q", tc.name, tc.job.Labels[LabelServer])
}
}
}
func TestUnlabelledFilesJobCountsAsWrite(t *testing.T) {
j := filesJob(t, "survival", fileedit.OpRead)
delete(j.Labels, LabelFilesMode)
if kind, ok := JobKind(&j); !ok || kind != KindFileWrite {
t.Fatalf("a files Job from before the mode label = %q, %v; want file-write", kind, ok)
}
}
func TestHolderFromJobs(t *testing.T) {
restoreRunning := restoreJob(t, "survival")
if kind, ok := Holder("survival", nil, []batchv1.Job{restoreRunning}, now); !ok || kind != KindRestore {
t.Fatalf("running restore: Holder = %q, %v", kind, ok)
}
for _, cond := range []batchv1.JobConditionType{
batchv1.JobComplete, batchv1.JobFailed, batchv1.JobSuccessCriteriaMet, batchv1.JobFailureTarget,
} {
if kind, ok := Holder("survival", nil, []batchv1.Job{finished(restoreRunning, cond)}, now); ok {
t.Errorf("restore with %s still holds as %q", cond, kind)
}
}
other := backupJob(t, "creative")
if kind, ok := Holder("survival", nil, []batchv1.Job{other}, now); ok {
t.Fatalf("another server's backup holds survival as %q", kind)
}
read := filesJob(t, "survival", fileedit.OpRead)
if kind, ok := Holder("survival", nil, []batchv1.Job{read}, now); ok {
t.Fatalf("a running file read holds the volume as %q", kind)
}
write := filesJob(t, "survival", fileedit.OpWrite)
if kind, ok := Holder("survival", nil, []batchv1.Job{read, write}, now); !ok || kind != KindFileWrite {
t.Fatalf("running file write: Holder = %q, %v", kind, ok)
}
}
func TestHolderFromLock(t *testing.T) {
for _, tc := range []struct {
name string
value string
held bool
}{
{"fresh", LockValue(KindBackup, now.Add(-10*time.Second)), true},
{"just inside grace", LockValue(KindBackup, now.Add(-Grace+time.Second)), true},
{"stale", LockValue(KindBackup, now.Add(-Grace)), false},
{"far future", LockValue(KindBackup, now.Add(time.Hour)), false},
{"garbage", "yes", false},
{"no kind", "@" + now.Format(time.RFC3339), false},
{"bad time", "restore@yesterday", false},
} {
kind, held := Holder("survival", map[string]string{Annotation: tc.value}, nil, now)
if held != tc.held {
t.Errorf("%s (%q): held = %v, want %v", tc.name, tc.value, held, tc.held)
}
if held && kind != KindBackup {
t.Errorf("%s: kind = %q", tc.name, kind)
}
}
}
+108
View File
@@ -0,0 +1,108 @@
package operator_test
import (
"context"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/maintenance"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
func maintenanceJob(managedBy string, finished bool) *batchv1.Job {
j := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "restore-survival", Namespace: "minecraft",
Labels: map[string]string{maintenance.LabelServer: "survival", maintenance.LabelManagedBy: managedBy}}}
if finished {
j.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
}
return j
}
// A desiredState flipped to Running behind felis-api's back (kubectl, a script)
// while a restore Job is unpacking into the world must not get a game pod.
func TestReconcileRunning_HeldDuringMaintenance(t *testing.T) {
r, c := newReconciler(t, fakeProber{}, runningServer(), rconSecret(), maintenanceJob("felis-restore", false))
r.Jobs = c
res := reconcile(t, r, "survival")
if res.RequeueAfter <= 0 {
t.Fatal("a held server must requeue to notice the Job finishing")
}
var sts appsv1.StatefulSet
err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &sts)
if !apierrors.IsNotFound(err) {
t.Fatalf("StatefulSet created while the restore runs (err=%v)", err)
}
s := getServer(t, c, "survival")
cond := meta.FindStatusCondition(s.Status.Conditions, v1alpha1.ConditionReady)
if cond == nil || cond.Reason != "MaintenanceInProgress" {
t.Fatalf("Ready condition = %+v, want reason MaintenanceInProgress", cond)
}
if s.Status.StartRequestedAt != nil {
t.Fatal("the hold must not start the startup-timeout clock")
}
// The Job finishes: the next pass starts the server.
var j batchv1.Job
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "restore-survival"}, &j); err != nil {
t.Fatalf("get job: %v", err)
}
j.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
if err := c.Status().Update(context.Background(), &j); err != nil {
if err := c.Update(context.Background(), &j); err != nil {
t.Fatalf("finish job: %v", err)
}
}
reconcile(t, r, "survival")
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &sts); err != nil {
t.Fatalf("StatefulSet not created after the restore finished: %v", err)
}
}
func TestReconcileRunning_FreshLockHolds(t *testing.T) {
ms := runningServer()
ms.Annotations = map[string]string{maintenance.Annotation: maintenance.LockValue(maintenance.KindBackup, fixedNow().Add(-5*time.Second))}
r, c := newReconciler(t, fakeProber{}, ms, rconSecret())
r.Jobs = c
reconcile(t, r, "survival")
var sts appsv1.StatefulSet
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &sts); !apierrors.IsNotFound(err) {
t.Fatalf("StatefulSet created under a fresh maintenance lock (err=%v)", err)
}
}
func TestReconcileRunning_NotHeldByReadsOrFinishedJobs(t *testing.T) {
read := maintenanceJob("felis-files", false)
read.Name = "files-survival-read"
read.Labels[maintenance.LabelFilesMode] = "read"
r, c := newReconciler(t, fakeProber{}, runningServer(), rconSecret(), read, maintenanceJob("felis-backup", true))
r.Jobs = c
reconcile(t, r, "survival")
var sts appsv1.StatefulSet
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &sts); err != nil {
t.Fatalf("StatefulSet not created: %v", err)
}
}
// A server that is already up is never held: its pod is the one on the volume,
// and holding would only stall its readiness bookkeeping.
func TestReconcileRunning_RunningServerNotHeld(t *testing.T) {
r, c := newReconciler(t, fakeProber{}, runningServer(), rconSecret())
reconcile(t, r, "survival") // creates the StatefulSet at replicas 1
if err := c.Create(context.Background(), maintenanceJob("felis-restore", false)); err != nil {
t.Fatalf("create job: %v", err)
}
r.Jobs = c
reconcile(t, r, "survival")
cond := meta.FindStatusCondition(getServer(t, c, "survival").Status.Conditions, v1alpha1.ConditionReady)
if cond != nil && cond.Reason == "MaintenanceInProgress" {
t.Fatal("a scaled-up server was held")
}
}
+50
View File
@@ -13,8 +13,10 @@ import (
"time" "time"
"felis.lolicon.best/internal/apis/felis/v1alpha1" "felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/maintenance"
"felis.lolicon.best/internal/metrics" "felis.lolicon.best/internal/metrics"
appsv1 "k8s.io/api/apps/v1" appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors" apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/meta"
@@ -37,6 +39,7 @@ const (
requeueStopping = 5 * time.Second requeueStopping = 5 * time.Second
requeueSecret = 10 * time.Second requeueSecret = 10 * time.Second
requeueIdleProbe = 30 * time.Second requeueIdleProbe = 30 * time.Second
requeueMaintenance = 5 * time.Second
defaultTimeoutSeconds = 300 defaultTimeoutSeconds = 300
defaultReadinessTimeoutSec = 300 defaultReadinessTimeoutSec = 300
) )
@@ -69,6 +72,10 @@ type Reconciler struct {
// Now is injectable for deterministic timestamps in tests; defaults to // Now is injectable for deterministic timestamps in tests; defaults to
// metav1.Now. // metav1.Now.
Now func() metav1.Time Now func() metav1.Time
// Jobs reads the minecraft namespace's Jobs for the world-volume lock
// (internal/maintenance). It is the manager's uncached API reader, so the
// operator needs jobs:list and no Job informer. Nil skips the check.
Jobs client.Reader
} }
func (r *Reconciler) now() metav1.Time { func (r *Reconciler) now() metav1.Time {
@@ -111,6 +118,20 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
} }
func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.MinecraftServer) (ctrl.Result, error) { func (r *Reconciler) reconcileRunning(ctx context.Context, server *v1alpha1.MinecraftServer) (ctrl.Result, error) {
if kind, held, err := r.maintenanceHold(ctx, server); err != nil {
return ctrl.Result{}, err
} else if held {
// Leave phase and the start anchor alone: nothing is starting yet, and a
// long restore must not be charged to the startup timeout.
server.Status.ObservedGeneration = server.Generation
r.setCondition(server, v1alpha1.ConditionReady, metav1.ConditionFalse, "MaintenanceInProgress",
"waiting for the "+kind+" on this server's world to finish before starting")
if err := r.patchStatus(ctx, server); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: requeueMaintenance}, nil
}
endpointAddress, err := r.ensureServices(ctx, server) endpointAddress, err := r.ensureServices(ctx, server)
if err != nil { if err != nil {
return ctrl.Result{}, err return ctrl.Result{}, err
@@ -283,6 +304,35 @@ func (r *Reconciler) reconcileStopped(ctx context.Context, server *v1alpha1.Mine
return ctrl.Result{}, r.patchStatus(ctx, server) return ctrl.Result{}, r.patchStatus(ctx, server)
} }
// maintenanceHold reports whether a restore, backup or file write holds the
// server's world volume (internal/maintenance) while its pod is about to be
// created. felis-api already refuses a wake in that state; this is the same rule
// for a desiredState flipped by anything else (kubectl, a script), since a game
// pod scheduled beside a restore Job boots on a half-extracted world. A server
// whose StatefulSet is already scaled up is never held: its pod exists, and
// stopping it here would only lose the players on it.
func (r *Reconciler) maintenanceHold(ctx context.Context, server *v1alpha1.MinecraftServer) (string, bool, error) {
if r.Jobs == nil {
return "", false, nil
}
var sts appsv1.StatefulSet
err := r.Get(ctx, types.NamespacedName{Namespace: server.Namespace, Name: server.Name}, &sts)
switch {
case apierrors.IsNotFound(err):
case err != nil:
return "", false, err
case sts.Spec.Replicas == nil || *sts.Spec.Replicas > 0 || sts.Status.Replicas > 0:
return "", false, nil
}
var jobs batchv1.JobList
if err := r.Jobs.List(ctx, &jobs, client.InNamespace(server.Namespace),
client.MatchingLabels{maintenance.LabelServer: server.Name}); err != nil {
return "", false, err
}
kind, held := maintenance.Holder(server.Name, server.Annotations, jobs.Items, r.now().Time)
return kind, held, nil
}
func (r *Reconciler) ensureServices(ctx context.Context, server *v1alpha1.MinecraftServer) (string, error) { func (r *Reconciler) ensureServices(ctx context.Context, server *v1alpha1.MinecraftServer) (string, error) {
endpointAddress := "" endpointAddress := ""
for _, svc := range []*corev1.Service{buildHeadlessService(server), buildClientService(server)} { for _, svc := range []*corev1.Service{buildHeadlessService(server), buildClientService(server)} {
+8 -2
View File
@@ -145,8 +145,11 @@ func APIBuildRole(p Params) *rbacv1.Role {
// status (Status().Update — `update` only) and patches spec.desiredState to // status (Status().Update — `update` only) and patches spec.desiredState to
// Stopped for idle auto-stop (spec §8 — the one spec field it may write, using // Stopped for idle auto-stop (spec §8 — the one spec field it may write, using
// the same merge patch as the reaper's Stop: without the grant the auto-stop // the same merge patch as the reaper's Stop: without the grant the auto-stop
// call fails closed with a 403), and reads RCON Secrets. It never touches pods, // call fails closed with a 403), and reads RCON Secrets. Jobs are list-only,
// PVCs, Events, or finalizers, so none appear here. // through the manager's uncached API reader: before scaling a server up from zero
// the operator checks that no restore/backup/file-write Job holds its world
// (internal/maintenance). It never touches pods, PVCs, Events, or finalizers, so
// none appear here.
func OperatorRole(p Params) *rbacv1.Role { func OperatorRole(p Params) *rbacv1.Role {
p = p.withDefaults() p = p.withDefaults()
return role(p.MinecraftNamespace, "felis-operator", ComponentOperator, []rbacv1.PolicyRule{ return role(p.MinecraftNamespace, "felis-operator", ComponentOperator, []rbacv1.PolicyRule{
@@ -161,6 +164,9 @@ func OperatorRole(p Params) *rbacv1.Role {
// did not already have. No update/delete — the password is written once and // did not already have. No update/delete — the password is written once and
// removed by garbage collection through its controller reference. // removed by garbage collection through its controller reference.
rule([]string{groupCore}, []string{"secrets"}, []string{"get", "list", "watch", "create"}), rule([]string{groupCore}, []string{"secrets"}, []string{"get", "list", "watch", "create"}),
// list only: an uncached List (no informer, so no watch) of the world-volume
// maintenance Jobs; the operator never creates or deletes a Job.
rule([]string{groupBatch}, []string{"jobs"}, []string{"list"}),
}) })
} }
+9
View File
@@ -188,6 +188,15 @@ func TestOperatorRole_ScopeExact(t *testing.T) {
t.Errorf("operator must NOT touch core/%s", res) t.Errorf("operator must NOT touch core/%s", res)
} }
} }
// The world-volume lock check lists Jobs uncached; it never writes one.
if !hasRule(op, groupBatch, "jobs", "list") {
t.Error("operator must have jobs:list (maintenance hold before scale-up)")
}
for _, v := range []string{"get", "watch", "create", "update", "patch", "delete"} {
if hasRule(op, groupBatch, "jobs", v) {
t.Errorf("operator must NOT have jobs:%s (list-only)", v)
}
}
} }
// TestReaperRole_ScopeExact pins the reaper's two destructive powers and confirms // TestReaperRole_ScopeExact pins the reaper's two destructive powers and confirms
@@ -14,6 +14,7 @@
"console_unavailable": "Can't reach the server console right now — try again shortly.", "console_unavailable": "Can't reach the server console right now — try again shortly.",
"no_backup": "There's no restorable backup for this server yet.", "no_backup": "There's no restorable backup for this server yet.",
"not_stopped": "Stop the server completely before restoring — a restore overwrites the live world volume.", "not_stopped": "Stop the server completely before restoring — a restore overwrites the live world volume.",
"maintenance_in_progress": "This server's world is busy with a restore, backup or file write — try again once it finishes, usually within a minute or two.",
"no_world_volume": "This server has no world volume yet — start it once so it is created, then retry.", "no_world_volume": "This server has no world volume yet — start it once so it is created, then retry.",
"restore_unavailable": "Restore isn't available right now — try again later.", "restore_unavailable": "Restore isn't available right now — try again later.",
"session_expired": "Your session expired — please sign in again.", "session_expired": "Your session expired — please sign in again.",
@@ -14,6 +14,7 @@
"console_unavailable": "暂时无法连接服务器控制台,请稍后重试。", "console_unavailable": "暂时无法连接服务器控制台,请稍后重试。",
"no_backup": "这台服务器暂时没有可回档的备份。", "no_backup": "这台服务器暂时没有可回档的备份。",
"not_stopped": "回档会覆盖世界的实时存储卷,请先把服务器完全停止再回档。", "not_stopped": "回档会覆盖世界的实时存储卷,请先把服务器完全停止再回档。",
"maintenance_in_progress": "这台服务器的世界正在回档、备份或写入文件——等它完成后再试,通常不超过一两分钟。",
"no_world_volume": "这台服务器还没有世界卷——先启动一次让它创建,然后再试。", "no_world_volume": "这台服务器还没有世界卷——先启动一次让它创建,然后再试。",
"restore_unavailable": "回档功能当前不可用,请稍后再试。", "restore_unavailable": "回档功能当前不可用,请稍后再试。",
"session_expired": "会话已过期——请重新登录。", "session_expired": "会话已过期——请重新登录。",
+5
View File
@@ -705,6 +705,11 @@ export function humanizeError(e: unknown): string {
return t("no_backup"); return t("no_backup");
case "not_stopped": case "not_stopped":
return t("not_stopped"); return t("not_stopped");
// World-volume lock: a restore, backup or file write is running on this
// server's world, so a wake or a second world operation is refused until the
// Job finishes (internal/maintenance).
case "maintenance_in_progress":
return t("maintenance_in_progress");
case "no_world_volume": case "no_world_volume":
return t("no_world_volume"); return t("no_world_volume");
case "restore_unavailable": case "restore_unavailable":
@@ -79,7 +79,8 @@ public final class FelisApiClient {
* wake pulls the domain-autostart lever for {@code name} on behalf of the * wake pulls the domain-autostart lever for {@code name} on behalf of the
* joining player (spec §9.1, §14). The reply (202) carries the current phase * joining player (spec §9.1, §14). The reply (202) carries the current phase
* and ready flag so the caller can decide whether to wait. A 403 (policy gate), * and ready flag so the caller can decide whether to wait. A 403 (policy gate),
* 429 (cooldown), or 503 {@code at_capacity} (running cap) arrives as a * 409 {@code maintenance_in_progress} (a restore, backup or file write holds the
* world), 429 (cooldown), or 503 {@code at_capacity} (running cap) arrives as a
* LinkException the caller branches on. * LinkException the caller branches on.
*/ */
public ServerView wake(String name, UUID mcUuid) throws LinkException { public ServerView wake(String name, UUID mcUuid) throws LinkException {
@@ -498,6 +498,20 @@ public final class WaitingRouter {
zh ? "你无权启动「" + serverName + "」。" zh ? "你无权启动「" + serverName + "」。"
: "You're not allowed to start « " + serverName + " ».", NamedTextColor.RED)); : "You're not allowed to start « " + serverName + " ».", NamedTextColor.RED));
return; return;
case 409:
if ("maintenance_in_progress".equals(e.errorCode())) {
// A restore, backup or file write owns the world right now and
// the server will not start until it finishes (minutes at most),
// so waiting here would only run into the queue timeout.
player.sendMessage(Component.text(
zh ? "「" + serverName + "」正在维护(回档、备份或改文件),请稍后再试。"
: "« " + serverName + " » is under maintenance (restore, backup or file edit)."
+ " Please try again shortly.",
NamedTextColor.YELLOW));
return;
}
logWakeFailure(player, serverName, zh, e);
return;
case 429: case 429:
break; // a wake is already in flight → join the existing wait break; // a wake is already in flight → join the existing wait
case 503: case 503:
@@ -513,13 +527,10 @@ public final class WaitingRouter {
} }
// A 503 without the at_capacity code is a plain outage, not a // A 503 without the at_capacity code is a plain outage, not a
// capacity verdict — report it like any other failure. // capacity verdict — report it like any other failure.
// fall through logWakeFailure(player, serverName, zh, e);
return;
default: default:
log.warn("Felis: wake {} failed (status={}): {}", serverName, e.statusCode(), e.getMessage()); logWakeFailure(player, serverName, zh, e);
player.sendMessage(Component.text(
zh ? "现在无法启动「" + serverName + "」。请稍后再试。"
: "Couldn't start « " + serverName + " » right now. Try again shortly.",
NamedTextColor.RED));
return; return;
} }
} }
@@ -531,6 +542,14 @@ public final class WaitingRouter {
serverName, System.currentTimeMillis() + WAIT_TIMEOUT_MILLIS, fromMenu)); serverName, System.currentTimeMillis() + WAIT_TIMEOUT_MILLIS, fromMenu));
} }
private void logWakeFailure(Player player, String serverName, boolean zh, LinkException e) {
log.warn("Felis: wake {} failed (status={}): {}", serverName, e.statusCode(), e.getMessage());
player.sendMessage(Component.text(
zh ? "现在无法启动「" + serverName + "」。请稍后再试。"
: "Couldn't start « " + serverName + " » right now. Try again shortly.",
NamedTextColor.RED));
}
private void transfer(Player player, String serverName, RegisteredServer backend) { private void transfer(Player player, String serverName, RegisteredServer backend) {
player.createConnectionRequest(backend).connect().whenComplete((result, err) -> { player.createConnectionRequest(backend).connect().whenComplete((result, err) -> {
if (err != null || (result != null && !result.isSuccessful())) { if (err != null || (result != null && !result.isSuccessful())) {