Unverified Commit abfe60d6 authored by Lemon-miaow's avatar Lemon-miaow
Browse files

fix(api): wake 与回档/备份/改文件按服务器互斥

parent 7819e5de

cmd/felis/operator.go

+3 −0
Changes for cmd/felis/operator.go: 3 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -107,6 +107,9 @@ func cmdOperator(args []string, _, stderr io.Writer) int { 
		// injects into user servers. The Deployment passes it as FELIS_IMAGE (see

		// platform.OperatorDeployment); absent, that injection is simply skipped.

		FelisImage: os.Getenv("FELIS_IMAGE"),

		// Uncached: the maintenance-lock check lists Jobs only when a server is

		// about to start, which does not justify a namespace-wide Job informer.

		Jobs: mgr.GetAPIReader(),

	}

	if err := r.SetupWithManager(mgr); err != nil {

		fmt.Fprintf(stderr, "felis operator: setup controller: %v\n", err)

docs/openapi.yaml

+14 −4
Changes for docs/openapi.yaml: 14 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -741,6 +741,11 @@ paths: 
          $ref: '#/components/responses/Forbidden'

        '404':

          $ref: '#/components/responses/NotFound'

        '409':

          description: A restore, backup or file write holds the server's world volume (maintenance_in_progress); nothing was started.

          content:

            application/json:

              schema: { $ref: '#/components/schemas/Error' }

        '429':

          description: Wake cooldown is still active for this server.

          content:
@@ -1193,7 +1198,7 @@ paths: 
            application/json:

              schema: { $ref: '#/components/schemas/Error' }

        '409':

          description: Server is not stopped (its world PVC is still mounted).

          description: Server is not stopped (not_stopped), or a restore, backup or file write already holds its world volume (maintenance_in_progress).

          content:

            application/json:

              schema: { $ref: '#/components/schemas/Error' }
@@ -1228,6 +1233,11 @@ paths: 
          $ref: '#/components/responses/Forbidden'

        '404':

          $ref: '#/components/responses/NotFound'

        '409':

          description: A restore, backup or file write holds the server's world volume (maintenance_in_progress); nothing was started.

          content:

            application/json:

              schema: { $ref: '#/components/schemas/Error' }

        '429':

          description: Wake cooldown is still active.

          content:
@@ -2828,7 +2838,7 @@ paths: 
            application/json:

              schema: { $ref: '#/components/schemas/Error' }

        '409':

          description: Submission has already been reviewed.

          description: Server is not stopped (not_stopped), or a restore, backup or file write already holds its world volume (maintenance_in_progress).

          content:

            application/json:

              schema: { $ref: '#/components/schemas/Error' }
@@ -2873,7 +2883,7 @@ paths: 
            application/json:

              schema: { $ref: '#/components/schemas/Error' }

        '409':

          description: Server is not stopped (its world PVC is still mounted).

          description: Server is not stopped (not_stopped), or a restore, backup or file write already holds its world volume (maintenance_in_progress).

          content:

            application/json:

              schema: { $ref: '#/components/schemas/Error' }
@@ -3122,7 +3132,7 @@ paths: 
            application/json:

              schema: { $ref: '#/components/schemas/Error' }

        '409':

          description: Server is not stopped (its world PVC is still mounted).

          description: Server is not stopped (not_stopped), or a restore, backup or file write already holds its world volume (maintenance_in_progress).

          content:

            application/json:

              schema: { $ref: '#/components/schemas/Error' }

docs/troubleshooting.md

+47 −2
Changes for docs/troubleshooting.md: 47 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -185,6 +185,7 @@ per-server cooldown → global running cap**. Map the API result: 
| HTTP | Code | Cause | Fix |

|---|---|---|---|

| `403` | `forbidden` | `autostartPolicy=allowlist` and UUID not allowlisted, or `ownerOnly` and caller is not owner | Add the UUID / claim the server / set `autostartPolicy=public` |

| `409` | `maintenance_in_progress` | A restore, backup or file write holds the server's world volume (§3b) | Wait for the Job to finish |

| `429` | (cooldown) | Wake retried within the 30s per-server `WakeCooldown` | Wait out the cooldown |

| `503` | `at_capacity` | Global `MaxRunningServers` cap reached | Stop another server or raise the cap |
@@ -194,11 +195,54 @@ gate; the proxy polls `GET /api/v1/internal/servers/{name}/status` every ~2s and 
teleports when `ready=true`.



The Velocity-side consumption of these codes (`403` → "You're not allowed to

start «server»"; `429` → re-queue; other → "Couldn't start … Try again

start «server»"; `409 maintenance_in_progress` → "«server» is under

maintenance", not queued; `429` → re-queue; other → "Couldn't start … Try again

shortly.") lives in the Java plugin and is **[CODE-ONLY]** — the codes it reacts

to are produced by the Go-tested `authorizeWakeByUUID` / cooldown limiter, so

grade the two halves separately.



### 3b. Wake, restore, backup or file save refused with `maintenance_in_progress`



A server's world volume is ReadWriteOnce, and on a single node RWO lets a game

pod and a restore Job mount it side by side. So felis-api serialises them per

server: a restore, a backup, or a file write takes the world, and until its Job

finishes every wake (panel or join) and every other world operation on that

server gets `409 maintenance_in_progress`. File reads and listings never hold

it. The operator applies the same rule when `desiredState` is flipped to

`Running` by anything other than felis-api: the StatefulSet is not scaled up,

and the `Ready` condition reads `MaintenanceInProgress` until the Job ends.



What holds the world, in order:



1. An unfinished Job labelled `felis.lolicon.best/server=<name>` with

   `app.kubernetes.io/managed-by` `felis-restore`, `felis-backup`, or

   `felis-files` plus `felis.lolicon.best/files-mode=write`:



   ```sh

   kubectl -n minecraft get jobs -l felis.lolicon.best/server=<name>

   ```



   A Job that is genuinely wedged is ended by its own `activeDeadlineSeconds`;

   deleting it by hand releases the world at once (`kubectl -n minecraft delete

   job <job>`), at the cost of whatever it was writing.



2. The admission lock `felis.lolicon.best/maintenance=<kind>@<RFC3339>` on the

   MinecraftServer. felis-api sets it for the milliseconds between admitting an

   operation and creating its Job; it holds for at most two minutes if felis-api

   died in between, and the next wake clears a stale one. To drop it by hand:



   ```sh

   kubectl -n minecraft annotate minecraftserver <name> felis.lolicon.best/maintenance-

   ```



A restore, backup or file write refused with `409 not_stopped` although the

panel shows `Stopped` means the game pod is still terminating (its preStop save

can take a while); retry once `kubectl -n minecraft get pods -l

felis.lolicon.best/server=<name>` shows nothing.



[GO-TESTED: `internal/maintenance`, `k8scluster_maintenance_test.go`,

`handlers_maintenance_test.go`, operator `maintenance_test.go`.]



---



## 4. Routing is disabled even though servers are up (online-mode coupling)
@@ -947,7 +991,8 @@ installer built — only hand-built tags need a manual re-mirror. 
| RCON secret/auth/port errors | §1b, §1c |

| Phase `Failed` | §2 |

| Players land in lobby / wrong place | §3, §4 |

| Wake refused / rate-limited (403/429/503) | §3a |

| Wake refused / rate-limited (403/409/429/503) | §3a |

| `maintenance_in_progress`; server won't start after a restore | §3b |

| Routing disabled, offline-mode | §4 |

| Panel 401/403; fails-closed; audience error | §5 |

| Local password login rejected | §5c |

internal/api/api_test.go

+22 −1
Changes for internal/api/api_test.go: 22 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -1454,12 +1454,19 @@ type fakeCluster struct { 
	noWorld   map[string]bool              // server names modeled WITHOUT a world volume (never started / reaped)

	createErr error

	pingErr   error

	// maintErr / wakeErr: what AcquireMaintenance / SetDesiredState(Running)

	// return for a server (the world-volume lock, internal/maintenance).

	maintErr map[string]error

	wakeErr  map[string]error

	acquired []string // "name:kind" per admitted AcquireMaintenance

	released []string // names per ReleaseMaintenance

}



func newFakeCluster() *fakeCluster {

	return &fakeCluster{byName: map[string]*ServerInfo{}, bySub: map[string]*ServerInfo{},

		desired: map[string]v1alpha1.DesiredState{}, created: map[string]CreateServerInput{},

		patched: map[string]ServerSpecPatch{}, noWorld: map[string]bool{}}

		patched: map[string]ServerSpecPatch{}, noWorld: map[string]bool{},

		maintErr: map[string]error{}, wakeErr: map[string]error{}}

}

func (c *fakeCluster) GetServer(_ context.Context, n string) (*ServerInfo, error) {

	if s, ok := c.byName[n]; ok {
@@ -1483,9 +1490,23 @@ func (c *fakeCluster) WorldVolumeExists(_ context.Context, n string) (bool, erro 
}



func (c *fakeCluster) SetDesiredState(_ context.Context, n string, s v1alpha1.DesiredState) error {

	if err := c.wakeErr[n]; err != nil && s == v1alpha1.DesiredRunning {

		return err

	}

	c.desired[n] = s

	return nil

}

func (c *fakeCluster) AcquireMaintenance(_ context.Context, n, kind string) error {

	if err := c.maintErr[n]; err != nil {

		return err

	}

	c.acquired = append(c.acquired, n+":"+kind)

	return nil

}

func (c *fakeCluster) ReleaseMaintenance(_ context.Context, n string) error {

	c.released = append(c.released, n)

	return nil

}

func (c *fakeCluster) CreateServer(_ context.Context, in CreateServerInput) error {

	if c.createErr != nil {

		return c.createErr

internal/api/cluster.go

+11 −1
Changes for internal/api/cluster.go: 11 added lines, 1 removed line.
Original line number Diff line number Diff line
@@ -94,8 +94,18 @@ type Cluster interface { 
	// velocity registration pull (spec §7 GET /servers).

	ListServers(ctx context.Context) ([]ServerInfo, error)

	// SetDesiredState flips spec.desiredState — the only write the API performs

	// against the CRD (spec §9.1). It is idempotent.

	// against the CRD (spec §9.1). It is idempotent. Flipping to Running returns a

	// *MaintenanceBusyError (errors.Is ErrMaintenanceInProgress) while a restore,

	// backup or file write holds the world volume.

	SetDesiredState(ctx context.Context, name string, state v1alpha1.DesiredState) error

	// AcquireMaintenance admits one world-volume operation (internal/maintenance

	// kind): ErrNotStopped unless the server is fully stopped, a

	// *MaintenanceBusyError while another operation holds the volume. The check

	// and the lock are one atomic write against a concurrent wake.

	AcquireMaintenance(ctx context.Context, name, kind string) error

	// ReleaseMaintenance drops the admission lock once the operation's Job exists

	// (or could not be created). It is idempotent.

	ReleaseMaintenance(ctx context.Context, name string) error

	// CreateServer creates a MinecraftServer CRD from the validated form (spec

	// §15). It returns ErrConflict if a server of that name already exists.

	CreateServer(ctx context.Context, in CreateServerInput) error