fix(servers): 主人可放弃服务器、管理员可删除服务器,由 reaper 归档世界后释放或移除
This commit is contained in:
55 files changed
+2774
-119
No files matched your search
@@ -516,6 +516,11 @@ func (a *API) externalAPIRoutes() []apiRoute {
|
||||
// access routes above (handlers_allowlist.go).
|
||||
{Method: "GET", Pattern: "/api/v1/servers/{name}/allowlist", h: a.handleAllowlistList},
|
||||
{Method: "PUT", Pattern: "/api/v1/servers/{name}/allowlist/{uuid}", h: a.handleAllowlistSetWake},
|
||||
// Retirement: the owner gives the server up, or an admin deletes it. The
|
||||
// request is recorded and the reaper carries it out on its next run
|
||||
// (handlers_retire.go); owner/admin-gated inside the handlers.
|
||||
{Method: "PUT", Pattern: "/api/v1/servers/{name}/retirement", h: a.handleRetire},
|
||||
{Method: "DELETE", Pattern: "/api/v1/servers/{name}/retirement", h: a.handleCancelRetire},
|
||||
{Method: "GET", Pattern: "/api/v1/servers/{name}/status", h: a.handleStatus},
|
||||
// Identity self-read (spec §14 tiering): the panel reads this once at boot to
|
||||
// learn its own tier and decide which navigation surfaces to render. App-tier —
|
||||
|
||||
@@ -824,6 +824,30 @@ func (f *fakeRepo) SetAllowlistWake(_ context.Context, n, uuid string, canWake b
|
||||
}
|
||||
return ErrNotFound
|
||||
}
|
||||
// RequestRetire and CancelRetire mirror PGRepo's: the first request time is
|
||||
// kept, a deletion stays a deletion, and only an admin cancels a deletion.
|
||||
func (f *fakeRepo) RequestRetire(_ context.Context, n string, del bool) (RetireState, error) {
|
||||
rec, ok := f.byName[n]
|
||||
if !ok {
|
||||
return RetireState{}, ErrNotFound
|
||||
}
|
||||
if rec.Retire == nil {
|
||||
rec.Retire = &RetireState{RequestedAt: time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)}
|
||||
}
|
||||
rec.Retire.Delete = rec.Retire.Delete || del
|
||||
return *rec.Retire, nil
|
||||
}
|
||||
func (f *fakeRepo) CancelRetire(_ context.Context, n string, mayCancelDelete bool) error {
|
||||
rec, ok := f.byName[n]
|
||||
if !ok {
|
||||
return ErrNotFound
|
||||
}
|
||||
if rec.Retire != nil && rec.Retire.Delete && !mayCancelDelete {
|
||||
return ErrConflict
|
||||
}
|
||||
rec.Retire = nil
|
||||
return nil
|
||||
}
|
||||
func (f *fakeRepo) UserByMCUUID(_ context.Context, uuid string) (string, error) {
|
||||
if u, ok := f.links[uuid]; ok && !f.seededDead(u) {
|
||||
return u, nil
|
||||
|
||||
@@ -46,6 +46,13 @@ type ServerInfo struct {
|
||||
// restart backoff and may yet come up on its own.
|
||||
AutoRestarts int32 `json:"autoRestarts,omitempty"`
|
||||
StartGaveUp bool `json:"startGaveUp,omitempty"`
|
||||
// ReaperExempt marks a system server (spec.reaperExempt, the lobby): the
|
||||
// reaper never touches it, so it cannot be given up or deleted either.
|
||||
ReaperExempt bool `json:"reaperExempt,omitempty"`
|
||||
// Retiring is the pending request to give the server up or delete it. It is
|
||||
// business state from Postgres, joined onto the owner's and staff's view by
|
||||
// the status route; the cluster never sets it.
|
||||
Retiring *RetireState `json:"retiring,omitempty"`
|
||||
// Resources is the spec's pod resource block. It stays off the wire; a spec
|
||||
// patch reads it so the fields the admin left out keep their values.
|
||||
Resources corev1.ResourceRequirements `json:"-"`
|
||||
|
||||
@@ -152,6 +152,12 @@ func (a *API) handleInternalWake(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
// Given up or being deleted: it stays down until the reaper archives it, and
|
||||
// velocity tells the player so instead of queueing them.
|
||||
if rec != nil && rec.Retire != nil {
|
||||
writeError(w, r, errServerRetiring)
|
||||
return
|
||||
}
|
||||
// A start whose automatic restarts are spent (or that can never succeed as
|
||||
// configured) stays down until a person looks at it. The 202 this used to
|
||||
// return queued the player for a server nothing was starting. The join leaves
|
||||
@@ -305,7 +311,8 @@ func (a *API) handleInternalClaim(w http.ResponseWriter, r *http.Request) {
|
||||
// the claim call's, not the menu's). The lobby uses it purely to choose between
|
||||
// rendering `Claim & Start` (ownerless) and `Join`/`Wake` (owned). A server known
|
||||
// to the cluster but missing its servers-row is treated as ownerless, so it still
|
||||
// renders a sane tile rather than erroring.
|
||||
// renders a sane tile rather than erroring. A server being deleted is not claimable
|
||||
// even while it has no owner.
|
||||
func (a *API) handleInternalMenuStatus(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("name")
|
||||
if err := naming.ValidateServerName(name); err != nil {
|
||||
@@ -323,7 +330,7 @@ func (a *API) handleInternalMenuStatus(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
if rec != nil && rec.OwnerID != "" {
|
||||
if rec != nil && (rec.OwnerID != "" || rec.Retire != nil) {
|
||||
claimable = false
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
||||
"felis.lolicon.best/internal/naming"
|
||||
)
|
||||
|
||||
// A server leaves its owner, or the platform, through a retirement: the owner
|
||||
// gives it up, or an admin asks for it to be deleted. felis-api only records the
|
||||
// request and stops the server; the reaper, the one component that deletes a
|
||||
// world, carries it out on its next daily run. It archives the world (a
|
||||
// "released" backup kept for the reaper's retention, recorded against the owner),
|
||||
// deletes the world volume and releases the server for someone else to claim, and
|
||||
// for a deletion also removes the MinecraftServer and marks the servers row
|
||||
// deleted, which frees the name and subdomain. Until the reaper gets to it the
|
||||
// owner or an admin can cancel it; the server cannot be woken or claimed in the
|
||||
// meantime, so the world the reaper archives is the one the owner left.
|
||||
|
||||
// retireRequest is the PUT /servers/{name}/retirement body. Confirm must repeat
|
||||
// the server's name, the same typed confirmation the panel asks for, so a stray
|
||||
// or replayed call cannot give a world away.
|
||||
type retireRequest struct {
|
||||
Confirm string `json:"confirm"`
|
||||
Delete bool `json:"delete"`
|
||||
}
|
||||
|
||||
// errServerRetiring refuses a wake or claim of a server with a pending
|
||||
// retirement.
|
||||
var errServerRetiring = newError(http.StatusConflict, "server_retiring",
|
||||
"this server is being given up or deleted; cancel that first")
|
||||
|
||||
// retireServer resolves {name} for the retirement routes and applies their gate:
|
||||
// 400 for a malformed name, 404 for a server that does not exist, 403 for a caller
|
||||
// who neither owns it nor is an admin.
|
||||
func (a *API) retireServer(w http.ResponseWriter, r *http.Request) (*ServerRecord, bool) {
|
||||
name := r.PathValue("name")
|
||||
if err := naming.ValidateServerName(name); err != nil {
|
||||
writeError(w, r, newError(http.StatusBadRequest, "bad_name", "invalid server name: %v", err))
|
||||
return nil, false
|
||||
}
|
||||
rec, err := a.Repo.ServerByName(r.Context(), name)
|
||||
if err != nil {
|
||||
a.writeLookupError(w, r, err)
|
||||
return nil, false
|
||||
}
|
||||
if !a.isOwnerOrAdmin(principalFromContext(r.Context()), rec) {
|
||||
writeError(w, r, errForbidden)
|
||||
return nil, false
|
||||
}
|
||||
return rec, true
|
||||
}
|
||||
|
||||
// handleRetire records a retirement and stops the server. The owner may give the
|
||||
// server up; deleting it is an admin's call. A system server (reaperExempt, the
|
||||
// lobby) is never retired: the reaper would not touch it and the request would
|
||||
// sit forever. A deletion of a server whose MinecraftServer is already gone (one
|
||||
// removed with kubectl) is accepted, and the reaper then only marks its row.
|
||||
func (a *API) handleRetire(w http.ResponseWriter, r *http.Request) {
|
||||
rec, ok := a.retireServer(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
p := principalFromContext(r.Context())
|
||||
var req retireRequest
|
||||
if err := decodeJSON(w, r, &req); err != nil {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
if req.Delete && !p.IsAdmin() {
|
||||
writeError(w, r, newError(http.StatusForbidden, "forbidden", "only an administrator can delete a server"))
|
||||
return
|
||||
}
|
||||
if req.Confirm != rec.Name {
|
||||
writeError(w, r, newError(http.StatusBadRequest, "confirm_mismatch",
|
||||
"type the server's name to confirm"))
|
||||
return
|
||||
}
|
||||
|
||||
info, err := a.Cluster.GetServer(r.Context(), rec.Name)
|
||||
switch {
|
||||
case errors.Is(err, ErrNotFound) && req.Delete:
|
||||
// The StatefulSet retains its claim, so a MinecraftServer removed by hand
|
||||
// can leave the world volume behind, and the reaper deletes no world it
|
||||
// cannot hold still to archive. That one is an operator's to deal with.
|
||||
switch exists, err := a.Cluster.WorldVolumeExists(r.Context(), rec.Name); {
|
||||
case err != nil:
|
||||
writeError(w, r, err)
|
||||
return
|
||||
case exists:
|
||||
writeError(w, r, newError(http.StatusConflict, "world_volume_orphaned",
|
||||
"this server's MinecraftServer is gone but its world volume remains; archive and remove the volume by hand first"))
|
||||
return
|
||||
}
|
||||
info = nil
|
||||
case err != nil:
|
||||
a.writeLookupError(w, r, err)
|
||||
return
|
||||
case info.ReaperExempt:
|
||||
writeError(w, r, newError(http.StatusConflict, "system_server",
|
||||
"a system server cannot be given up or deleted"))
|
||||
return
|
||||
}
|
||||
// Stop first: a failed stop leaves nothing recorded, and a wake that lands
|
||||
// after the request is refused. The reaper stops the server again before it
|
||||
// touches the world, so one that slips in between costs only time.
|
||||
if info != nil && info.DesiredState != string(v1alpha1.DesiredStopped) {
|
||||
if err := a.Cluster.SetDesiredState(r.Context(), rec.Name, v1alpha1.DesiredStopped); err != nil {
|
||||
a.writeLookupError(w, r, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
st, err := a.Repo.RequestRetire(r.Context(), rec.Name, req.Delete)
|
||||
if err != nil {
|
||||
a.writeLookupError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
e := AuditEntry{Actor: auditActor(p), Action: "server.release", ServerName: rec.Name}
|
||||
if st.Delete {
|
||||
e.Action = "server.delete"
|
||||
}
|
||||
if p != nil {
|
||||
e.ActorUserID = p.UserID
|
||||
}
|
||||
if rec.OwnerID != "" {
|
||||
e.Payload = auditPayload(map[string]any{"owner_id": rec.OwnerID})
|
||||
}
|
||||
a.auditEntry(r, e)
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"name": rec.Name, "retiring": st})
|
||||
}
|
||||
|
||||
// handleCancelRetire drops a pending retirement. The owner may take back giving
|
||||
// the server up, but a deletion an admin asked for is the admin's to cancel. The
|
||||
// server stays stopped; its owner starts it again when they want it.
|
||||
func (a *API) handleCancelRetire(w http.ResponseWriter, r *http.Request) {
|
||||
rec, ok := a.retireServer(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
p := principalFromContext(r.Context())
|
||||
pending := rec.Retire != nil
|
||||
if err := a.Repo.CancelRetire(r.Context(), rec.Name, p.IsAdmin()); err != nil {
|
||||
if errors.Is(err, ErrConflict) {
|
||||
writeError(w, r, newError(http.StatusForbidden, "forbidden",
|
||||
"only an administrator can cancel the deletion of a server"))
|
||||
return
|
||||
}
|
||||
a.writeLookupError(w, r, err)
|
||||
return
|
||||
}
|
||||
if pending {
|
||||
a.audit(r, "server.retire_cancel", rec.Name)
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
||||
)
|
||||
|
||||
var (
|
||||
retireOwner = &Principal{UserID: "owner1", Email: "[email protected]", Role: "user"}
|
||||
retireAdmin = &Principal{UserID: "admin1", Role: "admin", ViaAdminAccess: true}
|
||||
retireStranger = &Principal{UserID: "other", Email: "[email protected]", Role: "user"}
|
||||
)
|
||||
|
||||
// retireAPI serves survival, owned by owner1 and running, to p.
|
||||
func retireAPI(p *Principal) (*API, *fakeRepo, *fakeCluster) {
|
||||
repo := newFakeRepo()
|
||||
repo.byName["survival"] = &ServerRecord{Name: "survival", OwnerID: "owner1"}
|
||||
cl := newFakeCluster()
|
||||
cl.byName["survival"] = &ServerInfo{Name: "survival", Phase: "Running", Ready: true,
|
||||
DesiredState: string(v1alpha1.DesiredRunning), AutostartPolicy: "public"}
|
||||
a := newTestAPI(repo, cl)
|
||||
a.External = staticExternal{p: p}
|
||||
return a, repo, cl
|
||||
}
|
||||
|
||||
func putRetire(a *API, body string) *httpResult {
|
||||
return result(do(a.ExternalHandler(), "PUT", "/api/v1/servers/survival/retirement", body, jsonHeader))
|
||||
}
|
||||
|
||||
func cancelRetire(a *API) *httpResult {
|
||||
return result(do(a.ExternalHandler(), "DELETE", "/api/v1/servers/survival/retirement", "", nil))
|
||||
}
|
||||
|
||||
// TestRetireRequest covers PUT /servers/{name}/retirement: the owner may give the
|
||||
// server up and an admin may delete it, both only with the name typed back; the
|
||||
// server is stopped and the request recorded for the reaper, and audited. Every
|
||||
// refusal leaves the server running and nothing recorded.
|
||||
func TestRetireRequest(t *testing.T) {
|
||||
t.Run("owner gives the server up", func(t *testing.T) {
|
||||
a, repo, cl := retireAPI(retireOwner)
|
||||
res := putRetire(a, `{"confirm":"survival"}`)
|
||||
if res.code != http.StatusAccepted {
|
||||
t.Fatalf("code = %d (%s)", res.code, res.body)
|
||||
}
|
||||
var got struct {
|
||||
Name string `json:"name"`
|
||||
Retiring RetireState `json:"retiring"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(res.body), &got); err != nil || got.Name != "survival" ||
|
||||
got.Retiring.Delete || got.Retiring.RequestedAt.IsZero() {
|
||||
t.Fatalf("body = %s (%v)", res.body, err)
|
||||
}
|
||||
if cl.desired["survival"] != v1alpha1.DesiredStopped {
|
||||
t.Fatalf("desiredState = %q, want Stopped", cl.desired["survival"])
|
||||
}
|
||||
if r := repo.byName["survival"].Retire; r == nil || r.Delete {
|
||||
t.Fatalf("recorded = %+v, want a release", r)
|
||||
}
|
||||
if len(repo.audits) != 1 || repo.audits[0].Action != "server.release" || repo.audits[0].ActorUserID != "owner1" ||
|
||||
!strings.Contains(string(repo.audits[0].Payload), `"owner_id":"owner1"`) {
|
||||
t.Fatalf("audits = %+v", repo.audits)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("admin deletes the server", func(t *testing.T) {
|
||||
a, repo, cl := retireAPI(retireAdmin)
|
||||
res := putRetire(a, `{"confirm":"survival","delete":true}`)
|
||||
if res.code != http.StatusAccepted || !strings.Contains(res.body, `"delete":true`) {
|
||||
t.Fatalf("code = %d (%s)", res.code, res.body)
|
||||
}
|
||||
if cl.desired["survival"] != v1alpha1.DesiredStopped {
|
||||
t.Fatalf("desiredState = %q, want Stopped", cl.desired["survival"])
|
||||
}
|
||||
if r := repo.byName["survival"].Retire; r == nil || !r.Delete {
|
||||
t.Fatalf("recorded = %+v, want a deletion", r)
|
||||
}
|
||||
if len(repo.audits) != 1 || repo.audits[0].Action != "server.delete" || repo.audits[0].ActorUserID != "admin1" {
|
||||
t.Fatalf("audits = %+v", repo.audits)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a server whose MinecraftServer is gone can still be deleted", func(t *testing.T) {
|
||||
a, repo, cl := retireAPI(retireAdmin)
|
||||
delete(cl.byName, "survival")
|
||||
if res := putRetire(a, `{"confirm":"survival","delete":true}`); res.code != http.StatusAccepted {
|
||||
t.Fatalf("code = %d (%s)", res.code, res.body)
|
||||
}
|
||||
if _, set := cl.desired["survival"]; set || repo.byName["survival"].Retire == nil {
|
||||
t.Fatalf("desired %v, recorded %+v", cl.desired, repo.byName["survival"].Retire)
|
||||
}
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
p *Principal
|
||||
body string
|
||||
setup func(*fakeRepo, *fakeCluster)
|
||||
code int
|
||||
err string
|
||||
}{
|
||||
{"owner may not delete", retireOwner, `{"confirm":"survival","delete":true}`, nil, http.StatusForbidden, "forbidden"},
|
||||
{"stranger", retireStranger, `{"confirm":"survival"}`, nil, http.StatusForbidden, "forbidden"},
|
||||
{"name not typed back", retireOwner, `{"confirm":"Survival"}`, nil, http.StatusBadRequest, "confirm_mismatch"},
|
||||
{"no confirmation", retireAdmin, `{"delete":true}`, nil, http.StatusBadRequest, "confirm_mismatch"},
|
||||
{"system server", retireAdmin, `{"confirm":"survival","delete":true}`,
|
||||
func(_ *fakeRepo, cl *fakeCluster) { cl.byName["survival"].ReaperExempt = true },
|
||||
http.StatusConflict, "system_server"},
|
||||
{"unknown server", retireAdmin, `{"confirm":"survival","delete":true}`,
|
||||
func(repo *fakeRepo, _ *fakeCluster) { delete(repo.byName, "survival") },
|
||||
http.StatusNotFound, "not_found"},
|
||||
{"release of a server with no MinecraftServer", retireAdmin, `{"confirm":"survival"}`,
|
||||
func(_ *fakeRepo, cl *fakeCluster) { delete(cl.byName, "survival") },
|
||||
http.StatusNotFound, "not_found"},
|
||||
{"world volume left without its server", retireAdmin, `{"confirm":"survival","delete":true}`,
|
||||
func(_ *fakeRepo, cl *fakeCluster) {
|
||||
delete(cl.byName, "survival")
|
||||
cl.orphanWorld = map[string]bool{"survival": true}
|
||||
},
|
||||
http.StatusConflict, "world_volume_orphaned"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a, repo, cl := retireAPI(tc.p)
|
||||
if tc.setup != nil {
|
||||
tc.setup(repo, cl)
|
||||
}
|
||||
res := putRetire(a, tc.body)
|
||||
if res.code != tc.code || res.errCode() != tc.err {
|
||||
t.Fatalf("code = %d %q, want %d %q (%s)", res.code, res.errCode(), tc.code, tc.err, res.body)
|
||||
}
|
||||
if _, set := cl.desired["survival"]; set {
|
||||
t.Fatal("a refused request stopped the server")
|
||||
}
|
||||
if rec := repo.byName["survival"]; rec != nil && rec.Retire != nil {
|
||||
t.Fatalf("a refused request was recorded: %+v", rec.Retire)
|
||||
}
|
||||
if len(repo.audits) != 0 {
|
||||
t.Fatalf("a refused request was audited: %+v", repo.audits)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireCancel covers DELETE /servers/{name}/retirement: the owner takes back
|
||||
// giving the server up, but only an admin cancels a deletion.
|
||||
func TestRetireCancel(t *testing.T) {
|
||||
pending := func(repo *fakeRepo, del bool) {
|
||||
repo.byName["survival"].Retire = &RetireState{RequestedAt: time.Now(), Delete: del}
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
p *Principal
|
||||
del bool
|
||||
code int
|
||||
cleared bool
|
||||
}{
|
||||
{"owner cancels giving it up", retireOwner, false, http.StatusNoContent, true},
|
||||
{"owner may not cancel a deletion", retireOwner, true, http.StatusForbidden, false},
|
||||
{"admin cancels a deletion", retireAdmin, true, http.StatusNoContent, true},
|
||||
{"stranger", retireStranger, false, http.StatusForbidden, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
a, repo, _ := retireAPI(tc.p)
|
||||
pending(repo, tc.del)
|
||||
res := cancelRetire(a)
|
||||
if res.code != tc.code {
|
||||
t.Fatalf("code = %d, want %d (%s)", res.code, tc.code, res.body)
|
||||
}
|
||||
if cleared := repo.byName["survival"].Retire == nil; cleared != tc.cleared {
|
||||
t.Fatalf("cleared = %v, want %v", cleared, tc.cleared)
|
||||
}
|
||||
audited := len(repo.audits) == 1 && repo.audits[0].Action == "server.retire_cancel"
|
||||
if audited != tc.cleared || (!tc.cleared && len(repo.audits) != 0) {
|
||||
t.Fatalf("audits = %+v", repo.audits)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("nothing pending is a quiet no-op", func(t *testing.T) {
|
||||
a, repo, _ := retireAPI(retireOwner)
|
||||
if res := cancelRetire(a); res.code != http.StatusNoContent || len(repo.audits) != 0 {
|
||||
t.Fatalf("code = %d, audits %+v", res.code, repo.audits)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// A server with a pending retirement stays as its owner left it until the reaper
|
||||
// archives it: no face wakes or claims it, the lobby does not offer it, and the
|
||||
// owner's and admin's views say it is going.
|
||||
func TestRetiringServerIsFrozen(t *testing.T) {
|
||||
t.Run("external wake", func(t *testing.T) {
|
||||
a, repo, cl := retireAPI(retireOwner)
|
||||
cl.byName["survival"].DesiredState = string(v1alpha1.DesiredStopped)
|
||||
cl.byName["survival"].Phase, cl.byName["survival"].Ready = "Stopped", false
|
||||
repo.byName["survival"].Retire = &RetireState{RequestedAt: time.Now()}
|
||||
res := result(do(a.ExternalHandler(), "POST", "/api/v1/servers/survival/wake", "", nil))
|
||||
if res.code != http.StatusConflict || res.errCode() != "server_retiring" {
|
||||
t.Fatalf("code = %d %q (%s)", res.code, res.errCode(), res.body)
|
||||
}
|
||||
if _, set := cl.desired["survival"]; set {
|
||||
t.Fatal("the wake went through")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("internal wake", func(t *testing.T) {
|
||||
a, repo, cl := retireAPI(nil)
|
||||
cl.byName["survival"].DesiredState = string(v1alpha1.DesiredStopped)
|
||||
cl.byName["survival"].Phase, cl.byName["survival"].Ready = "Stopped", false
|
||||
repo.byName["survival"].Retire = &RetireState{RequestedAt: time.Now()}
|
||||
res := result(internalWake(a, `{"mc_uuid":"`+wakeUUID+`"}`))
|
||||
if res.code != http.StatusConflict || res.errCode() != "server_retiring" {
|
||||
t.Fatalf("code = %d %q (%s)", res.code, res.errCode(), res.body)
|
||||
}
|
||||
if _, set := cl.desired["survival"]; set {
|
||||
t.Fatal("the wake went through")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("claim of an unowned server being deleted", func(t *testing.T) {
|
||||
a, repo, _ := retireAPI(retireStranger)
|
||||
repo.byName["survival"] = &ServerRecord{Name: "survival", Retire: &RetireState{RequestedAt: time.Now(), Delete: true}}
|
||||
repo.linked["other"], repo.quota["other"], repo.claimOK["survival"] = true, true, true
|
||||
res := result(do(a.ExternalHandler(), "POST", "/api/v1/servers/survival/claim", "", nil))
|
||||
if res.code != http.StatusConflict || res.errCode() != "server_retiring" {
|
||||
t.Fatalf("code = %d %q (%s)", res.code, res.errCode(), res.body)
|
||||
}
|
||||
if len(repo.audits) != 0 {
|
||||
t.Fatalf("audits = %+v", repo.audits)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("lobby menu", func(t *testing.T) {
|
||||
a, repo, _ := retireAPI(nil)
|
||||
repo.byName["survival"] = &ServerRecord{Name: "survival", Retire: &RetireState{RequestedAt: time.Now(), Delete: true}}
|
||||
res := result(internalMenu(a))
|
||||
if res.code != http.StatusOK || !strings.Contains(res.body, `"claimable":false`) {
|
||||
t.Fatalf("code = %d (%s)", res.code, res.body)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("status shows the request to the owner only", func(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
p *Principal
|
||||
sees bool
|
||||
}{{retireOwner, true}, {retireAdmin, true}, {retireStranger, false}} {
|
||||
a, repo, _ := retireAPI(tc.p)
|
||||
repo.byName["survival"].Retire = &RetireState{RequestedAt: time.Now()}
|
||||
res := result(do(a.ExternalHandler(), "GET", "/api/v1/servers/survival/status", "", nil))
|
||||
if res.code != http.StatusOK || strings.Contains(res.body, `"retiring"`) != tc.sees {
|
||||
t.Fatalf("%s: code = %d (%s)", tc.p.UserID, res.code, res.body)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fleet", func(t *testing.T) {
|
||||
a, repo, cl := retireAPI(retireAdmin)
|
||||
cl.list = []ServerInfo{{Name: "survival", Phase: "Stopped"}, {Name: "creative", Phase: "Stopped"}}
|
||||
repo.owners["survival"] = ServerOwnership{Retire: &RetireState{RequestedAt: time.Now(), Delete: true}}
|
||||
repo.owners["creative"] = ServerOwnership{}
|
||||
res := result(do(a.ExternalHandler(), "GET", "/api/v1/fleet", "", nil))
|
||||
var got struct {
|
||||
Servers []fleetServerView `json:"servers"`
|
||||
}
|
||||
if res.code != http.StatusOK || json.Unmarshal([]byte(res.body), &got) != nil || len(got.Servers) != 2 {
|
||||
t.Fatalf("code = %d (%s)", res.code, res.body)
|
||||
}
|
||||
if s := got.Servers[0]; s.Claimable || s.Retiring == nil || !s.Retiring.Delete {
|
||||
t.Fatalf("survival = %+v, want retiring and not claimable", s)
|
||||
}
|
||||
if s := got.Servers[1]; !s.Claimable || s.Retiring != nil {
|
||||
t.Fatalf("creative = %+v, want claimable", s)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type httpResult struct {
|
||||
code int
|
||||
body string
|
||||
}
|
||||
|
||||
func result(w *httptest.ResponseRecorder) *httpResult {
|
||||
return &httpResult{code: w.Code, body: w.Body.String()}
|
||||
}
|
||||
|
||||
// errCode is the error envelope's code, "" for a body that is not one.
|
||||
func (r *httpResult) errCode() string {
|
||||
var env struct {
|
||||
Error struct {
|
||||
Code string `json:"code"`
|
||||
} `json:"error"`
|
||||
}
|
||||
_ = json.Unmarshal([]byte(r.body), &env)
|
||||
return env.Error.Code
|
||||
}
|
||||
@@ -39,6 +39,12 @@ func (a *API) handleWake(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, err)
|
||||
return
|
||||
}
|
||||
// A server its owner gave up, or an admin is deleting, stays stopped until the
|
||||
// reaper archives it: a start would change the world it archives.
|
||||
if rec != nil && rec.Retire != nil {
|
||||
writeError(w, r, errServerRetiring)
|
||||
return
|
||||
}
|
||||
if !a.limiter().allowed(name, a.WakeCooldown) {
|
||||
writeError(w, r, newError(http.StatusTooManyRequests, "cooldown", "wake is cooling down, retry shortly"))
|
||||
return
|
||||
@@ -135,6 +141,12 @@ func (a *API) handleClaim(w http.ResponseWriter, r *http.Request) {
|
||||
"link your Minecraft account before claiming (see /api/v1/account/link/start)"))
|
||||
return
|
||||
}
|
||||
// A server with a pending deletion is not claimable (ClaimServer refuses it
|
||||
// too); saying why beats the 409 already_claimed that would otherwise explain it.
|
||||
if rec, err := a.Repo.ServerByName(r.Context(), name); err == nil && rec.Retire != nil {
|
||||
writeError(w, r, errServerRetiring)
|
||||
return
|
||||
}
|
||||
|
||||
// ② quota gate, evaluated before the ownership write. All four dimensions
|
||||
// (servers, CPU, memory, storage) are checked against the user's quota caps
|
||||
@@ -224,6 +236,11 @@ func (a *API) handleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
if !a.isOwnerOrAdmin(p, rec) {
|
||||
info = publicServerInfo(info)
|
||||
} else if rec != nil && rec.Retire != nil {
|
||||
// A pending retirement is Postgres state the cluster does not hold.
|
||||
withRetire := *info
|
||||
withRetire.Retiring = rec.Retire
|
||||
info = &withRetire
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, info)
|
||||
@@ -348,9 +365,10 @@ func (a *API) handleFleet(w http.ResponseWriter, r *http.Request) {
|
||||
for i, s := range servers {
|
||||
o, known := owners[s.Name]
|
||||
system := naming.IsSystemServer(s.Name)
|
||||
s.Retiring = o.Retire
|
||||
views[i] = fleetServerView{ServerInfo: s, Owner: o.Owner, System: system,
|
||||
Owned: o.OwnerID != "" && o.OwnerID == p.UserID,
|
||||
Claimable: known && o.OwnerID == "" && !system,
|
||||
Claimable: known && o.OwnerID == "" && o.Retire == nil && !system,
|
||||
OwnerUnknown: unknown}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"servers": views})
|
||||
@@ -369,8 +387,8 @@ type fleetServerView struct {
|
||||
// Owned is true when the caller claimed this server, decided by account id so
|
||||
// an owner without an email is still recognized.
|
||||
Owned bool `json:"owned"`
|
||||
// Claimable is true for a live, unclaimed, non-system server: the same rule
|
||||
// ClaimServer enforces. It is false whenever ownership is unknown.
|
||||
// Claimable is true for a live, unclaimed, non-system server with no pending
|
||||
// deletion: the same rule ClaimServer enforces. It is false whenever ownership is unknown.
|
||||
Claimable bool `json:"claimable"`
|
||||
// OwnerUnknown is true when the best-effort owner lookup failed, so an empty
|
||||
// Owner says nothing about whether the server is claimed.
|
||||
|
||||
@@ -468,6 +468,7 @@ func serverInfo(ms *v1alpha1.MinecraftServer) *ServerInfo {
|
||||
LegacyForwarding: ms.Labels[v1alpha1.LabelForwarding] == v1alpha1.ForwardingLegacy,
|
||||
AutoRestarts: ms.Status.AutoRestarts,
|
||||
StartGaveUp: v1alpha1.StartGaveUp(&ms.Status),
|
||||
ReaperExempt: ms.Spec.ReaperExempt,
|
||||
Resources: *ms.Spec.Resources.DeepCopy(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -323,6 +323,27 @@ func TestServerInfoCarriesStartGaveUp(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A system server reaches the panel marked, so the console offers no give-up or
|
||||
// delete the API would refuse; every other server leaves the field out.
|
||||
func TestServerInfoCarriesReaperExempt(t *testing.T) {
|
||||
lobby := testServer("lobby", "lobby")
|
||||
lobby.Spec.ReaperExempt = true
|
||||
for _, tc := range []struct {
|
||||
ms *v1alpha1.MinecraftServer
|
||||
want bool
|
||||
}{{lobby, true}, {testServer("plain", "plain"), false}} {
|
||||
info := serverInfo(tc.ms)
|
||||
b, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.ReaperExempt != tc.want || strings.Contains(string(b), `"reaperExempt":true`) != tc.want ||
|
||||
(!tc.want && strings.Contains(string(b), "reaperExempt")) {
|
||||
t.Errorf("%s: reaperExempt = %v, JSON %s; want %v", tc.ms.Name, info.ReaperExempt, b, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An admin edits one resource at a time. The view hands the handler the whole
|
||||
// pod block, the handler lays the change over it, and the merge patch keeps
|
||||
// everything the admin left alone: memory-only keeps the CPU limit, CPU-only
|
||||
|
||||
+104
-14
@@ -21,33 +21,49 @@ func NewPGRepo(db *sql.DB) *PGRepo { return &PGRepo{db: db} }
|
||||
func (p *PGRepo) Ping(ctx context.Context) error { return p.db.PingContext(ctx) }
|
||||
|
||||
func (p *PGRepo) ServerBySubdomain(ctx context.Context, subdomain string) (*ServerRecord, error) {
|
||||
const q = `SELECT s.name, sa.subdomain, COALESCE(s.owner_id, ''), COALESCE(s.cached_phase, '')
|
||||
const q = `SELECT s.name, sa.subdomain, COALESCE(s.owner_id, ''), COALESCE(s.cached_phase, ''),
|
||||
s.retire_requested_at, s.retire_delete
|
||||
FROM server_aliases sa JOIN servers s ON s.name = sa.server_name
|
||||
WHERE sa.subdomain = $1 AND s.deleted_at IS NULL`
|
||||
var r ServerRecord
|
||||
switch err := p.db.QueryRowContext(ctx, q, subdomain).Scan(&r.Name, &r.Subdomain, &r.OwnerID, &r.CachedPhase); {
|
||||
var retireAt sql.NullTime
|
||||
var retireDelete bool
|
||||
switch err := p.db.QueryRowContext(ctx, q, subdomain).Scan(&r.Name, &r.Subdomain, &r.OwnerID, &r.CachedPhase, &retireAt, &retireDelete); {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
return nil, ErrNotFound
|
||||
case err != nil:
|
||||
return nil, err
|
||||
}
|
||||
r.Retire = retireState(retireAt, retireDelete)
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func (p *PGRepo) ServerByName(ctx context.Context, name string) (*ServerRecord, error) {
|
||||
const q = `SELECT s.name, COALESCE(sa.subdomain, ''), COALESCE(s.owner_id, ''), COALESCE(s.cached_phase, '')
|
||||
const q = `SELECT s.name, COALESCE(sa.subdomain, ''), COALESCE(s.owner_id, ''), COALESCE(s.cached_phase, ''),
|
||||
s.retire_requested_at, s.retire_delete
|
||||
FROM servers s LEFT JOIN server_aliases sa ON sa.server_name = s.name
|
||||
WHERE s.name = $1 AND s.deleted_at IS NULL`
|
||||
var r ServerRecord
|
||||
switch err := p.db.QueryRowContext(ctx, q, name).Scan(&r.Name, &r.Subdomain, &r.OwnerID, &r.CachedPhase); {
|
||||
var retireAt sql.NullTime
|
||||
var retireDelete bool
|
||||
switch err := p.db.QueryRowContext(ctx, q, name).Scan(&r.Name, &r.Subdomain, &r.OwnerID, &r.CachedPhase, &retireAt, &retireDelete); {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
return nil, ErrNotFound
|
||||
case err != nil:
|
||||
return nil, err
|
||||
}
|
||||
r.Retire = retireState(retireAt, retireDelete)
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// retireState is a servers row's pending retirement, nil when there is none.
|
||||
func retireState(at sql.NullTime, deleteServer bool) *RetireState {
|
||||
if !at.Valid {
|
||||
return nil
|
||||
}
|
||||
return &RetireState{RequestedAt: at.Time, Delete: deleteServer}
|
||||
}
|
||||
|
||||
func (p *PGRepo) IsLinked(ctx context.Context, userID string) (bool, error) {
|
||||
var ok bool
|
||||
err := p.db.QueryRowContext(ctx,
|
||||
@@ -443,11 +459,12 @@ func (p *PGRepo) ClaimServer(ctx context.Context, name, userID string) (bool, er
|
||||
}
|
||||
|
||||
var owned sql.NullString
|
||||
var retiring bool
|
||||
var cpu, mem, stor int
|
||||
switch err := tx.QueryRowContext(ctx,
|
||||
`SELECT owner_id, cached_cpu_milli, cached_memory_mb, cached_storage_mb
|
||||
`SELECT owner_id, retire_requested_at IS NOT NULL, cached_cpu_milli, cached_memory_mb, cached_storage_mb
|
||||
FROM servers WHERE name = $1 AND deleted_at IS NULL FOR UPDATE`,
|
||||
name).Scan(&owned, &cpu, &mem, &stor); {
|
||||
name).Scan(&owned, &retiring, &cpu, &mem, &stor); {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
return false, ErrNotFound
|
||||
case err != nil:
|
||||
@@ -456,6 +473,12 @@ func (p *PGRepo) ClaimServer(ctx context.Context, name, userID string) (bool, er
|
||||
if owned.Valid {
|
||||
return false, nil // already claimed → 409 at the handler
|
||||
}
|
||||
// An unowned server an admin is releasing or deleting: its world is about to
|
||||
// be archived and deleted, or the server itself removed. The handler refuses
|
||||
// it up front; this holds against a request that lands in between.
|
||||
if retiring {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// The four-dimension gate, re-run inside the transaction. A missing quota
|
||||
// row leaves every NullInt64 invalid → quotaAllows treats each dimension as
|
||||
@@ -481,7 +504,7 @@ func (p *PGRepo) ClaimServer(ctx context.Context, name, userID string) (bool, er
|
||||
|
||||
res, err := tx.ExecContext(ctx,
|
||||
`UPDATE servers SET owner_id = $2, claimed_at = now(), last_active_at = now(), warned_3d_at = NULL, warned_1d_at = NULL
|
||||
WHERE name = $1 AND owner_id IS NULL AND deleted_at IS NULL`,
|
||||
WHERE name = $1 AND owner_id IS NULL AND deleted_at IS NULL AND retire_requested_at IS NULL`,
|
||||
name, userID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
@@ -597,6 +620,54 @@ func (p *PGRepo) SetAllowlistWake(ctx context.Context, name, mcUUID string, canW
|
||||
return nil
|
||||
}
|
||||
|
||||
// RequestRetire records the server's retirement for the reaper to carry out.
|
||||
// Asking again keeps the first request time, so the panel's "since" stays true,
|
||||
// and a deletion once asked for is not turned back into a release by the owner
|
||||
// asking too.
|
||||
func (p *PGRepo) RequestRetire(ctx context.Context, name string, deleteServer bool) (RetireState, error) {
|
||||
var st RetireState
|
||||
switch err := p.db.QueryRowContext(ctx,
|
||||
`UPDATE servers
|
||||
SET retire_requested_at = COALESCE(retire_requested_at, now()),
|
||||
retire_delete = retire_delete OR $2
|
||||
WHERE name = $1 AND deleted_at IS NULL
|
||||
RETURNING retire_requested_at, retire_delete`, name, deleteServer).Scan(&st.RequestedAt, &st.Delete); {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
return RetireState{}, ErrNotFound
|
||||
case err != nil:
|
||||
return RetireState{}, err
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// CancelRetire drops the server's pending retirement. The row is locked while
|
||||
// the deletion flag is read, so an owner cannot cancel a deletion an admin asked
|
||||
// for in between.
|
||||
func (p *PGRepo) CancelRetire(ctx context.Context, name string, mayCancelDelete bool) error {
|
||||
tx, err := p.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback() //nolint:errcheck // no-op after commit
|
||||
var deleteServer bool
|
||||
switch err := tx.QueryRowContext(ctx,
|
||||
`SELECT retire_delete FROM servers WHERE name = $1 AND deleted_at IS NULL FOR UPDATE`,
|
||||
name).Scan(&deleteServer); {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
return ErrNotFound
|
||||
case err != nil:
|
||||
return err
|
||||
}
|
||||
if deleteServer && !mayCancelDelete {
|
||||
return ErrConflict
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx,
|
||||
`UPDATE servers SET retire_requested_at = NULL, retire_delete = false WHERE name = $1`, name); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// UserByMCUUID resolves a verified in-game UUID to its linked user_id (spec §10
|
||||
// account_links), or ErrNotFound when the UUID is not linked to any account. A
|
||||
// link whose account is dead reads the same as no link at all (audit #33), so the
|
||||
@@ -647,9 +718,13 @@ func (p *PGRepo) RecordJoin(ctx context.Context, name, mcUUID string) error {
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// MyServers lists the servers the user owns, each with its pending retirement,
|
||||
// and the unowned ones they may claim. An unowned server an admin is releasing or
|
||||
// deleting is listed but not claimable.
|
||||
func (p *PGRepo) MyServers(ctx context.Context, userID string) ([]MyServerView, error) {
|
||||
const q = `SELECT s.name, COALESCE(sa.subdomain, ''),
|
||||
COALESCE(s.owner_id = $1, false) AS owned, (s.owner_id IS NULL) AS claimable, COALESCE(s.cached_phase, '')
|
||||
COALESCE(s.owner_id = $1, false) AS owned, (s.owner_id IS NULL AND s.retire_requested_at IS NULL) AS claimable,
|
||||
COALESCE(s.cached_phase, ''), s.retire_requested_at, s.retire_delete
|
||||
FROM servers s LEFT JOIN server_aliases sa ON sa.server_name = s.name
|
||||
WHERE s.deleted_at IS NULL AND (s.owner_id = $1 OR s.owner_id IS NULL)
|
||||
ORDER BY s.name`
|
||||
@@ -661,9 +736,14 @@ func (p *PGRepo) MyServers(ctx context.Context, userID string) ([]MyServerView,
|
||||
var out []MyServerView
|
||||
for rows.Next() {
|
||||
var v MyServerView
|
||||
if err := rows.Scan(&v.Name, &v.Subdomain, &v.Owned, &v.Claimable, &v.Phase); err != nil {
|
||||
var retireAt sql.NullTime
|
||||
var retireDelete bool
|
||||
if err := rows.Scan(&v.Name, &v.Subdomain, &v.Owned, &v.Claimable, &v.Phase, &retireAt, &retireDelete); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v.Owned {
|
||||
v.Retiring = retireState(retireAt, retireDelete)
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
@@ -676,7 +756,8 @@ func (p *PGRepo) MyServers(ctx context.Context, userID string) ([]MyServerView,
|
||||
// log records as the human actor, §6) and falls back to the never-NULL username
|
||||
// when the address is absent.
|
||||
func (p *PGRepo) ServerOwners(ctx context.Context) (map[string]ServerOwnership, error) {
|
||||
const q = `SELECT s.name, COALESCE(s.owner_id, ''), COALESCE(NULLIF(u.email, ''), u.username, '')
|
||||
const q = `SELECT s.name, COALESCE(s.owner_id, ''), COALESCE(NULLIF(u.email, ''), u.username, ''),
|
||||
s.retire_requested_at, s.retire_delete
|
||||
FROM servers s LEFT JOIN users u ON u.id = s.owner_id
|
||||
WHERE s.deleted_at IS NULL`
|
||||
rows, err := p.db.QueryContext(ctx, q)
|
||||
@@ -688,9 +769,12 @@ func (p *PGRepo) ServerOwners(ctx context.Context) (map[string]ServerOwnership,
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var o ServerOwnership
|
||||
if err := rows.Scan(&name, &o.OwnerID, &o.Owner); err != nil {
|
||||
var retireAt sql.NullTime
|
||||
var retireDelete bool
|
||||
if err := rows.Scan(&name, &o.OwnerID, &o.Owner, &retireAt, &retireDelete); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o.Retire = retireState(retireAt, retireDelete)
|
||||
out[name] = o
|
||||
}
|
||||
return out, rows.Err()
|
||||
@@ -701,9 +785,11 @@ func (p *PGRepo) ServerOwners(ctx context.Context) (map[string]ServerOwnership,
|
||||
// claimed later, spec §9.3) and its subdomain alias. The create handler has
|
||||
// already found no server and no world volume of this name, so a row that is
|
||||
// here belongs to an earlier server of the same name: one removed with kubectl,
|
||||
// or a create whose CRD write failed. That row starts over, and the earlier
|
||||
// a create whose CRD write failed, or one the reaper deleted between removing
|
||||
// its MinecraftServer and marking the row. That row starts over, and the earlier
|
||||
// server's other aliases and allowlist go with it; nothing of its owner, claim,
|
||||
// activity clock or reaper warnings reaches the new server. A retried create
|
||||
// activity clock, reaper warnings or pending retirement reaches the new server,
|
||||
// so the reaper's unfinished deletion no longer applies to it. A retried create
|
||||
// lands on the same fresh state. The alias subdomain is a PRIMARY KEY: bound to
|
||||
// another server, it rolls the whole seed back and returns ErrConflict, letting
|
||||
// the create handler answer 409 before it touches the CRD.
|
||||
@@ -722,6 +808,7 @@ func (p *PGRepo) SeedServer(ctx context.Context, name, subdomain string, cpuMill
|
||||
`INSERT INTO servers (name, cached_cpu_milli, cached_memory_mb, cached_storage_mb) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (name) DO UPDATE SET owner_id = NULL, claimed_at = NULL, last_active_at = now(),
|
||||
warned_3d_at = NULL, warned_1d_at = NULL, cached_phase = NULL, created_at = now(), deleted_at = NULL,
|
||||
retire_requested_at = NULL, retire_delete = false,
|
||||
cached_cpu_milli = EXCLUDED.cached_cpu_milli, cached_memory_mb = EXCLUDED.cached_memory_mb,
|
||||
cached_storage_mb = EXCLUDED.cached_storage_mb`,
|
||||
name, cpuMilli, memoryMB, storageMB); err != nil {
|
||||
@@ -865,7 +952,9 @@ func (p *PGRepo) BackupStoreBytes(ctx context.Context) (int64, error) {
|
||||
// world's point is its current owner's newest intact scheduled backup taken
|
||||
// since they claimed it: a previous owner's backups say nothing about the world
|
||||
// the new owner has built, and a corrupt one restores nothing. last_active_at
|
||||
// moves on every join, so a world nobody joined since its point is skipped.
|
||||
// moves on every join, so a world nobody joined since its point is skipped. A
|
||||
// world being given up is skipped too: the reaper archives it anyway, and a
|
||||
// backup Job holding it when the reaper runs would put the release off a day.
|
||||
func (p *PGRepo) ScheduledBackupCandidates(ctx context.Context, before time.Time) ([]ScheduledCandidate, error) {
|
||||
rows, err := p.db.QueryContext(ctx,
|
||||
`SELECT s.name, s.owner_id FROM servers s
|
||||
@@ -876,6 +965,7 @@ func (p *PGRepo) ScheduledBackupCandidates(ctx context.Context, before time.Time
|
||||
AND b.created_at >= COALESCE(s.claimed_at, '-infinity')
|
||||
) pt ON true
|
||||
WHERE s.deleted_at IS NULL AND s.owner_id IS NOT NULL
|
||||
AND s.retire_requested_at IS NULL
|
||||
AND s.last_active_at > COALESCE(pt.at, '-infinity')
|
||||
AND COALESCE(pt.at, '-infinity') < $1
|
||||
ORDER BY pt.at NULLS FIRST, s.name`, before)
|
||||
|
||||
@@ -15,6 +15,19 @@ type ServerRecord struct {
|
||||
OwnerID string
|
||||
// CachedPhase is the non-authoritative phase projection used for fast lists.
|
||||
CachedPhase string
|
||||
// Retire is the pending request to give the server up or delete it, nil when
|
||||
// there is none (PUT /servers/{name}/retirement).
|
||||
Retire *RetireState
|
||||
}
|
||||
|
||||
// RetireState is a server's pending retirement: its owner gave it up, or an admin
|
||||
// asked for it to be deleted (Delete). The reaper carries it out on its next run
|
||||
// (archive the world, delete the volume, release the server, and with Delete also
|
||||
// remove the server itself); until then the server stays stopped and cannot be
|
||||
// woken or claimed.
|
||||
type RetireState struct {
|
||||
RequestedAt time.Time `json:"requested_at"`
|
||||
Delete bool `json:"delete"`
|
||||
}
|
||||
|
||||
// MyServerView is a row of GET /api/v1/me/servers: a server the caller owns,
|
||||
@@ -38,6 +51,8 @@ type MyServerView struct {
|
||||
PlayerCountUnknown bool `json:"playerCountUnknown,omitempty"`
|
||||
AutoRestarts int32 `json:"autoRestarts,omitempty"`
|
||||
StartGaveUp bool `json:"startGaveUp,omitempty"`
|
||||
// Retiring is the pending retirement of a server the caller owns.
|
||||
Retiring *RetireState `json:"retiring,omitempty"`
|
||||
}
|
||||
|
||||
// ServerOwnership is one live server's claim state as the fleet read joins it.
|
||||
@@ -49,6 +64,9 @@ type ServerOwnership struct {
|
||||
// Owner is the claiming account's display identity (email, or username when
|
||||
// the address is absent), "" while unclaimed.
|
||||
Owner string
|
||||
// Retire is the server's pending retirement, nil when there is none. A server
|
||||
// being deleted is not claimable even while it has no owner.
|
||||
Retire *RetireState
|
||||
}
|
||||
|
||||
// AuditEntry is one row written to audit_logs (spec §6). Actor is display text:
|
||||
@@ -325,6 +343,7 @@ type Repo interface {
|
||||
// QuotaCheck remains the advisory pre-check for the handler's fast-path 403.
|
||||
// A successful claim resets last_active_at to now and clears warned_*, so the
|
||||
// reaper counts idleness from the claim.
|
||||
// A server with a pending retirement is not claimable either (false).
|
||||
ClaimServer(ctx context.Context, name, userID string) (bool, error)
|
||||
// UserInAllowlist reports whether the user's linked UUID is on the server
|
||||
// allowlist (spec §9.4).
|
||||
@@ -345,6 +364,15 @@ type Repo interface {
|
||||
// Both allowlist checks above skip revoked entries, and a change of owner
|
||||
// (claim, reaper release, account deletion) empties the list.
|
||||
SetAllowlistWake(ctx context.Context, name, mcUUID string, canWake bool) error
|
||||
// RequestRetire records that the server is to be given up, or deleted when
|
||||
// deleteServer, and returns the pending request. Asking again keeps the first
|
||||
// request time, and a deletion once asked for stays one. ErrNotFound when the
|
||||
// server does not exist.
|
||||
RequestRetire(ctx context.Context, name string, deleteServer bool) (RetireState, error)
|
||||
// CancelRetire drops the server's pending retirement; with none pending it
|
||||
// changes nothing. A pending deletion is dropped only when mayCancelDelete,
|
||||
// and otherwise ErrConflict. ErrNotFound when the server does not exist.
|
||||
CancelRetire(ctx context.Context, name string, mayCancelDelete bool) error
|
||||
// UserByMCUUID resolves a verified in-game UUID to the user_id it is linked to
|
||||
// (spec §10 account_links), or ErrNotFound when the UUID is not linked. The
|
||||
// internal-face wake uses it to apply the owner bypass for a player known only
|
||||
|
||||
@@ -647,32 +647,34 @@ func TestSeedServerReusedNameStartsClean(t *testing.T) {
|
||||
}
|
||||
past := time.Now().Add(-90 * 24 * time.Hour)
|
||||
exec(`UPDATE servers SET owner_id = $2, claimed_at = $3, last_active_at = $3, warned_3d_at = $3,
|
||||
warned_1d_at = $3, cached_phase = 'Running', created_at = $3, deleted_at = $3 WHERE name = $1`, name, u.ID, past)
|
||||
warned_1d_at = $3, cached_phase = 'Running', created_at = $3, deleted_at = $3,
|
||||
retire_requested_at = $3, retire_delete = true WHERE name = $1`, name, u.ID, past)
|
||||
exec(`INSERT INTO server_aliases (subdomain, server_name) VALUES ($1, $2)`, oldSub2, name)
|
||||
exec(`INSERT INTO server_allowlist (server_name, mc_uuid) VALUES ($1, $2)`, name, testUUID(t))
|
||||
|
||||
state := func() string {
|
||||
t.Helper()
|
||||
var owner, phase sql.NullString
|
||||
var claimed, w3, w1, deleted sql.NullTime
|
||||
var claimed, w3, w1, deleted, retireAt sql.NullTime
|
||||
var retireDelete bool
|
||||
var created, active time.Time
|
||||
var cpu, mem, stor, allow int
|
||||
var aliases string
|
||||
if err := db.QueryRowContext(ctx,
|
||||
`SELECT owner_id, claimed_at, warned_3d_at, warned_1d_at, cached_phase, deleted_at, created_at, last_active_at,
|
||||
cached_cpu_milli, cached_memory_mb, cached_storage_mb,
|
||||
cached_cpu_milli, cached_memory_mb, cached_storage_mb, retire_requested_at, retire_delete,
|
||||
(SELECT count(*) FROM server_allowlist WHERE server_name = s.name),
|
||||
(SELECT COALESCE(string_agg(subdomain, ',' ORDER BY subdomain), '') FROM server_aliases WHERE server_name = s.name)
|
||||
FROM servers s WHERE name = $1`, name).Scan(
|
||||
&owner, &claimed, &w3, &w1, &phase, &deleted, &created, &active, &cpu, &mem, &stor, &allow, &aliases); err != nil {
|
||||
&owner, &claimed, &w3, &w1, &phase, &deleted, &created, &active, &cpu, &mem, &stor, &retireAt, &retireDelete, &allow, &aliases); err != nil {
|
||||
t.Fatalf("read the servers row: %v", err)
|
||||
}
|
||||
recent := func(at time.Time) bool { return time.Since(at) < time.Hour }
|
||||
return fmt.Sprintf("owner=%v claimed=%v warned=%v/%v phase=%v deleted=%v fresh=%v/%v cache=%d/%d/%d allow=%d aliases=%s",
|
||||
return fmt.Sprintf("owner=%v claimed=%v warned=%v/%v phase=%v deleted=%v fresh=%v/%v cache=%d/%d/%d retire=%v/%v allow=%d aliases=%s",
|
||||
owner.Valid, claimed.Valid, w3.Valid, w1.Valid, phase.Valid, deleted.Valid, recent(created), recent(active),
|
||||
cpu, mem, stor, allow, strings.ReplaceAll(strings.ReplaceAll(aliases, oldSub2, "old2"), oldSub, "old"))
|
||||
cpu, mem, stor, retireAt.Valid, retireDelete, allow, strings.ReplaceAll(strings.ReplaceAll(aliases, oldSub2, "old2"), oldSub, "old"))
|
||||
}
|
||||
earlier := "owner=true claimed=true warned=true/true phase=true deleted=true fresh=false/false cache=1000/2048/10240 allow=1 aliases=old,old2"
|
||||
earlier := "owner=true claimed=true warned=true/true phase=true deleted=true fresh=false/false cache=1000/2048/10240 retire=true/true allow=1 aliases=old,old2"
|
||||
if got := state(); got != earlier {
|
||||
t.Fatalf("setup: %s, want %s", got, earlier)
|
||||
}
|
||||
@@ -689,7 +691,7 @@ func TestSeedServerReusedNameStartsClean(t *testing.T) {
|
||||
t.Fatalf("a refused seed changed the row: %s, want %s", got, earlier)
|
||||
}
|
||||
|
||||
clean := "owner=false claimed=false warned=false/false phase=false deleted=false fresh=true/true cache=2000/4096/20480 allow=0 aliases=" + newSub
|
||||
clean := "owner=false claimed=false warned=false/false phase=false deleted=false fresh=true/true cache=2000/4096/20480 retire=false/false allow=0 aliases=" + newSub
|
||||
for i := 0; i < 2; i++ { // a retried create lands on the same state
|
||||
if err := repo.SeedServer(ctx, name, newSub, 2000, 4096, 20480); err != nil {
|
||||
t.Fatalf("seed the new server (try %d): %v", i+1, err)
|
||||
|
||||
@@ -40,6 +40,11 @@ func (c *reclaimCluster) HoldWorld(ctx context.Context, _ string) (context.Conte
|
||||
|
||||
func (c *reclaimCluster) WorldExists(context.Context, string) (bool, error) { return true, nil }
|
||||
|
||||
// DeleteServer is only reached by a retirement, which these tests do not ask for.
|
||||
func (c *reclaimCluster) DeleteServer(_ context.Context, name, _ string) error {
|
||||
return errors.New("unexpected deletion of " + name)
|
||||
}
|
||||
|
||||
type reclaimArchiver struct{ archived []string }
|
||||
|
||||
func (a *reclaimArchiver) Archive(_ context.Context, server, _ string) (backup.Archived, error) {
|
||||
|
||||
@@ -0,0 +1,493 @@
|
||||
//go:build pgint
|
||||
|
||||
package pgint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/api"
|
||||
"felis.lolicon.best/internal/reaper"
|
||||
)
|
||||
|
||||
// ---- retirement (migration 0035) --------------------------------------------------
|
||||
|
||||
func retireColumns(t *testing.T, name string) (sql.NullTime, bool) {
|
||||
t.Helper()
|
||||
var at sql.NullTime
|
||||
var del bool
|
||||
if err := db.QueryRow(`SELECT retire_requested_at, retire_delete FROM servers WHERE name = $1`, name).Scan(&at, &del); err != nil {
|
||||
t.Fatalf("retirement of %s: %v", name, err)
|
||||
}
|
||||
return at, del
|
||||
}
|
||||
|
||||
// TestRetireRequestContract pins PUT and DELETE /servers/{name}/retirement at the
|
||||
// SQL: asking again keeps the first request's time, a deletion once asked for
|
||||
// stays one whoever asks after, only an admin takes a deletion back, and a
|
||||
// server that is gone is not found.
|
||||
func TestRetireRequestContract(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
owner := newUser(t, "user", "retire")
|
||||
name, sub := "rt-"+suffix(t), "rts-"+suffix(t)
|
||||
if err := repo.SeedServer(ctx, name, sub, 100, 128, 1); err != nil {
|
||||
t.Fatalf("SeedServer: %v", err)
|
||||
}
|
||||
mustExec(t, `UPDATE servers SET owner_id = $2 WHERE name = $1`, name, owner.ID)
|
||||
if rec, err := repo.ServerByName(ctx, name); err != nil || rec.Retire != nil {
|
||||
t.Fatalf("before any request: ServerByName = %+v, %v; want no retirement", rec, err)
|
||||
}
|
||||
|
||||
first, err := repo.RequestRetire(ctx, name, false)
|
||||
if err != nil || first.Delete || time.Since(first.RequestedAt) > time.Minute {
|
||||
t.Fatalf("RequestRetire = %+v, %v; want a release asked just now", first, err)
|
||||
}
|
||||
for _, ask := range []struct {
|
||||
del, want bool
|
||||
}{{false, false}, {true, true}, {false, true}} {
|
||||
st, err := repo.RequestRetire(ctx, name, ask.del)
|
||||
if err != nil || !st.RequestedAt.Equal(first.RequestedAt) || st.Delete != ask.want {
|
||||
t.Fatalf("RequestRetire(delete=%v) = %+v, %v; want the first time %v and delete=%v",
|
||||
ask.del, st, err, first.RequestedAt, ask.want)
|
||||
}
|
||||
}
|
||||
for how, get := range map[string]func() (*api.ServerRecord, error){
|
||||
"name": func() (*api.ServerRecord, error) { return repo.ServerByName(ctx, name) },
|
||||
"subdomain": func() (*api.ServerRecord, error) { return repo.ServerBySubdomain(ctx, sub) },
|
||||
} {
|
||||
rec, err := get()
|
||||
if err != nil || rec.Retire == nil || !rec.Retire.Delete || !rec.Retire.RequestedAt.Equal(first.RequestedAt) {
|
||||
t.Fatalf("server by %s = %+v, %v; want the pending deletion", how, rec, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := repo.CancelRetire(ctx, name, false); !errors.Is(err, api.ErrConflict) {
|
||||
t.Fatalf("the owner cancelling a deletion = %v, want ErrConflict", err)
|
||||
}
|
||||
if at, del := retireColumns(t, name); !at.Valid || !at.Time.Equal(first.RequestedAt) || !del {
|
||||
t.Fatalf("a refused cancel changed the request: at=%v delete=%v", at, del)
|
||||
}
|
||||
if err := repo.CancelRetire(ctx, name, true); err != nil {
|
||||
t.Fatalf("an admin cancelling a deletion: %v", err)
|
||||
}
|
||||
if at, del := retireColumns(t, name); at.Valid || del {
|
||||
t.Fatalf("after the cancel: at=%v delete=%v; want no request", at, del)
|
||||
}
|
||||
if rec, err := repo.ServerByName(ctx, name); err != nil || rec.Retire != nil {
|
||||
t.Fatalf("after the cancel: ServerByName = %+v, %v", rec, err)
|
||||
}
|
||||
|
||||
// A give-up is the owner's to take back, and a new one starts its own clock.
|
||||
again, err := repo.RequestRetire(ctx, name, false)
|
||||
if err != nil || again.Delete || !again.RequestedAt.After(first.RequestedAt) {
|
||||
t.Fatalf("a new request = %+v, %v; want a release after %v", again, err, first.RequestedAt)
|
||||
}
|
||||
if err := repo.CancelRetire(ctx, name, false); err != nil {
|
||||
t.Fatalf("the owner cancelling a give-up: %v", err)
|
||||
}
|
||||
if at, del := retireColumns(t, name); at.Valid || del {
|
||||
t.Fatalf("after the owner's cancel: at=%v delete=%v", at, del)
|
||||
}
|
||||
|
||||
gone := "rtg-" + suffix(t)
|
||||
seedOwnedServer(t, gone, owner.ID, true)
|
||||
for _, n := range []string{gone, "rt-none-" + suffix(t)} {
|
||||
if st, err := repo.RequestRetire(ctx, n, true); !errors.Is(err, api.ErrNotFound) {
|
||||
t.Fatalf("RequestRetire(%s) = %+v, %v; want ErrNotFound", n, st, err)
|
||||
}
|
||||
if err := repo.CancelRetire(ctx, n, true); !errors.Is(err, api.ErrNotFound) {
|
||||
t.Fatalf("CancelRetire(%s) = %v; want ErrNotFound", n, err)
|
||||
}
|
||||
}
|
||||
if at, _ := retireColumns(t, gone); at.Valid {
|
||||
t.Fatalf("a deleted server's row took a request")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetiringServerIsNotClaimable: a server an admin is releasing or deleting
|
||||
// cannot be claimed, even unowned, and the claim lists say so; only its owner is
|
||||
// told of the request, and the admin fleet read sees it on every row.
|
||||
func TestRetiringServerIsNotClaimable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
u := newUser(t, "user", "rtclaim")
|
||||
other := newUser(t, "user", "rtclaim-other")
|
||||
sfx := suffix(t)
|
||||
free, freeRetiring, mine, mineRetiring, theirs := "rcf-"+sfx, "rcr-"+sfx, "rco-"+sfx, "rcm-"+sfx, "rct-"+sfx
|
||||
for _, n := range []string{free, freeRetiring} {
|
||||
mustExec(t, `INSERT INTO servers (name, cached_cpu_milli, cached_memory_mb, cached_storage_mb) VALUES ($1, 100, 128, 1)`, n)
|
||||
}
|
||||
seedOwnedServer(t, mine, u.ID, false)
|
||||
seedOwnedServer(t, mineRetiring, u.ID, false)
|
||||
seedOwnedServer(t, theirs, other.ID, false)
|
||||
released, err := repo.RequestRetire(ctx, freeRetiring, false)
|
||||
if err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", freeRetiring, err)
|
||||
}
|
||||
deleting, err := repo.RequestRetire(ctx, mineRetiring, true)
|
||||
if err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", mineRetiring, err)
|
||||
}
|
||||
if _, err := repo.RequestRetire(ctx, theirs, false); err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", theirs, err)
|
||||
}
|
||||
|
||||
if ok, err := repo.ClaimServer(ctx, freeRetiring, u.ID); err != nil || ok {
|
||||
t.Fatalf("claiming a server being released = (%v, %v), want (false, nil)", ok, err)
|
||||
}
|
||||
if o := serverOwner(t, freeRetiring); o != "" {
|
||||
t.Fatalf("the refused claim left owner %q", o)
|
||||
}
|
||||
|
||||
views, err := repo.MyServers(ctx, u.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("MyServers: %v", err)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, v := range views {
|
||||
if strings.HasSuffix(v.Name, sfx) {
|
||||
retiring := "none"
|
||||
if v.Retiring != nil {
|
||||
retiring = fmt.Sprintf("delete=%v", v.Retiring.Delete)
|
||||
if !v.Retiring.RequestedAt.Equal(deleting.RequestedAt) {
|
||||
retiring += " at another time"
|
||||
}
|
||||
}
|
||||
got[v.Name] = fmt.Sprintf("owned=%v claimable=%v retiring=%s", v.Owned, v.Claimable, retiring)
|
||||
}
|
||||
}
|
||||
want := map[string]string{
|
||||
free: "owned=false claimable=true retiring=none",
|
||||
freeRetiring: "owned=false claimable=false retiring=none",
|
||||
mine: "owned=true claimable=false retiring=none",
|
||||
mineRetiring: "owned=true claimable=false retiring=delete=true",
|
||||
}
|
||||
if fmt.Sprint(got) != fmt.Sprint(want) {
|
||||
t.Fatalf("MyServers = %v\nwant %v", got, want)
|
||||
}
|
||||
|
||||
owners, err := repo.ServerOwners(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ServerOwners: %v", err)
|
||||
}
|
||||
for n, w := range map[string]*api.RetireState{free: nil, mine: nil, freeRetiring: &released, mineRetiring: &deleting} {
|
||||
r := owners[n].Retire
|
||||
if (r == nil) != (w == nil) || (r != nil && (r.Delete != w.Delete || !r.RequestedAt.Equal(w.RequestedAt))) {
|
||||
t.Errorf("ServerOwners[%s].Retire = %+v, want %+v", n, r, w)
|
||||
}
|
||||
}
|
||||
if r := owners[theirs].Retire; r == nil || r.Delete {
|
||||
t.Errorf("ServerOwners[%s].Retire = %+v, want another owner's give-up", theirs, r)
|
||||
}
|
||||
|
||||
// Taken back, the server may be claimed again.
|
||||
if err := repo.CancelRetire(ctx, freeRetiring, true); err != nil {
|
||||
t.Fatalf("CancelRetire: %v", err)
|
||||
}
|
||||
if ok, err := repo.ClaimServer(ctx, freeRetiring, u.ID); err != nil || !ok {
|
||||
t.Fatalf("claim after the cancel = (%v, %v)", ok, err)
|
||||
}
|
||||
for _, n := range []string{mineRetiring, theirs} {
|
||||
if err := repo.CancelRetire(ctx, n, true); err != nil {
|
||||
t.Fatalf("CancelRetire(%s): %v", n, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireReaperStore pins the reaper's reads and writes of a retirement: the
|
||||
// request reaches its candidates, a retirement's archive holds the world as an
|
||||
// idle reap's does and is never evicted while it is the only copy, releasing the
|
||||
// world finishes a give-up and leaves a deletion for the next run, and marking
|
||||
// the row deleted takes only a server asked to be deleted.
|
||||
func TestRetireReaperStore(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := reaper.NewPGStore(db)
|
||||
u := newUser(t, "user", "rtstore")
|
||||
sfx := suffix(t)
|
||||
given, doomed, plain := "rsg-"+sfx, "rsd-"+sfx, "rsp-"+sfx
|
||||
for _, n := range []string{given, doomed, plain} {
|
||||
seedOwnedServer(t, n, u.ID, false)
|
||||
}
|
||||
givenAt, err := repo.RequestRetire(ctx, given, false)
|
||||
if err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", given, err)
|
||||
}
|
||||
doomedAt, err := repo.RequestRetire(ctx, doomed, true)
|
||||
if err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", doomed, err)
|
||||
}
|
||||
|
||||
cands, err := st.ListActiveServers(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListActiveServers: %v", err)
|
||||
}
|
||||
seen := map[string]reaper.Candidate{}
|
||||
for _, c := range cands {
|
||||
seen[c.Name] = c
|
||||
}
|
||||
for n, w := range map[string]api.RetireState{given: givenAt, doomed: doomedAt, plain: {}} {
|
||||
c, ok := seen[n]
|
||||
if !ok || !c.RetireRequestedAt.Equal(w.RequestedAt) || c.RetireDelete != w.Delete {
|
||||
t.Errorf("candidate %s = %+v (listed %v); want retirement %+v", n, c, ok, w)
|
||||
}
|
||||
}
|
||||
|
||||
// A retirement's archive is the reaper's own: it holds the world, and it is
|
||||
// kept like an idle reap's until it has its off-site copy.
|
||||
now := time.Now()
|
||||
backup := func(server, id, reason string, created time.Time, offsite bool) {
|
||||
t.Helper()
|
||||
var offsiteAt any
|
||||
if offsite {
|
||||
offsiteAt = created
|
||||
}
|
||||
mustExec(t, `INSERT INTO world_backups (id, server_name, backup_ref, size_bytes, reason, status, created_at, expires_at, offsite_at)
|
||||
VALUES ($1, $2, $3, 1, $4, 'present', $5, $6, $7)`,
|
||||
id, server, "/archives/"+id+".tar.gz", reason, created, created.Add(90*reaper.Day), offsiteAt)
|
||||
}
|
||||
manualID, soleID, copiedID := "bk-rsm-"+sfx, "bk-rss-"+sfx, "bk-rsc-"+sfx
|
||||
backup(plain, manualID, "manual", now.Add(-time.Minute), false)
|
||||
if f, ok, err := st.FreshBackup(ctx, plain, now.Add(-time.Hour)); err != nil || ok {
|
||||
t.Fatalf("FreshBackup over a manual backup = (%+v, %v, %v); only the reaper's archives count", f, ok, err)
|
||||
}
|
||||
backup(plain, soleID, reaper.ReasonReleased, now.Add(-3*time.Minute), false)
|
||||
backup(plain, copiedID, reaper.ReasonReleased, now.Add(-2*time.Minute), true)
|
||||
f, ok, err := st.FreshBackup(ctx, plain, now.Add(-time.Hour))
|
||||
if err != nil || !ok || f.ID != copiedID || !f.Offsite {
|
||||
t.Fatalf("FreshBackup = (%+v, %v, %v); want the copied retirement archive %s", f, ok, err, copiedID)
|
||||
}
|
||||
if f, ok, err := st.FreshBackup(ctx, plain, now); err != nil || ok {
|
||||
t.Fatalf("FreshBackup since now = (%+v, %v, %v); an archive older than since holds no world", f, ok, err)
|
||||
}
|
||||
all, err := st.EvictableBackups(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("EvictableBackups: %v", err)
|
||||
}
|
||||
var order []string
|
||||
for _, b := range all {
|
||||
if b.ServerName == plain {
|
||||
order = append(order, b.ID)
|
||||
}
|
||||
}
|
||||
if strings.Join(order, ",") != manualID+","+copiedID {
|
||||
t.Fatalf("eviction order = %v; want %s, then %s, and never the only copy %s", order, manualID, copiedID, soleID)
|
||||
}
|
||||
|
||||
// Each server keeps an allowlist entry and an extra alias to see what goes.
|
||||
for _, n := range []string{given, doomed, plain} {
|
||||
mustExec(t, `INSERT INTO server_aliases (subdomain, server_name) VALUES ($1, $2)`, n+"-a", n)
|
||||
mustExec(t, `INSERT INTO server_allowlist (server_name, mc_uuid) VALUES ($1, $2)`, n, testUUID(t))
|
||||
}
|
||||
state := func(n string) string {
|
||||
t.Helper()
|
||||
var owner sql.NullString
|
||||
var deleted, retireAt sql.NullTime
|
||||
var del bool
|
||||
var aliases, allow int
|
||||
if err := db.QueryRowContext(ctx,
|
||||
`SELECT owner_id, deleted_at, retire_requested_at, retire_delete,
|
||||
(SELECT count(*) FROM server_aliases WHERE server_name = s.name),
|
||||
(SELECT count(*) FROM server_allowlist WHERE server_name = s.name)
|
||||
FROM servers s WHERE name = $1`, n).Scan(&owner, &deleted, &retireAt, &del, &aliases, &allow); err != nil {
|
||||
t.Fatalf("read %s: %v", n, err)
|
||||
}
|
||||
return fmt.Sprintf("owner=%v deleted=%v retire=%v/%v aliases=%d allow=%d",
|
||||
owner.Valid, deleted.Valid, retireAt.Valid, del, aliases, allow)
|
||||
}
|
||||
|
||||
// Only a server asked to be deleted loses its row.
|
||||
for _, n := range []string{given, plain} {
|
||||
before := state(n)
|
||||
if err := st.DeleteServerRow(ctx, n, now); err != nil {
|
||||
t.Fatalf("DeleteServerRow(%s): %v", n, err)
|
||||
}
|
||||
if got := state(n); got != before {
|
||||
t.Fatalf("DeleteServerRow(%s) touched a server nobody is deleting: %s, was %s", n, got, before)
|
||||
}
|
||||
}
|
||||
|
||||
// Releasing the world finishes a give-up; a deletion stays pending.
|
||||
for _, n := range []string{given, doomed} {
|
||||
if err := st.ReleaseWorld(ctx, n, now); err != nil {
|
||||
t.Fatalf("ReleaseWorld(%s): %v", n, err)
|
||||
}
|
||||
}
|
||||
if got, want := state(given), "owner=false deleted=false retire=false/false aliases=1 allow=0"; got != want {
|
||||
t.Fatalf("given up and released: %s, want %s", got, want)
|
||||
}
|
||||
if got, want := state(doomed), "owner=false deleted=false retire=true/true aliases=1 allow=0"; got != want {
|
||||
t.Fatalf("released while being deleted: %s, want %s", got, want)
|
||||
}
|
||||
if at, _ := retireColumns(t, doomed); !at.Time.Equal(doomedAt.RequestedAt) {
|
||||
t.Fatalf("the release moved the deletion's request time to %v, want %v", at.Time, doomedAt.RequestedAt)
|
||||
}
|
||||
|
||||
mustExec(t, `INSERT INTO server_allowlist (server_name, mc_uuid) VALUES ($1, $2)`, doomed, testUUID(t))
|
||||
if err := st.DeleteServerRow(ctx, doomed, now); err != nil {
|
||||
t.Fatalf("DeleteServerRow(%s): %v", doomed, err)
|
||||
}
|
||||
if got, want := state(doomed), "owner=false deleted=true retire=false/false aliases=0 allow=0"; got != want {
|
||||
t.Fatalf("deleted: %s, want %s", got, want)
|
||||
}
|
||||
if got, want := state(plain), "owner=true deleted=false retire=false/false aliases=1 allow=1"; got != want {
|
||||
t.Fatalf("a bystander changed: %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// fleetCluster knows a set of servers by uid; every other row in the shared
|
||||
// schema reads as a server whose CRD is gone and whose world volume is still
|
||||
// there, which the reaper never touches.
|
||||
type fleetCluster struct {
|
||||
uids map[string]string
|
||||
deletedPVC []string
|
||||
deletedServer []string
|
||||
}
|
||||
|
||||
func (c *fleetCluster) Inspect(_ context.Context, name string) (reaper.ServerCRD, error) {
|
||||
uid, ok := c.uids[name]
|
||||
if !ok {
|
||||
return reaper.ServerCRD{}, reaper.ErrNotFound
|
||||
}
|
||||
return reaper.ServerCRD{PVC: reaper.WorldPVCName(name), UID: uid}, nil
|
||||
}
|
||||
|
||||
func (c *fleetCluster) HoldWorld(ctx context.Context, _ string) (context.Context, func(), error) {
|
||||
return ctx, func() {}, nil
|
||||
}
|
||||
|
||||
func (c *fleetCluster) WorldExists(_ context.Context, pvc string) (bool, error) {
|
||||
for _, gone := range c.deletedPVC {
|
||||
if gone == pvc {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (c *fleetCluster) DeletePVC(_ context.Context, pvc string) error {
|
||||
c.deletedPVC = append(c.deletedPVC, pvc)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *fleetCluster) DeleteServer(_ context.Context, name, uid string) error {
|
||||
if c.uids[name] != uid {
|
||||
return fmt.Errorf("DeleteServer(%s, %s): the server has uid %s", name, uid, c.uids[name])
|
||||
}
|
||||
c.deletedServer = append(c.deletedServer, name)
|
||||
delete(c.uids, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestRetirementThroughTheReaper runs the reaper over a give-up and a deletion
|
||||
// felis-api recorded, beside a server played minutes ago: the two go on the
|
||||
// first run however recently they were played, each archived first (in name
|
||||
// order, as the reaper lists them), and a second
|
||||
// run finds nothing left to do. The deleted server's name and subdomain are free
|
||||
// for a new server, which starts clean.
|
||||
func TestRetirementThroughTheReaper(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := reaper.NewPGStore(db)
|
||||
u := newUser(t, "user", "rtreap")
|
||||
sfx := suffix(t)
|
||||
given, doomed, bystander := "rrg-"+sfx, "rrd-"+sfx, "rrb-"+sfx
|
||||
cl := &fleetCluster{uids: map[string]string{}}
|
||||
for _, n := range []string{given, doomed, bystander} {
|
||||
if err := repo.SeedServer(ctx, n, n+"-s", 100, 128, 1); err != nil {
|
||||
t.Fatalf("SeedServer(%s): %v", n, err)
|
||||
}
|
||||
mustExec(t, `UPDATE servers SET owner_id = $2, claimed_at = now() - interval '30 days', last_active_at = now() - interval '5 minutes'
|
||||
WHERE name = $1`, n, u.ID)
|
||||
mustExec(t, `INSERT INTO server_allowlist (server_name, mc_uuid) VALUES ($1, $2)`, n, testUUID(t))
|
||||
cl.uids[n] = "uid-" + n
|
||||
}
|
||||
if _, err := repo.RequestRetire(ctx, given, false); err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", given, err)
|
||||
}
|
||||
if _, err := repo.RequestRetire(ctx, doomed, true); err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", doomed, err)
|
||||
}
|
||||
|
||||
ar := &reclaimArchiver{}
|
||||
r := &reaper.Reaper{Cfg: reaper.DefaultConfig(), Store: st, Cluster: cl, Archiver: ar}
|
||||
if _, err := r.RunOnce(ctx); err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
mineOnly := func(refs []string) string {
|
||||
var out []string
|
||||
for _, ref := range refs {
|
||||
for _, n := range []string{given, doomed, bystander} {
|
||||
if strings.Contains(ref, n) {
|
||||
out = append(out, strings.Replace(ref, n, map[string]string{given: "given", doomed: "doomed", bystander: "bystander"}[n], 1))
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Join(out, " ")
|
||||
}
|
||||
if got, want := mineOnly(cl.deletedPVC), "world-doomed-0 world-given-0"; got != want {
|
||||
t.Fatalf("volumes deleted: %q, want %q", got, want)
|
||||
}
|
||||
if got, want := mineOnly(cl.deletedServer), "doomed"; got != want {
|
||||
t.Fatalf("servers deleted: %q, want %q", got, want)
|
||||
}
|
||||
if got, want := mineOnly(ar.archived), "/archives/doomed-new.tar.gz /archives/given-new.tar.gz"; got != want {
|
||||
t.Fatalf("archived: %q, want %q", got, want)
|
||||
}
|
||||
|
||||
type row struct {
|
||||
owner sql.NullString
|
||||
deleted, retir sql.NullTime
|
||||
aliases, allow int
|
||||
archives string
|
||||
audit string
|
||||
}
|
||||
read := func(n string) row {
|
||||
t.Helper()
|
||||
var r row
|
||||
if err := db.QueryRowContext(ctx,
|
||||
`SELECT owner_id, deleted_at, retire_requested_at,
|
||||
(SELECT count(*) FROM server_aliases WHERE server_name = s.name),
|
||||
(SELECT count(*) FROM server_allowlist WHERE server_name = s.name),
|
||||
(SELECT COALESCE(string_agg(reason || ':' || status || ':' || COALESCE(former_owner, ''), ','), '') FROM world_backups WHERE server_name = s.name),
|
||||
(SELECT COALESCE(string_agg(action, ',' ORDER BY id), '') FROM audit_logs WHERE server_name = s.name AND source = 'reaper')
|
||||
FROM servers s WHERE name = $1`, n).Scan(&r.owner, &r.deleted, &r.retir, &r.aliases, &r.allow, &r.archives, &r.audit); err != nil {
|
||||
t.Fatalf("read %s: %v", n, err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
g := read(given)
|
||||
if g.owner.Valid || g.deleted.Valid || g.retir.Valid || g.aliases != 1 || g.allow != 0 ||
|
||||
g.archives != "released:present:"+u.ID || g.audit != reaper.ActionReleaseWorld {
|
||||
t.Fatalf("given up: %+v; want released, still listed at its subdomain, its archive kept as the owner's", g)
|
||||
}
|
||||
d := read(doomed)
|
||||
if d.owner.Valid || !d.deleted.Valid || d.retir.Valid || d.aliases != 0 || d.allow != 0 ||
|
||||
d.archives != "released:present:"+u.ID || d.audit != reaper.ActionDeleteServer {
|
||||
t.Fatalf("deleted: %+v; want the row marked deleted, its subdomain freed, its archive kept", d)
|
||||
}
|
||||
b := read(bystander)
|
||||
if b.owner.String != u.ID || b.deleted.Valid || b.aliases != 1 || b.allow != 1 || b.archives != "" || b.audit != "" {
|
||||
t.Fatalf("the bystander changed: %+v", b)
|
||||
}
|
||||
|
||||
// Nothing is left for the next run.
|
||||
if _, err := r.RunOnce(ctx); err != nil {
|
||||
t.Fatalf("second RunOnce: %v", err)
|
||||
}
|
||||
if got := mineOnly(ar.archived) + " | " + mineOnly(cl.deletedServer); got != "/archives/doomed-new.tar.gz /archives/given-new.tar.gz | doomed" {
|
||||
t.Fatalf("the second run did more: %s", got)
|
||||
}
|
||||
|
||||
if err := repo.SeedServer(ctx, doomed, doomed+"-s", 200, 256, 2); err != nil {
|
||||
t.Fatalf("a new server under the deleted one's name: %v", err)
|
||||
}
|
||||
if at, del := retireColumns(t, doomed); at.Valid || del {
|
||||
t.Fatalf("the new server inherited a retirement: at=%v delete=%v", at, del)
|
||||
}
|
||||
if rec, err := repo.ServerBySubdomain(ctx, doomed+"-s"); err != nil || rec.Name != doomed || rec.OwnerID != "" || rec.Retire != nil {
|
||||
t.Fatalf("the new server = %+v, %v; want it unowned at the freed subdomain", rec, err)
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,8 @@ import (
|
||||
// backups: an owned world joined since its owner's newest intact scheduled
|
||||
// backup is due once that backup is older than the period, worlds without one
|
||||
// first. A previous owner's, a corrupt, a deleted, a pre-claim or a manual
|
||||
// backup is no restore point of the current owner's world.
|
||||
// backup is no restore point of the current owner's world. A world being given
|
||||
// up is never due: the reaper archives it anyway.
|
||||
func TestScheduledBackupCandidates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
u := newUser(t, "user", "sched")
|
||||
@@ -75,6 +76,10 @@ func TestScheduledBackupCandidates(t *testing.T) {
|
||||
backup(manual, u.ID, "manual", "present", now.Add(-30*time.Minute), false)
|
||||
server("unowned", "", claimed, now.Add(-h), false)
|
||||
server("gone", u.ID, claimed, now.Add(-h), true)
|
||||
retiring := server("retiring", u.ID, claimed, now.Add(-h), false)
|
||||
if _, err := repo.RequestRetire(ctx, retiring, false); err != nil {
|
||||
t.Fatalf("RequestRetire: %v", err)
|
||||
}
|
||||
|
||||
got, err := repo.ScheduledBackupCandidates(ctx, now.Add(-24*h))
|
||||
if err != nil {
|
||||
|
||||
+17
-13
@@ -185,12 +185,16 @@ func OperatorRole(p Params) *rbacv1.Role {
|
||||
})
|
||||
}
|
||||
|
||||
// ReaperRole grants felis-reaper its two destructive, disjoint powers
|
||||
// (internal/reaper.k8scluster): patch a MinecraftServer to Stop it and delete its
|
||||
// world PVC. Candidate servers come from the Postgres store, not a cluster List,
|
||||
// so minecraftservers need no list/watch; the reaper uses a direct client. It can
|
||||
// read+patch minecraftservers but cannot create them, and holds no power over
|
||||
// StatefulSets, Services, or Secrets — those belong to the operator and api.
|
||||
// ReaperRole grants felis-reaper its destructive powers
|
||||
// (internal/reaper.k8scluster): patch a MinecraftServer to Stop it, delete its
|
||||
// world PVC, and delete the MinecraftServer of a server an admin deleted.
|
||||
// Candidate servers come from the Postgres store, not a cluster List, so
|
||||
// minecraftservers need no list/watch; the reaper uses a direct client. It can
|
||||
// read, patch and delete minecraftservers but cannot create them, and holds no
|
||||
// power over StatefulSets, Services, or Secrets — those belong to the operator
|
||||
// and api (deleting a MinecraftServer lets garbage collection take the ones the
|
||||
// operator made for it; the world PVC is retained by the StatefulSet and the
|
||||
// reaper deletes it first, after archiving it).
|
||||
//
|
||||
// The same patch holds the world maintenance lock while a world is archived and
|
||||
// reclaimed (internal/maintenance). Taking it needs the two reads felis-api makes
|
||||
@@ -202,16 +206,16 @@ func OperatorRole(p Params) *rbacv1.Role {
|
||||
// stock local-path directory name. get is strictly weaker than the delete the
|
||||
// same rule already grants, so it widens nothing.
|
||||
//
|
||||
// Note no identity anywhere holds minecraftservers:delete. That is intentional, not
|
||||
// a missing grant: reaping releases a server by flipping desiredState=Stopped and
|
||||
// reclaiming the world PVC (k8scluster.go does "nothing else"), leaving the CR in
|
||||
// place so a former owner can re-claim it within the retention window (spec §466).
|
||||
// The MinecraftServer CR is the lifecycle source of truth and is retained, never
|
||||
// hard-deleted, so the delete verb is deliberately absent from every Role.
|
||||
// The reaper is the only identity with minecraftservers:delete. Reaping an idle
|
||||
// world releases the server by flipping desiredState=Stopped and reclaiming the
|
||||
// world PVC, leaving the CR in place so it can be claimed again; the CR goes only
|
||||
// when an admin deletes the server (PUT /servers/{name}/retirement), and then only
|
||||
// once the reaper has archived and deleted its world. felis-api records that
|
||||
// request and never deletes a CR itself.
|
||||
func ReaperRole(p Params) *rbacv1.Role {
|
||||
p = p.withDefaults()
|
||||
return role(p.MinecraftNamespace, "felis-reaper", ComponentReaper, []rbacv1.PolicyRule{
|
||||
rule([]string{groupFelis}, []string{"minecraftservers"}, []string{"get", "patch"}),
|
||||
rule([]string{groupFelis}, []string{"minecraftservers"}, []string{"get", "patch", "delete"}),
|
||||
rule([]string{groupCore}, []string{"persistentvolumeclaims"}, []string{"get", "delete"}),
|
||||
rule([]string{groupCore}, []string{"pods"}, []string{"list"}),
|
||||
rule([]string{groupBatch}, []string{"jobs"}, []string{"list"}),
|
||||
|
||||
@@ -331,17 +331,32 @@ func TestReaperRBAC_GatedOnRetention(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoIdentityDeletesMinecraftServers locks the lifecycle invariant: the
|
||||
// MinecraftServer CR is the retained source of truth (released by Stop + PVC
|
||||
// reclaim, never hard-deleted, so a former owner can re-claim within the retention
|
||||
// window — spec §466). No control-plane identity may hold minecraftservers:delete;
|
||||
// if a future change adds it, this fails loudly so the decision is deliberate.
|
||||
func TestNoIdentityDeletesMinecraftServers(t *testing.T) {
|
||||
// TestOnlyReaperDeletesMinecraftServers locks the lifecycle invariant: a
|
||||
// MinecraftServer is retained when its world is reaped (released by Stop + PVC
|
||||
// reclaim, so it can be claimed again), and removed only when an admin deletes the
|
||||
// server, by the reaper, after it archived and deleted the world. No other
|
||||
// control-plane identity may hold minecraftservers:delete, and without a reaper
|
||||
// deployed none does.
|
||||
func TestOnlyReaperDeletesMinecraftServers(t *testing.T) {
|
||||
for _, role := range ControlPlaneRBAC(testParams()).Roles {
|
||||
if hasRule(role, groupFelis, "minecraftservers", "delete") {
|
||||
t.Errorf("%s grants minecraftservers:delete — the CR is retained, never hard-deleted", role.Name)
|
||||
t.Errorf("%s grants minecraftservers:delete with no reaper deployed", role.Name)
|
||||
}
|
||||
}
|
||||
reaper := 0
|
||||
for _, role := range ControlPlaneRBAC(reaperParams()).Roles {
|
||||
if !hasRule(role, groupFelis, "minecraftservers", "delete") {
|
||||
continue
|
||||
}
|
||||
if role.Name != "felis-reaper" {
|
||||
t.Errorf("%s grants minecraftservers:delete — only the reaper removes a server, after archiving its world", role.Name)
|
||||
continue
|
||||
}
|
||||
reaper++
|
||||
}
|
||||
if reaper != 1 {
|
||||
t.Error("the reaper must delete minecraftservers (an admin's deletion of a server)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoClusterScopedRBAC enforces the §22 red line: nothing in the bundle is a
|
||||
|
||||
@@ -33,7 +33,8 @@ const gamePodComponent = "server"
|
||||
|
||||
// K8sCluster is the production Cluster backed by a controller-runtime client
|
||||
// (spec §4, §18). It reads spec.reaperExempt, stops a server and holds its world
|
||||
// volume through the maintenance lock, and deletes the world PVC — nothing else.
|
||||
// volume through the maintenance lock, deletes the world PVC, and removes the
|
||||
// MinecraftServer of a server an admin deleted — nothing else.
|
||||
type K8sCluster struct {
|
||||
c client.Client
|
||||
namespace string
|
||||
@@ -60,7 +61,7 @@ func (k *K8sCluster) Inspect(ctx context.Context, name string) (ServerCRD, error
|
||||
if err := k.get(ctx, name, &ms); err != nil {
|
||||
return ServerCRD{}, err
|
||||
}
|
||||
return ServerCRD{Exempt: ms.Spec.ReaperExempt, PVC: WorldPVCName(name)}, nil
|
||||
return ServerCRD{Exempt: ms.Spec.ReaperExempt, PVC: WorldPVCName(name), UID: string(ms.UID)}, nil
|
||||
}
|
||||
|
||||
// HoldWorld implements Cluster. The lock is the same Annotation felis-api
|
||||
@@ -232,6 +233,30 @@ func (k *K8sCluster) DeletePVC(ctx context.Context, pvc string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteServer implements Cluster. The read and the delete are one step (the
|
||||
// delete carries the resourceVersion read), so the object removed is the one
|
||||
// whose uid was checked: a server made again under the name is refused.
|
||||
func (k *K8sCluster) DeleteServer(ctx context.Context, name, uid string) error {
|
||||
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
var ms v1alpha1.MinecraftServer
|
||||
switch err := k.get(ctx, name, &ms); {
|
||||
case errors.Is(err, ErrNotFound):
|
||||
return nil
|
||||
case err != nil:
|
||||
return err
|
||||
}
|
||||
if string(ms.UID) != uid {
|
||||
return fmt.Errorf("MinecraftServer %s is another server now (uid %s, inspected %s)", name, ms.UID, uid)
|
||||
}
|
||||
rv, u := ms.ResourceVersion, ms.UID
|
||||
err := k.c.Delete(ctx, &ms, client.Preconditions{UID: &u, ResourceVersion: &rv})
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (k *K8sCluster) get(ctx context.Context, name string, ms *v1alpha1.MinecraftServer) error {
|
||||
if err := k.c.Get(ctx, types.NamespacedName{Namespace: k.namespace, Name: name}, ms); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
|
||||
@@ -216,3 +216,36 @@ func TestGamePodComponentMatchesOperator(t *testing.T) {
|
||||
t.Fatalf("gamePodComponent = %q, operator labels its pods %q", gamePodComponent, operator.ComponentValue)
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteServer removes the MinecraftServer Inspect returned, and nothing that
|
||||
// merely carries its name: a server made again under the name (another uid)
|
||||
// stays, and one already gone is not an error.
|
||||
func TestDeleteServerRemovesOnlyTheInspectedServer(t *testing.T) {
|
||||
ms := holdServer(v1alpha1.DesiredStopped, v1alpha1.PhaseStopped)
|
||||
ms.UID = "uid-1"
|
||||
k, c, _ := holdCluster(t, ms)
|
||||
ctx := context.Background()
|
||||
|
||||
crd, err := k.Inspect(ctx, "survival")
|
||||
if err != nil || crd.UID != "uid-1" {
|
||||
t.Fatalf("Inspect = %+v, %v; want uid-1", crd, err)
|
||||
}
|
||||
|
||||
if err := k.DeleteServer(ctx, "survival", "uid-0"); err == nil {
|
||||
t.Fatal("DeleteServer with another uid succeeded")
|
||||
}
|
||||
var got v1alpha1.MinecraftServer
|
||||
if err := c.Get(ctx, types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &got); err != nil {
|
||||
t.Fatalf("the server of another uid was deleted: %v", err)
|
||||
}
|
||||
|
||||
if err := k.DeleteServer(ctx, "survival", crd.UID); err != nil {
|
||||
t.Fatalf("DeleteServer: %v", err)
|
||||
}
|
||||
if err := c.Get(ctx, types.NamespacedName{Namespace: "minecraft", Name: "survival"}, &got); err == nil {
|
||||
t.Fatal("the server is still there")
|
||||
}
|
||||
if err := k.DeleteServer(ctx, "survival", crd.UID); err != nil {
|
||||
t.Fatalf("DeleteServer of a server already gone = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
+34
-11
@@ -20,7 +20,7 @@ type PGStore struct {
|
||||
func NewPGStore(db *sql.DB) *PGStore { return &PGStore{db: db} }
|
||||
|
||||
func (s *PGStore) ListActiveServers(ctx context.Context) ([]Candidate, error) {
|
||||
const q = `SELECT name, owner_id, last_active_at, warned_3d_at, warned_1d_at
|
||||
const q = `SELECT name, owner_id, last_active_at, warned_3d_at, warned_1d_at, retire_requested_at, retire_delete
|
||||
FROM servers WHERE deleted_at IS NULL ORDER BY name`
|
||||
rows, err := s.db.QueryContext(ctx, q)
|
||||
if err != nil {
|
||||
@@ -33,8 +33,9 @@ func (s *PGStore) ListActiveServers(ctx context.Context) ([]Candidate, error) {
|
||||
c Candidate
|
||||
owner sql.NullString
|
||||
w3, w1 sql.NullTime
|
||||
retire sql.NullTime
|
||||
)
|
||||
if err := rows.Scan(&c.Name, &owner, &c.LastActiveAt, &w3, &w1); err != nil {
|
||||
if err := rows.Scan(&c.Name, &owner, &c.LastActiveAt, &w3, &w1, &retire, &c.RetireDelete); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.OwnerID = owner.String
|
||||
@@ -44,18 +45,21 @@ func (s *PGStore) ListActiveServers(ctx context.Context) ([]Candidate, error) {
|
||||
if w1.Valid {
|
||||
c.Warned1dAt = w1.Time
|
||||
}
|
||||
if retire.Valid {
|
||||
c.RetireRequestedAt = retire.Time
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *PGStore) FreshBackup(ctx context.Context, server string, since time.Time) (Fresh, bool, error) {
|
||||
// Only the reaper's own archives count, and only those taken since the
|
||||
// current owner claimed the server: a manual backup may predate a panel edit
|
||||
// that did not move last_active_at, and an archive from before the claim is
|
||||
// the previous owner's world. One found corrupt is never reused.
|
||||
// Only the reaper's own archives count (an idle reap's or a retirement's),
|
||||
// and only those taken since the current owner claimed the server: a manual
|
||||
// backup may predate a panel edit that did not move last_active_at, and an
|
||||
// archive from before the claim is the previous owner's world. One found corrupt is never reused.
|
||||
const q = `SELECT b.id, b.backup_ref, COALESCE(b.sha256, ''), b.offsite_at IS NOT NULL FROM world_backups b
|
||||
WHERE b.server_name = $1 AND b.status = 'present' AND b.reason = 'inactive_15d' AND b.created_at >= $2
|
||||
WHERE b.server_name = $1 AND b.status = 'present' AND b.reason IN ('inactive_15d', 'released') AND b.created_at >= $2
|
||||
AND b.corrupt_at IS NULL
|
||||
AND b.created_at >= COALESCE((SELECT s.claimed_at FROM servers s WHERE s.name = $1 AND s.deleted_at IS NULL), '-infinity')
|
||||
ORDER BY b.offsite_at IS NOT NULL DESC, b.created_at DESC LIMIT 1`
|
||||
@@ -85,17 +89,36 @@ func (s *PGStore) InsertBackup(ctx context.Context, rec BackupRecord) error {
|
||||
// gated on that size and counts it. The wake allowlist is emptied in the same
|
||||
// statement: its players were vouched for by the owner being released, and an
|
||||
// ownerless server set to autostartPolicy=allowlist would otherwise stay
|
||||
// wakeable by them.
|
||||
// wakeable by them. A pending release is done with once the world is gone;
|
||||
// a pending deletion stays, so an idle reap that lands on a server an admin is
|
||||
// deleting leaves the deletion for the next run.
|
||||
func (s *PGStore) ReleaseWorld(ctx context.Context, name string, at time.Time) error {
|
||||
const q = `WITH released AS (
|
||||
UPDATE servers
|
||||
SET owner_id = NULL, last_active_at = $2, warned_3d_at = NULL, warned_1d_at = NULL
|
||||
SET owner_id = NULL, last_active_at = $2, warned_3d_at = NULL, warned_1d_at = NULL,
|
||||
retire_requested_at = CASE WHEN retire_delete THEN retire_requested_at END
|
||||
WHERE name = $1 AND deleted_at IS NULL RETURNING name)
|
||||
DELETE FROM server_allowlist WHERE server_name IN (SELECT name FROM released)`
|
||||
_, err := s.db.ExecContext(ctx, q, name, at)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteServerRow marks a server deleted once its MinecraftServer is gone. The
|
||||
// row stays (red line ②) and so do its backups (red line ③), recorded against
|
||||
// the name; the aliases go, which frees the subdomain, and the allowlist with
|
||||
// them. A create under the name later starts the row over (api.SeedServer).
|
||||
func (s *PGStore) DeleteServerRow(ctx context.Context, name string, at time.Time) error {
|
||||
const q = `WITH gone AS (
|
||||
UPDATE servers
|
||||
SET deleted_at = $2, owner_id = NULL, retire_requested_at = NULL, retire_delete = false,
|
||||
warned_3d_at = NULL, warned_1d_at = NULL
|
||||
WHERE name = $1 AND deleted_at IS NULL AND retire_delete RETURNING name),
|
||||
aliases AS (DELETE FROM server_aliases WHERE server_name IN (SELECT name FROM gone))
|
||||
DELETE FROM server_allowlist WHERE server_name IN (SELECT name FROM gone)`
|
||||
_, err := s.db.ExecContext(ctx, q, name, at)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *PGStore) RestartClock(ctx context.Context, name string, at time.Time) error {
|
||||
const q = `UPDATE servers SET last_active_at = $2, warned_3d_at = NULL, warned_1d_at = NULL
|
||||
WHERE name = $1 AND deleted_at IS NULL`
|
||||
@@ -123,8 +146,8 @@ func (s *PGStore) PresentBackupBytes(ctx context.Context) (int64, error) {
|
||||
|
||||
func (s *PGStore) EvictableBackups(ctx context.Context) ([]StoredBackup, error) {
|
||||
const q = `SELECT id, server_name, backup_ref, size_bytes, reason, COALESCE(sha256, '') FROM world_backups
|
||||
WHERE status = 'present' AND (reason <> 'inactive_15d' OR offsite_at IS NOT NULL)
|
||||
ORDER BY reason = 'inactive_15d', created_at ASC`
|
||||
WHERE status = 'present' AND (reason NOT IN ('inactive_15d', 'released') OR offsite_at IS NOT NULL)
|
||||
ORDER BY reason IN ('inactive_15d', 'released'), created_at ASC`
|
||||
return s.queryBackups(ctx, q)
|
||||
}
|
||||
|
||||
|
||||
+148
-21
@@ -43,11 +43,18 @@ const Day = 24 * time.Hour
|
||||
// (spec §18). It is a stable label, not a literal restatement of the deadline.
|
||||
const ReasonInactive = "inactive_15d"
|
||||
|
||||
// ReasonReleased is the world_backups.reason for the archive of a world whose
|
||||
// server its owner gave up or an admin deleted (PUT /servers/{name}/retirement).
|
||||
// Like an idle reap's, it is the world's copy after the volume is gone.
|
||||
const ReasonReleased = "released"
|
||||
|
||||
// Audit actions emitted by the reaper. The actor/source are a system identity
|
||||
// ("reaper") because no human Access email is in play here (spec §14).
|
||||
const (
|
||||
ActionReapWorld = "reap_world"
|
||||
ActionEvictBackup = "evict_backup_early"
|
||||
ActionReapWorld = "reap_world"
|
||||
ActionEvictBackup = "evict_backup_early"
|
||||
ActionReleaseWorld = "release_world"
|
||||
ActionDeleteServer = "delete_server"
|
||||
)
|
||||
|
||||
// ErrNotFound is returned by Cluster.Inspect when the MinecraftServer CRD for a
|
||||
@@ -151,6 +158,12 @@ type Candidate struct {
|
||||
LastActiveAt time.Time
|
||||
Warned3dAt time.Time // zero = not yet sent
|
||||
Warned1dAt time.Time // zero = not yet sent
|
||||
// RetireRequestedAt is when the owner gave the server up or an admin asked
|
||||
// for it to be deleted (zero = no request); RetireDelete marks a deletion.
|
||||
// Either is carried out on the next run, however recently the world was
|
||||
// played.
|
||||
RetireRequestedAt time.Time
|
||||
RetireDelete bool
|
||||
}
|
||||
|
||||
func (c Candidate) warnedAt(t Tier) time.Time {
|
||||
@@ -160,11 +173,25 @@ func (c Candidate) warnedAt(t Tier) time.Time {
|
||||
return c.Warned3dAt
|
||||
}
|
||||
|
||||
func (c Candidate) retiring() bool { return !c.RetireRequestedAt.IsZero() }
|
||||
|
||||
// worldSince is how recent an archive must be to hold the current world: taken
|
||||
// after the last join, and for a retirement after the request, so the world
|
||||
// archived is the one its owner left.
|
||||
func (c Candidate) worldSince() time.Time {
|
||||
if c.RetireRequestedAt.After(c.LastActiveAt) {
|
||||
return c.RetireRequestedAt
|
||||
}
|
||||
return c.LastActiveAt
|
||||
}
|
||||
|
||||
// ServerCRD is the slice of the MinecraftServer CRD the reaper needs: the
|
||||
// exemption flag (red line ①) and the world PVC to archive then delete.
|
||||
// exemption flag (red line ①), the world PVC to archive then delete, and the
|
||||
// object's uid, so a deletion removes the server that was inspected.
|
||||
type ServerCRD struct {
|
||||
Exempt bool
|
||||
PVC string
|
||||
UID string
|
||||
}
|
||||
|
||||
// BackupRecord is a world_backups insert. FormerOwner is captured so the
|
||||
@@ -233,9 +260,17 @@ type Store interface {
|
||||
|
||||
// ReleaseWorld is the post-delete business mutation: owner_id→NULL,
|
||||
// last_active_at→at (clock reset), warned_*→NULL. It does NOT delete the
|
||||
// row (red line ②).
|
||||
// row (red line ②). A pending release is done with; a pending deletion
|
||||
// stays for the next run to finish.
|
||||
ReleaseWorld(ctx context.Context, name string, at time.Time) error
|
||||
|
||||
// DeleteServerRow finishes an admin's deletion once the MinecraftServer is
|
||||
// gone: the row is marked deleted, which frees its name and subdomain, and
|
||||
// loses its owner, aliases and allowlist. Only a row with a pending deletion
|
||||
// is touched, so a server created again under the name is left alone. The
|
||||
// row itself stays, like every reaped server's (red line ②).
|
||||
DeleteServerRow(ctx context.Context, name string, at time.Time) error
|
||||
|
||||
// RestartClock sets last_active_at→at and clears warned_* on a server with
|
||||
// no world to reclaim, so it is not found idle again every run.
|
||||
RestartClock(ctx context.Context, name string, at time.Time) error
|
||||
@@ -295,6 +330,11 @@ type Cluster interface {
|
||||
WorldExists(ctx context.Context, pvc string) (bool, error)
|
||||
// DeletePVC deletes the world PersistentVolumeClaim.
|
||||
DeletePVC(ctx context.Context, pvc string) error
|
||||
// DeleteServer removes the MinecraftServer whose uid Inspect returned, and
|
||||
// with it what the operator made for it (StatefulSet, Service, Secret). The
|
||||
// world volume is not among them: it is deleted first. A server already gone
|
||||
// is not an error; one of the same name with another uid is left alone.
|
||||
DeleteServer(ctx context.Context, name, uid string) error
|
||||
}
|
||||
|
||||
// Warner delivers an impending-reap notice. It is optional and best-effort: a
|
||||
@@ -323,10 +363,15 @@ type Reaper struct {
|
||||
type Summary struct {
|
||||
Evaluated int
|
||||
WorldsReaped int
|
||||
Warned int
|
||||
// Released are servers given up by their owner (or released by an admin)
|
||||
// and ServersDeleted the ones an admin deleted, carried out this run.
|
||||
Released int
|
||||
ServersDeleted int
|
||||
Warned int
|
||||
// Skipped are servers the run failed on (archive, store, cluster or
|
||||
// capacity errors); their worlds are kept and retried next run. Exempt
|
||||
// servers and rows whose CRD is gone are not counted.
|
||||
// servers and rows whose CRD is gone are not counted, except a deletion
|
||||
// left with a world volume and no MinecraftServer to hold it by.
|
||||
Skipped int
|
||||
// StoreFull are the Skipped servers kept because the backup store was at
|
||||
// capacity and eviction could not make room.
|
||||
@@ -453,6 +498,11 @@ func (r *Reaper) evaluate(ctx context.Context, now time.Time, offs []time.Durati
|
||||
crd, err := r.Cluster.Inspect(ctx, c.Name)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
if c.RetireDelete {
|
||||
// An earlier run removed the MinecraftServer and stopped before
|
||||
// marking the row, or it was removed by hand.
|
||||
return r.forgetServer(ctx, now, c, sum)
|
||||
}
|
||||
// CRD gone but the row lingers — nothing safe to do; not a failure.
|
||||
r.log().Warn("reaper: CRD missing, skipping", "server", c.Name)
|
||||
return nil
|
||||
@@ -460,9 +510,18 @@ func (r *Reaper) evaluate(ctx context.Context, now time.Time, offs []time.Durati
|
||||
return fmt.Errorf("inspect: %w", err)
|
||||
}
|
||||
if crd.Exempt {
|
||||
// Red line ①: system servers (lobby/proxy) are never reaped.
|
||||
// Red line ①: system servers (lobby/proxy) are never reaped. felis-api
|
||||
// refuses to retire one, so a request here predates the flag.
|
||||
if c.retiring() {
|
||||
r.log().Warn("reaper: a system server is never given up or deleted; request ignored", "server", c.Name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if c.retiring() {
|
||||
// The owner gave the server up, or an admin is deleting it: the world
|
||||
// goes now, archived like an idle one, and no warning is owed.
|
||||
return r.reap(ctx, now, c, crd, sum)
|
||||
}
|
||||
|
||||
idle := now.Sub(c.LastActiveAt)
|
||||
if idle > r.Cfg.IdleBeforeReap {
|
||||
@@ -490,7 +549,7 @@ func (r *Reaper) reap(ctx context.Context, now time.Time, c Candidate, crd Serve
|
||||
return fmt.Errorf("look up world volume: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
return r.reapNoWorld(ctx, now, c, sum)
|
||||
return r.reapNoWorld(ctx, now, c, crd, sum)
|
||||
}
|
||||
|
||||
// §26 soft cap: free space before adding a backup. If the store cannot be
|
||||
@@ -510,7 +569,7 @@ func (r *Reaper) reap(ctx context.Context, now time.Time, c Candidate, crd Serve
|
||||
// world but failed before deleting the PVC, reuse that backup rather than
|
||||
// writing a duplicate. The world has not changed since last_active_at, so
|
||||
// any present backup created after it still describes the current world.
|
||||
fresh, ok, err := r.Store.FreshBackup(ctx, c.Name, c.LastActiveAt)
|
||||
fresh, ok, err := r.Store.FreshBackup(ctx, c.Name, c.worldSince())
|
||||
if err != nil {
|
||||
return fmt.Errorf("lookup fresh backup: %w", err)
|
||||
}
|
||||
@@ -537,13 +596,17 @@ func (r *Reaper) reap(ctx context.Context, now time.Time, c Candidate, crd Serve
|
||||
r.log().Warn("reaper: archive leaves out entries that are not plain files or directories",
|
||||
"server", c.Name, "count", len(a.Skipped), "first", a.Skipped[:min(len(a.Skipped), 5)])
|
||||
}
|
||||
reason := ReasonInactive
|
||||
if c.retiring() {
|
||||
reason = ReasonReleased
|
||||
}
|
||||
rec := BackupRecord{
|
||||
ID: r.id(),
|
||||
ServerName: c.Name,
|
||||
FormerOwner: c.OwnerID,
|
||||
BackupRef: string(a.Ref),
|
||||
SizeBytes: a.Size,
|
||||
Reason: ReasonInactive,
|
||||
Reason: reason,
|
||||
ExpiresAt: now.Add(r.Cfg.Retention),
|
||||
SHA256: a.SHA256,
|
||||
SkippedEntries: len(a.Skipped),
|
||||
@@ -577,16 +640,24 @@ func (r *Reaper) reap(ctx context.Context, now time.Time, c Candidate, crd Serve
|
||||
// the delete, so no duplicate archive is created.
|
||||
return fmt.Errorf("delete pvc: %w", err)
|
||||
}
|
||||
return r.finishReap(ctx, now, c, ref, sum)
|
||||
return r.finishReap(ctx, now, c, crd, ref, sum)
|
||||
}
|
||||
|
||||
// finishReap releases a world whose PVC is gone and records the reap.
|
||||
func (r *Reaper) finishReap(ctx context.Context, now time.Time, c Candidate, ref string, sum *Summary) error {
|
||||
if err := r.Store.ReleaseWorld(ctx, c.Name, now); err != nil {
|
||||
return fmt.Errorf("release world: %w", err)
|
||||
}
|
||||
if err := r.Store.Audit(ctx, AuditRecord{Action: ActionReapWorld, ServerName: c.Name, FormerOwner: c.OwnerID}); err != nil {
|
||||
r.log().Error("reaper: audit reap_world failed", "server", c.Name, "err", err)
|
||||
// finishReap releases a world whose PVC is gone and records the reap. A
|
||||
// retirement is finished with it: the server is released, or for a deletion
|
||||
// removed.
|
||||
func (r *Reaper) finishReap(ctx context.Context, now time.Time, c Candidate, crd ServerCRD, ref string, sum *Summary) error {
|
||||
if c.retiring() {
|
||||
if err := r.retire(ctx, now, c, crd, sum); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := r.Store.ReleaseWorld(ctx, c.Name, now); err != nil {
|
||||
return fmt.Errorf("release world: %w", err)
|
||||
}
|
||||
if err := r.Store.Audit(ctx, AuditRecord{Action: ActionReapWorld, ServerName: c.Name, FormerOwner: c.OwnerID}); err != nil {
|
||||
r.log().Error("reaper: audit reap_world failed", "server", c.Name, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
sum.WorldsReaped++
|
||||
@@ -604,13 +675,17 @@ func (r *Reaper) finishReap(ctx context.Context, now time.Time, c Candidate, ref
|
||||
// no world to reclaim: an owner who never started the server gives it up
|
||||
// (nothing to back up), and an unowned one — typically a world reaped earlier —
|
||||
// only has its clock restarted, so it is not reaped over and over.
|
||||
func (r *Reaper) reapNoWorld(ctx context.Context, now time.Time, c Candidate, sum *Summary) error {
|
||||
fresh, ok, err := r.Store.FreshBackup(ctx, c.Name, c.LastActiveAt)
|
||||
func (r *Reaper) reapNoWorld(ctx context.Context, now time.Time, c Candidate, crd ServerCRD, sum *Summary) error {
|
||||
fresh, ok, err := r.Store.FreshBackup(ctx, c.Name, c.worldSince())
|
||||
if err != nil {
|
||||
return fmt.Errorf("lookup fresh backup: %w", err)
|
||||
}
|
||||
if ok {
|
||||
return r.finishReap(ctx, now, c, fresh.Ref, sum)
|
||||
return r.finishReap(ctx, now, c, crd, fresh.Ref, sum)
|
||||
}
|
||||
if c.retiring() {
|
||||
// A retired server that never had a world has nothing to archive.
|
||||
return r.retire(ctx, now, c, crd, sum)
|
||||
}
|
||||
if c.OwnerID == "" {
|
||||
if err := r.Store.RestartClock(ctx, c.Name, now); err != nil {
|
||||
@@ -628,6 +703,58 @@ func (r *Reaper) reapNoWorld(ctx context.Context, now time.Time, c Candidate, su
|
||||
return nil
|
||||
}
|
||||
|
||||
// retire carries out a retirement once the world is archived and its volume
|
||||
// deleted, or there was none: a given-up server is released for someone else to
|
||||
// claim, and a deleted one loses its MinecraftServer and then its row.
|
||||
func (r *Reaper) retire(ctx context.Context, now time.Time, c Candidate, crd ServerCRD, sum *Summary) error {
|
||||
if !c.RetireDelete {
|
||||
if err := r.Store.ReleaseWorld(ctx, c.Name, now); err != nil {
|
||||
return fmt.Errorf("release world: %w", err)
|
||||
}
|
||||
if err := r.Store.Audit(ctx, AuditRecord{Action: ActionReleaseWorld, ServerName: c.Name, FormerOwner: c.OwnerID}); err != nil {
|
||||
r.log().Error("reaper: audit release_world failed", "server", c.Name, "err", err)
|
||||
}
|
||||
sum.Released++
|
||||
r.log().Info("reaper: server given up and released", "server", c.Name, "former_owner", c.OwnerID)
|
||||
return nil
|
||||
}
|
||||
// The row goes last: a failure in between leaves a row that still asks for
|
||||
// its deletion, and the next run finishes it (forgetServer).
|
||||
if err := r.Cluster.DeleteServer(ctx, c.Name, crd.UID); err != nil {
|
||||
return fmt.Errorf("delete server: %w", err)
|
||||
}
|
||||
return r.deleteRow(ctx, now, c, sum)
|
||||
}
|
||||
|
||||
// forgetServer finishes the deletion of a server whose MinecraftServer is gone.
|
||||
// A world volume left behind (the StatefulSet retains claims, so removing the
|
||||
// MinecraftServer by hand leaves it) is never deleted unarchived, and without the
|
||||
// MinecraftServer the reaper cannot hold it still to archive it: an operator
|
||||
// takes it from there, and the run reports the server until then.
|
||||
func (r *Reaper) forgetServer(ctx context.Context, now time.Time, c Candidate, sum *Summary) error {
|
||||
pvc := WorldPVCName(c.Name)
|
||||
exists, err := r.Cluster.WorldExists(ctx, pvc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("look up world volume: %w", err)
|
||||
}
|
||||
if exists {
|
||||
return fmt.Errorf("its MinecraftServer is gone but world volume %s is still there; archive and remove it by hand to finish the deletion", pvc)
|
||||
}
|
||||
return r.deleteRow(ctx, now, c, sum)
|
||||
}
|
||||
|
||||
func (r *Reaper) deleteRow(ctx context.Context, now time.Time, c Candidate, sum *Summary) error {
|
||||
if err := r.Store.DeleteServerRow(ctx, c.Name, now); err != nil {
|
||||
return fmt.Errorf("mark server deleted: %w", err)
|
||||
}
|
||||
if err := r.Store.Audit(ctx, AuditRecord{Action: ActionDeleteServer, ServerName: c.Name, FormerOwner: c.OwnerID}); err != nil {
|
||||
r.log().Error("reaper: audit delete_server failed", "server", c.Name, "err", err)
|
||||
}
|
||||
sum.ServersDeleted++
|
||||
r.log().Info("reaper: server deleted", "server", c.Name, "former_owner", c.OwnerID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureCapacity frees the backup store down under MaxLocalBytes by evicting the
|
||||
// oldest present backups early. Early eviction is destructive (it removes
|
||||
// not-yet-expired backups), so each eviction is alerted and audited. It returns
|
||||
|
||||
@@ -115,6 +115,9 @@ type fakeCluster struct {
|
||||
held map[string]bool // servers held right now
|
||||
holds []string
|
||||
lost context.CancelCauseFunc
|
||||
|
||||
deletedServers []string // name/uid of each DeleteServer
|
||||
deleteServerErr error
|
||||
}
|
||||
|
||||
func (c *fakeCluster) HoldWorld(ctx context.Context, name string) (context.Context, func(), error) {
|
||||
@@ -167,10 +170,24 @@ func (c *fakeCluster) DeletePVC(ctx context.Context, pvc string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *fakeCluster) DeleteServer(ctx context.Context, name, uid string) error {
|
||||
if c.deleteServerErr != nil {
|
||||
return c.deleteServerErr
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
c.deletedServers = append(c.deletedServers, name+"/"+uid)
|
||||
delete(c.crds, name)
|
||||
c.rec.add("deleteServer")
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- fake Store -----------------------------------------------------------
|
||||
|
||||
type fakeBackup struct {
|
||||
id, server, ref string
|
||||
owner string
|
||||
reason string
|
||||
size int64
|
||||
status string // present | deleted
|
||||
@@ -190,6 +207,8 @@ type fakeStore struct {
|
||||
backups []*fakeBackup
|
||||
audits []AuditRecord
|
||||
released []string
|
||||
deleted []string
|
||||
deleteErr error
|
||||
listErr error
|
||||
insertErr error
|
||||
liveErr error
|
||||
@@ -210,7 +229,7 @@ func (s *fakeStore) ListActiveServers(context.Context) ([]Candidate, error) {
|
||||
func (s *fakeStore) FreshBackup(_ context.Context, server string, since time.Time) (Fresh, bool, error) {
|
||||
var found *fakeBackup
|
||||
for _, b := range s.backups {
|
||||
if b.server == server && b.status == "present" && b.reason == ReasonInactive && !b.createdAt.Before(since) && b.corruptAt.IsZero() {
|
||||
if b.server == server && b.status == "present" && (b.reason == ReasonInactive || b.reason == ReasonReleased) && !b.createdAt.Before(since) && b.corruptAt.IsZero() {
|
||||
if found == nil || (b.offsite && !found.offsite) {
|
||||
found = b
|
||||
}
|
||||
@@ -227,7 +246,7 @@ func (s *fakeStore) InsertBackup(_ context.Context, rec BackupRecord) error {
|
||||
return s.insertErr
|
||||
}
|
||||
s.backups = append(s.backups, &fakeBackup{
|
||||
id: rec.ID, server: rec.ServerName, ref: rec.BackupRef, reason: rec.Reason, size: rec.SizeBytes,
|
||||
id: rec.ID, server: rec.ServerName, owner: rec.FormerOwner, ref: rec.BackupRef, reason: rec.Reason, size: rec.SizeBytes,
|
||||
status: "present", createdAt: s.clock, expires: rec.ExpiresAt, sha: rec.SHA256, skipped: rec.SkippedEntries,
|
||||
})
|
||||
s.rec.add("insert")
|
||||
@@ -240,11 +259,36 @@ func (s *fakeStore) ReleaseWorld(_ context.Context, name string, at time.Time) e
|
||||
c.LastActiveAt = at
|
||||
c.Warned3dAt = time.Time{}
|
||||
c.Warned1dAt = time.Time{}
|
||||
if !c.RetireDelete {
|
||||
c.RetireRequestedAt = time.Time{}
|
||||
}
|
||||
s.released = append(s.released, name)
|
||||
s.rec.add("release")
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteServerRow drops the row from the listing, as deleted_at does, and only
|
||||
// when it asks for its deletion (PGStore's condition).
|
||||
func (s *fakeStore) DeleteServerRow(_ context.Context, name string, _ time.Time) error {
|
||||
if s.deleteErr != nil {
|
||||
return s.deleteErr
|
||||
}
|
||||
c := s.byName[name]
|
||||
if c == nil || !c.RetireDelete {
|
||||
return nil
|
||||
}
|
||||
delete(s.byName, name)
|
||||
for i, n := range s.order {
|
||||
if n == name {
|
||||
s.order = append(s.order[:i], s.order[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
s.deleted = append(s.deleted, name)
|
||||
s.rec.add("deleteRow")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakeStore) RestartClock(_ context.Context, name string, at time.Time) error {
|
||||
c := s.byName[name]
|
||||
c.LastActiveAt = at
|
||||
@@ -277,12 +321,12 @@ func (s *fakeStore) PresentBackupBytes(context.Context) (int64, error) {
|
||||
func (s *fakeStore) EvictableBackups(context.Context) ([]StoredBackup, error) {
|
||||
var ps []*fakeBackup
|
||||
for _, b := range s.backups {
|
||||
if b.status == "present" && (b.reason != ReasonInactive || b.offsite) {
|
||||
if b.status == "present" && ((b.reason != ReasonInactive && b.reason != ReasonReleased) || b.offsite) {
|
||||
ps = append(ps, b)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(ps, func(i, j int) bool {
|
||||
if ri, rj := ps[i].reason == ReasonInactive, ps[j].reason == ReasonInactive; ri != rj {
|
||||
if ri, rj := isArchive(ps[i].reason), isArchive(ps[j].reason); ri != rj {
|
||||
return rj
|
||||
}
|
||||
return ps[i].createdAt.Before(ps[j].createdAt)
|
||||
@@ -294,6 +338,8 @@ func (s *fakeStore) EvictableBackups(context.Context) ([]StoredBackup, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func isArchive(reason string) bool { return reason == ReasonInactive || reason == ReasonReleased }
|
||||
|
||||
func (s *fakeStore) ListExpiredBackups(_ context.Context, now time.Time) ([]StoredBackup, error) {
|
||||
var out []StoredBackup
|
||||
for _, b := range s.backups {
|
||||
@@ -397,7 +443,7 @@ func newReaper(cfg Config, cands ...Candidate) (*Reaper, *fakeStore, *fakeCluste
|
||||
cc := cands[i]
|
||||
st.byName[cc.Name] = &cc
|
||||
st.order = append(st.order, cc.Name)
|
||||
cl.crds[cc.Name] = ServerCRD{PVC: "world-" + cc.Name + "-0"}
|
||||
cl.crds[cc.Name] = ServerCRD{PVC: "world-" + cc.Name + "-0", UID: "uid-" + cc.Name}
|
||||
}
|
||||
ar := &fakeArchiver{rec: rec}
|
||||
r := &Reaper{
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
package reaper
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// retiring is a candidate whose owner gave it up an hour ago, a day after they
|
||||
// last played: nowhere near idle, so only the request makes the reaper act.
|
||||
func retiring(name, owner string, del bool) Candidate {
|
||||
return Candidate{Name: name, OwnerID: owner, LastActiveAt: idleBy(Day),
|
||||
RetireRequestedAt: idleBy(time.Hour), RetireDelete: del}
|
||||
}
|
||||
|
||||
// A server its owner gave up goes on the next run however recently it was
|
||||
// played: archived as a "released" backup under the owner, volume deleted, and
|
||||
// released for someone else to claim, with the request done with.
|
||||
func TestRetireReleaseArchivesThenReleases(t *testing.T) {
|
||||
r, st, cl, _ := newReaper(DefaultConfig(), retiring("alpha", "user-7", false))
|
||||
|
||||
sum := mustRun(t, r)
|
||||
|
||||
want := []string{"hold", "archive", "insert", "deletePVC", "release", "audit:" + ActionReleaseWorld, "unhold"}
|
||||
if !reflect.DeepEqual(st.rec.events, want) {
|
||||
t.Fatalf("call order = %v, want %v", st.rec.events, want)
|
||||
}
|
||||
if len(st.backups) != 1 || st.backups[0].reason != ReasonReleased || st.backups[0].owner != "user-7" {
|
||||
t.Fatalf("backups = %+v, want one released archive recorded against user-7", st.backups)
|
||||
}
|
||||
if want := testNow.Add(DefaultConfig().Retention); !st.backups[0].expires.Equal(want) {
|
||||
t.Fatalf("archive expires %v, want %v", st.backups[0].expires, want)
|
||||
}
|
||||
if len(st.audits) != 1 || st.audits[0].FormerOwner != "user-7" {
|
||||
t.Fatalf("audits = %+v", st.audits)
|
||||
}
|
||||
c := st.byName["alpha"]
|
||||
if c.OwnerID != "" || !c.RetireRequestedAt.IsZero() {
|
||||
t.Fatalf("after release: %+v, want no owner and no pending request", c)
|
||||
}
|
||||
if len(cl.deletedServers) != 0 || len(st.deleted) != 0 {
|
||||
t.Fatalf("a release deleted the server: %v %v", cl.deletedServers, st.deleted)
|
||||
}
|
||||
if sum.WorldsReaped != 1 || sum.Released != 1 || sum.ServersDeleted != 0 || sum.Skipped != 0 {
|
||||
t.Fatalf("summary = %+v", sum)
|
||||
}
|
||||
}
|
||||
|
||||
// An admin's deletion archives the world the same way, then removes the
|
||||
// MinecraftServer that was inspected (by uid) and only after it the row.
|
||||
func TestRetireDeleteRemovesServerThenRow(t *testing.T) {
|
||||
r, st, cl, _ := newReaper(DefaultConfig(), retiring("beta", "user-2", true))
|
||||
|
||||
sum := mustRun(t, r)
|
||||
|
||||
want := []string{"hold", "archive", "insert", "deletePVC", "deleteServer", "deleteRow", "audit:" + ActionDeleteServer, "unhold"}
|
||||
if !reflect.DeepEqual(st.rec.events, want) {
|
||||
t.Fatalf("call order = %v, want %v", st.rec.events, want)
|
||||
}
|
||||
if !reflect.DeepEqual(cl.deletedServers, []string{"beta/uid-beta"}) {
|
||||
t.Fatalf("deleted servers = %v, want [beta/uid-beta]", cl.deletedServers)
|
||||
}
|
||||
if !reflect.DeepEqual(st.deleted, []string{"beta"}) || len(st.released) != 0 {
|
||||
t.Fatalf("rows deleted %v released %v", st.deleted, st.released)
|
||||
}
|
||||
if len(st.backups) != 1 || st.backups[0].reason != ReasonReleased || st.backups[0].owner != "user-2" {
|
||||
t.Fatalf("backups = %+v", st.backups)
|
||||
}
|
||||
if sum.WorldsReaped != 1 || sum.ServersDeleted != 1 || sum.Released != 0 {
|
||||
t.Fatalf("summary = %+v", sum)
|
||||
}
|
||||
}
|
||||
|
||||
// The archive a retirement leaves must be of the world as its owner left it: one
|
||||
// taken before the request (a reap waiting for its off-site copy, say) is not
|
||||
// reused, one taken after it is.
|
||||
func TestRetireReusesOnlyAnArchiveTakenAfterTheRequest(t *testing.T) {
|
||||
c := retiring("gamma", "user-3", false)
|
||||
|
||||
r, st, _, ar := newReaper(DefaultConfig(), c)
|
||||
st.backups = append(st.backups, &fakeBackup{id: "old", server: "gamma", ref: "ref-old", reason: ReasonInactive,
|
||||
status: "present", createdAt: c.RetireRequestedAt.Add(-1), sha: "sha-ref-old"})
|
||||
mustRun(t, r)
|
||||
if ar.archives != 1 {
|
||||
t.Fatalf("archives = %d: an archive older than the request was reused", ar.archives)
|
||||
}
|
||||
|
||||
r, st, _, ar = newReaper(DefaultConfig(), c)
|
||||
st.backups = append(st.backups, &fakeBackup{id: "new", server: "gamma", ref: "ref-new", reason: ReasonReleased,
|
||||
status: "present", createdAt: c.RetireRequestedAt.Add(1), sha: "sha-ref-new"})
|
||||
mustRun(t, r)
|
||||
if ar.archives != 0 {
|
||||
t.Fatalf("archives = %d: the archive taken after the request was not reused", ar.archives)
|
||||
}
|
||||
}
|
||||
|
||||
// A retired server that never had a world has nothing to archive: it is
|
||||
// released (an unowned one too, where an idle one only restarts its clock), or
|
||||
// deleted.
|
||||
func TestRetireWithNoWorld(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
c Candidate
|
||||
want []string
|
||||
check func(*testing.T, *fakeStore, *fakeCluster, Summary)
|
||||
}{
|
||||
{"owned release", retiring("a", "user-1", false),
|
||||
[]string{"hold", "release", "audit:" + ActionReleaseWorld, "unhold"},
|
||||
func(t *testing.T, st *fakeStore, _ *fakeCluster, sum Summary) {
|
||||
if sum.Released != 1 || sum.WorldsReaped != 0 {
|
||||
t.Errorf("summary = %+v", sum)
|
||||
}
|
||||
}},
|
||||
{"unowned release", retiring("a", "", false),
|
||||
[]string{"hold", "release", "audit:" + ActionReleaseWorld, "unhold"},
|
||||
func(t *testing.T, st *fakeStore, _ *fakeCluster, _ Summary) {
|
||||
if !st.byName["a"].RetireRequestedAt.IsZero() {
|
||||
t.Errorf("request still pending: %+v", st.byName["a"])
|
||||
}
|
||||
}},
|
||||
{"delete", retiring("a", "user-1", true),
|
||||
[]string{"hold", "deleteServer", "deleteRow", "audit:" + ActionDeleteServer, "unhold"},
|
||||
func(t *testing.T, _ *fakeStore, cl *fakeCluster, sum Summary) {
|
||||
if sum.ServersDeleted != 1 || sum.WorldsReaped != 0 || len(cl.deletedServers) != 1 {
|
||||
t.Errorf("summary = %+v, deleted %v", sum, cl.deletedServers)
|
||||
}
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
r, st, cl, ar := newReaper(DefaultConfig(), tc.c)
|
||||
cl.noWorld = map[string]bool{"world-a-0": true}
|
||||
sum := mustRun(t, r)
|
||||
if !reflect.DeepEqual(st.rec.events, tc.want) {
|
||||
t.Fatalf("call order = %v, want %v", st.rec.events, tc.want)
|
||||
}
|
||||
if ar.archives != 0 || cl.deletePVCCalls != 0 {
|
||||
t.Fatalf("archived %d, deleted %d PVCs of a server with no world", ar.archives, cl.deletePVCCalls)
|
||||
}
|
||||
tc.check(t, st, cl, sum)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A deletion interrupted after the MinecraftServer went (or one removed by hand)
|
||||
// is finished from the row alone, unless a world volume is still there: that is
|
||||
// never deleted unarchived, and the run reports it. A release request with no
|
||||
// MinecraftServer is left alone as before.
|
||||
func TestRetireDeleteWithTheServerGone(t *testing.T) {
|
||||
r, st, cl, _ := newReaper(DefaultConfig(), retiring("gone", "user-4", true))
|
||||
delete(cl.crds, "gone")
|
||||
cl.noWorld = map[string]bool{"world-gone-0": true}
|
||||
sum := mustRun(t, r)
|
||||
if !reflect.DeepEqual(st.rec.events, []string{"deleteRow", "audit:" + ActionDeleteServer}) || sum.ServersDeleted != 1 {
|
||||
t.Fatalf("events %v, summary %+v", st.rec.events, sum)
|
||||
}
|
||||
|
||||
r, st, cl, _ = newReaper(DefaultConfig(), retiring("gone", "user-4", true))
|
||||
delete(cl.crds, "gone")
|
||||
sum = mustRun(t, r)
|
||||
if len(st.deleted) != 0 || sum.Skipped != 1 || sum.ServersDeleted != 0 {
|
||||
t.Fatalf("with its world volume left: deleted %v, summary %+v", st.deleted, sum)
|
||||
}
|
||||
|
||||
r, st, cl, _ = newReaper(DefaultConfig(), retiring("gone", "user-4", false))
|
||||
delete(cl.crds, "gone")
|
||||
sum = mustRun(t, r)
|
||||
if len(st.rec.events) != 0 || sum.Skipped != 0 {
|
||||
t.Fatalf("release with no server: events %v, summary %+v", st.rec.events, sum)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed MinecraftServer delete keeps the row asking for its deletion, and the
|
||||
// next run finishes it from the archive the first one left.
|
||||
func TestRetireDeleteServerFailureIsRetried(t *testing.T) {
|
||||
r, st, cl, ar := newReaper(DefaultConfig(), retiring("delta", "user-5", true))
|
||||
cl.deleteServerErr = errors.New("apiserver down")
|
||||
|
||||
sum := mustRun(t, r)
|
||||
if sum.Skipped != 1 || len(st.deleted) != 0 || st.byName["delta"] == nil || !st.byName["delta"].RetireDelete {
|
||||
t.Fatalf("after a failed delete: summary %+v, deleted %v, row %+v", sum, st.deleted, st.byName["delta"])
|
||||
}
|
||||
|
||||
cl.deleteServerErr = nil
|
||||
cl.noWorld = map[string]bool{"world-delta-0": true} // the first run deleted it
|
||||
sum = mustRun(t, r)
|
||||
if ar.archives != 1 || !reflect.DeepEqual(st.deleted, []string{"delta"}) || sum.ServersDeleted != 1 {
|
||||
t.Fatalf("retry: archives %d, deleted %v, summary %+v", ar.archives, st.deleted, sum)
|
||||
}
|
||||
}
|
||||
|
||||
// A system server is never given up, whatever its row says.
|
||||
func TestRetireExemptServerIgnored(t *testing.T) {
|
||||
r, st, cl, ar := newReaper(DefaultConfig(), retiring("lobby", "", true))
|
||||
cl.crds["lobby"] = ServerCRD{Exempt: true, PVC: "world-lobby-0", UID: "uid-lobby"}
|
||||
sum := mustRun(t, r)
|
||||
if len(st.rec.events) != 0 || ar.archives != 0 || sum.Skipped != 0 {
|
||||
t.Fatalf("exempt server touched: events %v, summary %+v", st.rec.events, sum)
|
||||
}
|
||||
}
|
||||
|
||||
// With the off-site copy required, a retirement waits for it like an idle reap.
|
||||
func TestRetireWaitsForTheOffsiteCopy(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
cfg.RequireOffsite = true
|
||||
r, st, cl, _ := newReaper(cfg, retiring("eps", "user-6", true))
|
||||
sum := mustRun(t, r)
|
||||
if sum.AwaitingOffsite != 1 || cl.deletePVCCalls != 0 || len(cl.deletedServers) != 0 || len(st.deleted) != 0 {
|
||||
t.Fatalf("summary %+v, deletePVC %d, servers %v", sum, cl.deletePVCCalls, cl.deletedServers)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
-- An owner can give a server up, and an admin can retire one for good
|
||||
-- (PUT /servers/{name}/retirement). The request is recorded here and carried out
|
||||
-- by the reaper, the one component that deletes a world: on its next run it
|
||||
-- archives the world, deletes the volume and releases the server, and with
|
||||
-- retire_delete it also removes the MinecraftServer and marks this row deleted.
|
||||
-- Until then the server stays stopped and cannot be woken or claimed.
|
||||
ALTER TABLE servers ADD COLUMN retire_requested_at timestamptz;
|
||||
ALTER TABLE servers ADD COLUMN retire_delete boolean NOT NULL DEFAULT false;
|
||||
Reference in new issue
Block a user