fix(servers): 主人可放弃服务器、管理员可删除服务器,由 reaper 归档世界后释放或移除
This commit is contained in:
55 files changed
+2774
-119
No files matched your search
@@ -647,32 +647,34 @@ func TestSeedServerReusedNameStartsClean(t *testing.T) {
|
||||
}
|
||||
past := time.Now().Add(-90 * 24 * time.Hour)
|
||||
exec(`UPDATE servers SET owner_id = $2, claimed_at = $3, last_active_at = $3, warned_3d_at = $3,
|
||||
warned_1d_at = $3, cached_phase = 'Running', created_at = $3, deleted_at = $3 WHERE name = $1`, name, u.ID, past)
|
||||
warned_1d_at = $3, cached_phase = 'Running', created_at = $3, deleted_at = $3,
|
||||
retire_requested_at = $3, retire_delete = true WHERE name = $1`, name, u.ID, past)
|
||||
exec(`INSERT INTO server_aliases (subdomain, server_name) VALUES ($1, $2)`, oldSub2, name)
|
||||
exec(`INSERT INTO server_allowlist (server_name, mc_uuid) VALUES ($1, $2)`, name, testUUID(t))
|
||||
|
||||
state := func() string {
|
||||
t.Helper()
|
||||
var owner, phase sql.NullString
|
||||
var claimed, w3, w1, deleted sql.NullTime
|
||||
var claimed, w3, w1, deleted, retireAt sql.NullTime
|
||||
var retireDelete bool
|
||||
var created, active time.Time
|
||||
var cpu, mem, stor, allow int
|
||||
var aliases string
|
||||
if err := db.QueryRowContext(ctx,
|
||||
`SELECT owner_id, claimed_at, warned_3d_at, warned_1d_at, cached_phase, deleted_at, created_at, last_active_at,
|
||||
cached_cpu_milli, cached_memory_mb, cached_storage_mb,
|
||||
cached_cpu_milli, cached_memory_mb, cached_storage_mb, retire_requested_at, retire_delete,
|
||||
(SELECT count(*) FROM server_allowlist WHERE server_name = s.name),
|
||||
(SELECT COALESCE(string_agg(subdomain, ',' ORDER BY subdomain), '') FROM server_aliases WHERE server_name = s.name)
|
||||
FROM servers s WHERE name = $1`, name).Scan(
|
||||
&owner, &claimed, &w3, &w1, &phase, &deleted, &created, &active, &cpu, &mem, &stor, &allow, &aliases); err != nil {
|
||||
&owner, &claimed, &w3, &w1, &phase, &deleted, &created, &active, &cpu, &mem, &stor, &retireAt, &retireDelete, &allow, &aliases); err != nil {
|
||||
t.Fatalf("read the servers row: %v", err)
|
||||
}
|
||||
recent := func(at time.Time) bool { return time.Since(at) < time.Hour }
|
||||
return fmt.Sprintf("owner=%v claimed=%v warned=%v/%v phase=%v deleted=%v fresh=%v/%v cache=%d/%d/%d allow=%d aliases=%s",
|
||||
return fmt.Sprintf("owner=%v claimed=%v warned=%v/%v phase=%v deleted=%v fresh=%v/%v cache=%d/%d/%d retire=%v/%v allow=%d aliases=%s",
|
||||
owner.Valid, claimed.Valid, w3.Valid, w1.Valid, phase.Valid, deleted.Valid, recent(created), recent(active),
|
||||
cpu, mem, stor, allow, strings.ReplaceAll(strings.ReplaceAll(aliases, oldSub2, "old2"), oldSub, "old"))
|
||||
cpu, mem, stor, retireAt.Valid, retireDelete, allow, strings.ReplaceAll(strings.ReplaceAll(aliases, oldSub2, "old2"), oldSub, "old"))
|
||||
}
|
||||
earlier := "owner=true claimed=true warned=true/true phase=true deleted=true fresh=false/false cache=1000/2048/10240 allow=1 aliases=old,old2"
|
||||
earlier := "owner=true claimed=true warned=true/true phase=true deleted=true fresh=false/false cache=1000/2048/10240 retire=true/true allow=1 aliases=old,old2"
|
||||
if got := state(); got != earlier {
|
||||
t.Fatalf("setup: %s, want %s", got, earlier)
|
||||
}
|
||||
@@ -689,7 +691,7 @@ func TestSeedServerReusedNameStartsClean(t *testing.T) {
|
||||
t.Fatalf("a refused seed changed the row: %s, want %s", got, earlier)
|
||||
}
|
||||
|
||||
clean := "owner=false claimed=false warned=false/false phase=false deleted=false fresh=true/true cache=2000/4096/20480 allow=0 aliases=" + newSub
|
||||
clean := "owner=false claimed=false warned=false/false phase=false deleted=false fresh=true/true cache=2000/4096/20480 retire=false/false allow=0 aliases=" + newSub
|
||||
for i := 0; i < 2; i++ { // a retried create lands on the same state
|
||||
if err := repo.SeedServer(ctx, name, newSub, 2000, 4096, 20480); err != nil {
|
||||
t.Fatalf("seed the new server (try %d): %v", i+1, err)
|
||||
|
||||
@@ -40,6 +40,11 @@ func (c *reclaimCluster) HoldWorld(ctx context.Context, _ string) (context.Conte
|
||||
|
||||
func (c *reclaimCluster) WorldExists(context.Context, string) (bool, error) { return true, nil }
|
||||
|
||||
// DeleteServer is only reached by a retirement, which these tests do not ask for.
|
||||
func (c *reclaimCluster) DeleteServer(_ context.Context, name, _ string) error {
|
||||
return errors.New("unexpected deletion of " + name)
|
||||
}
|
||||
|
||||
type reclaimArchiver struct{ archived []string }
|
||||
|
||||
func (a *reclaimArchiver) Archive(_ context.Context, server, _ string) (backup.Archived, error) {
|
||||
|
||||
@@ -0,0 +1,493 @@
|
||||
//go:build pgint
|
||||
|
||||
package pgint
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"felis.lolicon.best/internal/api"
|
||||
"felis.lolicon.best/internal/reaper"
|
||||
)
|
||||
|
||||
// ---- retirement (migration 0035) --------------------------------------------------
|
||||
|
||||
func retireColumns(t *testing.T, name string) (sql.NullTime, bool) {
|
||||
t.Helper()
|
||||
var at sql.NullTime
|
||||
var del bool
|
||||
if err := db.QueryRow(`SELECT retire_requested_at, retire_delete FROM servers WHERE name = $1`, name).Scan(&at, &del); err != nil {
|
||||
t.Fatalf("retirement of %s: %v", name, err)
|
||||
}
|
||||
return at, del
|
||||
}
|
||||
|
||||
// TestRetireRequestContract pins PUT and DELETE /servers/{name}/retirement at the
|
||||
// SQL: asking again keeps the first request's time, a deletion once asked for
|
||||
// stays one whoever asks after, only an admin takes a deletion back, and a
|
||||
// server that is gone is not found.
|
||||
func TestRetireRequestContract(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
owner := newUser(t, "user", "retire")
|
||||
name, sub := "rt-"+suffix(t), "rts-"+suffix(t)
|
||||
if err := repo.SeedServer(ctx, name, sub, 100, 128, 1); err != nil {
|
||||
t.Fatalf("SeedServer: %v", err)
|
||||
}
|
||||
mustExec(t, `UPDATE servers SET owner_id = $2 WHERE name = $1`, name, owner.ID)
|
||||
if rec, err := repo.ServerByName(ctx, name); err != nil || rec.Retire != nil {
|
||||
t.Fatalf("before any request: ServerByName = %+v, %v; want no retirement", rec, err)
|
||||
}
|
||||
|
||||
first, err := repo.RequestRetire(ctx, name, false)
|
||||
if err != nil || first.Delete || time.Since(first.RequestedAt) > time.Minute {
|
||||
t.Fatalf("RequestRetire = %+v, %v; want a release asked just now", first, err)
|
||||
}
|
||||
for _, ask := range []struct {
|
||||
del, want bool
|
||||
}{{false, false}, {true, true}, {false, true}} {
|
||||
st, err := repo.RequestRetire(ctx, name, ask.del)
|
||||
if err != nil || !st.RequestedAt.Equal(first.RequestedAt) || st.Delete != ask.want {
|
||||
t.Fatalf("RequestRetire(delete=%v) = %+v, %v; want the first time %v and delete=%v",
|
||||
ask.del, st, err, first.RequestedAt, ask.want)
|
||||
}
|
||||
}
|
||||
for how, get := range map[string]func() (*api.ServerRecord, error){
|
||||
"name": func() (*api.ServerRecord, error) { return repo.ServerByName(ctx, name) },
|
||||
"subdomain": func() (*api.ServerRecord, error) { return repo.ServerBySubdomain(ctx, sub) },
|
||||
} {
|
||||
rec, err := get()
|
||||
if err != nil || rec.Retire == nil || !rec.Retire.Delete || !rec.Retire.RequestedAt.Equal(first.RequestedAt) {
|
||||
t.Fatalf("server by %s = %+v, %v; want the pending deletion", how, rec, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := repo.CancelRetire(ctx, name, false); !errors.Is(err, api.ErrConflict) {
|
||||
t.Fatalf("the owner cancelling a deletion = %v, want ErrConflict", err)
|
||||
}
|
||||
if at, del := retireColumns(t, name); !at.Valid || !at.Time.Equal(first.RequestedAt) || !del {
|
||||
t.Fatalf("a refused cancel changed the request: at=%v delete=%v", at, del)
|
||||
}
|
||||
if err := repo.CancelRetire(ctx, name, true); err != nil {
|
||||
t.Fatalf("an admin cancelling a deletion: %v", err)
|
||||
}
|
||||
if at, del := retireColumns(t, name); at.Valid || del {
|
||||
t.Fatalf("after the cancel: at=%v delete=%v; want no request", at, del)
|
||||
}
|
||||
if rec, err := repo.ServerByName(ctx, name); err != nil || rec.Retire != nil {
|
||||
t.Fatalf("after the cancel: ServerByName = %+v, %v", rec, err)
|
||||
}
|
||||
|
||||
// A give-up is the owner's to take back, and a new one starts its own clock.
|
||||
again, err := repo.RequestRetire(ctx, name, false)
|
||||
if err != nil || again.Delete || !again.RequestedAt.After(first.RequestedAt) {
|
||||
t.Fatalf("a new request = %+v, %v; want a release after %v", again, err, first.RequestedAt)
|
||||
}
|
||||
if err := repo.CancelRetire(ctx, name, false); err != nil {
|
||||
t.Fatalf("the owner cancelling a give-up: %v", err)
|
||||
}
|
||||
if at, del := retireColumns(t, name); at.Valid || del {
|
||||
t.Fatalf("after the owner's cancel: at=%v delete=%v", at, del)
|
||||
}
|
||||
|
||||
gone := "rtg-" + suffix(t)
|
||||
seedOwnedServer(t, gone, owner.ID, true)
|
||||
for _, n := range []string{gone, "rt-none-" + suffix(t)} {
|
||||
if st, err := repo.RequestRetire(ctx, n, true); !errors.Is(err, api.ErrNotFound) {
|
||||
t.Fatalf("RequestRetire(%s) = %+v, %v; want ErrNotFound", n, st, err)
|
||||
}
|
||||
if err := repo.CancelRetire(ctx, n, true); !errors.Is(err, api.ErrNotFound) {
|
||||
t.Fatalf("CancelRetire(%s) = %v; want ErrNotFound", n, err)
|
||||
}
|
||||
}
|
||||
if at, _ := retireColumns(t, gone); at.Valid {
|
||||
t.Fatalf("a deleted server's row took a request")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetiringServerIsNotClaimable: a server an admin is releasing or deleting
|
||||
// cannot be claimed, even unowned, and the claim lists say so; only its owner is
|
||||
// told of the request, and the admin fleet read sees it on every row.
|
||||
func TestRetiringServerIsNotClaimable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
u := newUser(t, "user", "rtclaim")
|
||||
other := newUser(t, "user", "rtclaim-other")
|
||||
sfx := suffix(t)
|
||||
free, freeRetiring, mine, mineRetiring, theirs := "rcf-"+sfx, "rcr-"+sfx, "rco-"+sfx, "rcm-"+sfx, "rct-"+sfx
|
||||
for _, n := range []string{free, freeRetiring} {
|
||||
mustExec(t, `INSERT INTO servers (name, cached_cpu_milli, cached_memory_mb, cached_storage_mb) VALUES ($1, 100, 128, 1)`, n)
|
||||
}
|
||||
seedOwnedServer(t, mine, u.ID, false)
|
||||
seedOwnedServer(t, mineRetiring, u.ID, false)
|
||||
seedOwnedServer(t, theirs, other.ID, false)
|
||||
released, err := repo.RequestRetire(ctx, freeRetiring, false)
|
||||
if err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", freeRetiring, err)
|
||||
}
|
||||
deleting, err := repo.RequestRetire(ctx, mineRetiring, true)
|
||||
if err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", mineRetiring, err)
|
||||
}
|
||||
if _, err := repo.RequestRetire(ctx, theirs, false); err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", theirs, err)
|
||||
}
|
||||
|
||||
if ok, err := repo.ClaimServer(ctx, freeRetiring, u.ID); err != nil || ok {
|
||||
t.Fatalf("claiming a server being released = (%v, %v), want (false, nil)", ok, err)
|
||||
}
|
||||
if o := serverOwner(t, freeRetiring); o != "" {
|
||||
t.Fatalf("the refused claim left owner %q", o)
|
||||
}
|
||||
|
||||
views, err := repo.MyServers(ctx, u.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("MyServers: %v", err)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, v := range views {
|
||||
if strings.HasSuffix(v.Name, sfx) {
|
||||
retiring := "none"
|
||||
if v.Retiring != nil {
|
||||
retiring = fmt.Sprintf("delete=%v", v.Retiring.Delete)
|
||||
if !v.Retiring.RequestedAt.Equal(deleting.RequestedAt) {
|
||||
retiring += " at another time"
|
||||
}
|
||||
}
|
||||
got[v.Name] = fmt.Sprintf("owned=%v claimable=%v retiring=%s", v.Owned, v.Claimable, retiring)
|
||||
}
|
||||
}
|
||||
want := map[string]string{
|
||||
free: "owned=false claimable=true retiring=none",
|
||||
freeRetiring: "owned=false claimable=false retiring=none",
|
||||
mine: "owned=true claimable=false retiring=none",
|
||||
mineRetiring: "owned=true claimable=false retiring=delete=true",
|
||||
}
|
||||
if fmt.Sprint(got) != fmt.Sprint(want) {
|
||||
t.Fatalf("MyServers = %v\nwant %v", got, want)
|
||||
}
|
||||
|
||||
owners, err := repo.ServerOwners(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ServerOwners: %v", err)
|
||||
}
|
||||
for n, w := range map[string]*api.RetireState{free: nil, mine: nil, freeRetiring: &released, mineRetiring: &deleting} {
|
||||
r := owners[n].Retire
|
||||
if (r == nil) != (w == nil) || (r != nil && (r.Delete != w.Delete || !r.RequestedAt.Equal(w.RequestedAt))) {
|
||||
t.Errorf("ServerOwners[%s].Retire = %+v, want %+v", n, r, w)
|
||||
}
|
||||
}
|
||||
if r := owners[theirs].Retire; r == nil || r.Delete {
|
||||
t.Errorf("ServerOwners[%s].Retire = %+v, want another owner's give-up", theirs, r)
|
||||
}
|
||||
|
||||
// Taken back, the server may be claimed again.
|
||||
if err := repo.CancelRetire(ctx, freeRetiring, true); err != nil {
|
||||
t.Fatalf("CancelRetire: %v", err)
|
||||
}
|
||||
if ok, err := repo.ClaimServer(ctx, freeRetiring, u.ID); err != nil || !ok {
|
||||
t.Fatalf("claim after the cancel = (%v, %v)", ok, err)
|
||||
}
|
||||
for _, n := range []string{mineRetiring, theirs} {
|
||||
if err := repo.CancelRetire(ctx, n, true); err != nil {
|
||||
t.Fatalf("CancelRetire(%s): %v", n, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetireReaperStore pins the reaper's reads and writes of a retirement: the
|
||||
// request reaches its candidates, a retirement's archive holds the world as an
|
||||
// idle reap's does and is never evicted while it is the only copy, releasing the
|
||||
// world finishes a give-up and leaves a deletion for the next run, and marking
|
||||
// the row deleted takes only a server asked to be deleted.
|
||||
func TestRetireReaperStore(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := reaper.NewPGStore(db)
|
||||
u := newUser(t, "user", "rtstore")
|
||||
sfx := suffix(t)
|
||||
given, doomed, plain := "rsg-"+sfx, "rsd-"+sfx, "rsp-"+sfx
|
||||
for _, n := range []string{given, doomed, plain} {
|
||||
seedOwnedServer(t, n, u.ID, false)
|
||||
}
|
||||
givenAt, err := repo.RequestRetire(ctx, given, false)
|
||||
if err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", given, err)
|
||||
}
|
||||
doomedAt, err := repo.RequestRetire(ctx, doomed, true)
|
||||
if err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", doomed, err)
|
||||
}
|
||||
|
||||
cands, err := st.ListActiveServers(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListActiveServers: %v", err)
|
||||
}
|
||||
seen := map[string]reaper.Candidate{}
|
||||
for _, c := range cands {
|
||||
seen[c.Name] = c
|
||||
}
|
||||
for n, w := range map[string]api.RetireState{given: givenAt, doomed: doomedAt, plain: {}} {
|
||||
c, ok := seen[n]
|
||||
if !ok || !c.RetireRequestedAt.Equal(w.RequestedAt) || c.RetireDelete != w.Delete {
|
||||
t.Errorf("candidate %s = %+v (listed %v); want retirement %+v", n, c, ok, w)
|
||||
}
|
||||
}
|
||||
|
||||
// A retirement's archive is the reaper's own: it holds the world, and it is
|
||||
// kept like an idle reap's until it has its off-site copy.
|
||||
now := time.Now()
|
||||
backup := func(server, id, reason string, created time.Time, offsite bool) {
|
||||
t.Helper()
|
||||
var offsiteAt any
|
||||
if offsite {
|
||||
offsiteAt = created
|
||||
}
|
||||
mustExec(t, `INSERT INTO world_backups (id, server_name, backup_ref, size_bytes, reason, status, created_at, expires_at, offsite_at)
|
||||
VALUES ($1, $2, $3, 1, $4, 'present', $5, $6, $7)`,
|
||||
id, server, "/archives/"+id+".tar.gz", reason, created, created.Add(90*reaper.Day), offsiteAt)
|
||||
}
|
||||
manualID, soleID, copiedID := "bk-rsm-"+sfx, "bk-rss-"+sfx, "bk-rsc-"+sfx
|
||||
backup(plain, manualID, "manual", now.Add(-time.Minute), false)
|
||||
if f, ok, err := st.FreshBackup(ctx, plain, now.Add(-time.Hour)); err != nil || ok {
|
||||
t.Fatalf("FreshBackup over a manual backup = (%+v, %v, %v); only the reaper's archives count", f, ok, err)
|
||||
}
|
||||
backup(plain, soleID, reaper.ReasonReleased, now.Add(-3*time.Minute), false)
|
||||
backup(plain, copiedID, reaper.ReasonReleased, now.Add(-2*time.Minute), true)
|
||||
f, ok, err := st.FreshBackup(ctx, plain, now.Add(-time.Hour))
|
||||
if err != nil || !ok || f.ID != copiedID || !f.Offsite {
|
||||
t.Fatalf("FreshBackup = (%+v, %v, %v); want the copied retirement archive %s", f, ok, err, copiedID)
|
||||
}
|
||||
if f, ok, err := st.FreshBackup(ctx, plain, now); err != nil || ok {
|
||||
t.Fatalf("FreshBackup since now = (%+v, %v, %v); an archive older than since holds no world", f, ok, err)
|
||||
}
|
||||
all, err := st.EvictableBackups(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("EvictableBackups: %v", err)
|
||||
}
|
||||
var order []string
|
||||
for _, b := range all {
|
||||
if b.ServerName == plain {
|
||||
order = append(order, b.ID)
|
||||
}
|
||||
}
|
||||
if strings.Join(order, ",") != manualID+","+copiedID {
|
||||
t.Fatalf("eviction order = %v; want %s, then %s, and never the only copy %s", order, manualID, copiedID, soleID)
|
||||
}
|
||||
|
||||
// Each server keeps an allowlist entry and an extra alias to see what goes.
|
||||
for _, n := range []string{given, doomed, plain} {
|
||||
mustExec(t, `INSERT INTO server_aliases (subdomain, server_name) VALUES ($1, $2)`, n+"-a", n)
|
||||
mustExec(t, `INSERT INTO server_allowlist (server_name, mc_uuid) VALUES ($1, $2)`, n, testUUID(t))
|
||||
}
|
||||
state := func(n string) string {
|
||||
t.Helper()
|
||||
var owner sql.NullString
|
||||
var deleted, retireAt sql.NullTime
|
||||
var del bool
|
||||
var aliases, allow int
|
||||
if err := db.QueryRowContext(ctx,
|
||||
`SELECT owner_id, deleted_at, retire_requested_at, retire_delete,
|
||||
(SELECT count(*) FROM server_aliases WHERE server_name = s.name),
|
||||
(SELECT count(*) FROM server_allowlist WHERE server_name = s.name)
|
||||
FROM servers s WHERE name = $1`, n).Scan(&owner, &deleted, &retireAt, &del, &aliases, &allow); err != nil {
|
||||
t.Fatalf("read %s: %v", n, err)
|
||||
}
|
||||
return fmt.Sprintf("owner=%v deleted=%v retire=%v/%v aliases=%d allow=%d",
|
||||
owner.Valid, deleted.Valid, retireAt.Valid, del, aliases, allow)
|
||||
}
|
||||
|
||||
// Only a server asked to be deleted loses its row.
|
||||
for _, n := range []string{given, plain} {
|
||||
before := state(n)
|
||||
if err := st.DeleteServerRow(ctx, n, now); err != nil {
|
||||
t.Fatalf("DeleteServerRow(%s): %v", n, err)
|
||||
}
|
||||
if got := state(n); got != before {
|
||||
t.Fatalf("DeleteServerRow(%s) touched a server nobody is deleting: %s, was %s", n, got, before)
|
||||
}
|
||||
}
|
||||
|
||||
// Releasing the world finishes a give-up; a deletion stays pending.
|
||||
for _, n := range []string{given, doomed} {
|
||||
if err := st.ReleaseWorld(ctx, n, now); err != nil {
|
||||
t.Fatalf("ReleaseWorld(%s): %v", n, err)
|
||||
}
|
||||
}
|
||||
if got, want := state(given), "owner=false deleted=false retire=false/false aliases=1 allow=0"; got != want {
|
||||
t.Fatalf("given up and released: %s, want %s", got, want)
|
||||
}
|
||||
if got, want := state(doomed), "owner=false deleted=false retire=true/true aliases=1 allow=0"; got != want {
|
||||
t.Fatalf("released while being deleted: %s, want %s", got, want)
|
||||
}
|
||||
if at, _ := retireColumns(t, doomed); !at.Time.Equal(doomedAt.RequestedAt) {
|
||||
t.Fatalf("the release moved the deletion's request time to %v, want %v", at.Time, doomedAt.RequestedAt)
|
||||
}
|
||||
|
||||
mustExec(t, `INSERT INTO server_allowlist (server_name, mc_uuid) VALUES ($1, $2)`, doomed, testUUID(t))
|
||||
if err := st.DeleteServerRow(ctx, doomed, now); err != nil {
|
||||
t.Fatalf("DeleteServerRow(%s): %v", doomed, err)
|
||||
}
|
||||
if got, want := state(doomed), "owner=false deleted=true retire=false/false aliases=0 allow=0"; got != want {
|
||||
t.Fatalf("deleted: %s, want %s", got, want)
|
||||
}
|
||||
if got, want := state(plain), "owner=true deleted=false retire=false/false aliases=1 allow=1"; got != want {
|
||||
t.Fatalf("a bystander changed: %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// fleetCluster knows a set of servers by uid; every other row in the shared
|
||||
// schema reads as a server whose CRD is gone and whose world volume is still
|
||||
// there, which the reaper never touches.
|
||||
type fleetCluster struct {
|
||||
uids map[string]string
|
||||
deletedPVC []string
|
||||
deletedServer []string
|
||||
}
|
||||
|
||||
func (c *fleetCluster) Inspect(_ context.Context, name string) (reaper.ServerCRD, error) {
|
||||
uid, ok := c.uids[name]
|
||||
if !ok {
|
||||
return reaper.ServerCRD{}, reaper.ErrNotFound
|
||||
}
|
||||
return reaper.ServerCRD{PVC: reaper.WorldPVCName(name), UID: uid}, nil
|
||||
}
|
||||
|
||||
func (c *fleetCluster) HoldWorld(ctx context.Context, _ string) (context.Context, func(), error) {
|
||||
return ctx, func() {}, nil
|
||||
}
|
||||
|
||||
func (c *fleetCluster) WorldExists(_ context.Context, pvc string) (bool, error) {
|
||||
for _, gone := range c.deletedPVC {
|
||||
if gone == pvc {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (c *fleetCluster) DeletePVC(_ context.Context, pvc string) error {
|
||||
c.deletedPVC = append(c.deletedPVC, pvc)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *fleetCluster) DeleteServer(_ context.Context, name, uid string) error {
|
||||
if c.uids[name] != uid {
|
||||
return fmt.Errorf("DeleteServer(%s, %s): the server has uid %s", name, uid, c.uids[name])
|
||||
}
|
||||
c.deletedServer = append(c.deletedServer, name)
|
||||
delete(c.uids, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestRetirementThroughTheReaper runs the reaper over a give-up and a deletion
|
||||
// felis-api recorded, beside a server played minutes ago: the two go on the
|
||||
// first run however recently they were played, each archived first (in name
|
||||
// order, as the reaper lists them), and a second
|
||||
// run finds nothing left to do. The deleted server's name and subdomain are free
|
||||
// for a new server, which starts clean.
|
||||
func TestRetirementThroughTheReaper(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := reaper.NewPGStore(db)
|
||||
u := newUser(t, "user", "rtreap")
|
||||
sfx := suffix(t)
|
||||
given, doomed, bystander := "rrg-"+sfx, "rrd-"+sfx, "rrb-"+sfx
|
||||
cl := &fleetCluster{uids: map[string]string{}}
|
||||
for _, n := range []string{given, doomed, bystander} {
|
||||
if err := repo.SeedServer(ctx, n, n+"-s", 100, 128, 1); err != nil {
|
||||
t.Fatalf("SeedServer(%s): %v", n, err)
|
||||
}
|
||||
mustExec(t, `UPDATE servers SET owner_id = $2, claimed_at = now() - interval '30 days', last_active_at = now() - interval '5 minutes'
|
||||
WHERE name = $1`, n, u.ID)
|
||||
mustExec(t, `INSERT INTO server_allowlist (server_name, mc_uuid) VALUES ($1, $2)`, n, testUUID(t))
|
||||
cl.uids[n] = "uid-" + n
|
||||
}
|
||||
if _, err := repo.RequestRetire(ctx, given, false); err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", given, err)
|
||||
}
|
||||
if _, err := repo.RequestRetire(ctx, doomed, true); err != nil {
|
||||
t.Fatalf("RequestRetire(%s): %v", doomed, err)
|
||||
}
|
||||
|
||||
ar := &reclaimArchiver{}
|
||||
r := &reaper.Reaper{Cfg: reaper.DefaultConfig(), Store: st, Cluster: cl, Archiver: ar}
|
||||
if _, err := r.RunOnce(ctx); err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
mineOnly := func(refs []string) string {
|
||||
var out []string
|
||||
for _, ref := range refs {
|
||||
for _, n := range []string{given, doomed, bystander} {
|
||||
if strings.Contains(ref, n) {
|
||||
out = append(out, strings.Replace(ref, n, map[string]string{given: "given", doomed: "doomed", bystander: "bystander"}[n], 1))
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Join(out, " ")
|
||||
}
|
||||
if got, want := mineOnly(cl.deletedPVC), "world-doomed-0 world-given-0"; got != want {
|
||||
t.Fatalf("volumes deleted: %q, want %q", got, want)
|
||||
}
|
||||
if got, want := mineOnly(cl.deletedServer), "doomed"; got != want {
|
||||
t.Fatalf("servers deleted: %q, want %q", got, want)
|
||||
}
|
||||
if got, want := mineOnly(ar.archived), "/archives/doomed-new.tar.gz /archives/given-new.tar.gz"; got != want {
|
||||
t.Fatalf("archived: %q, want %q", got, want)
|
||||
}
|
||||
|
||||
type row struct {
|
||||
owner sql.NullString
|
||||
deleted, retir sql.NullTime
|
||||
aliases, allow int
|
||||
archives string
|
||||
audit string
|
||||
}
|
||||
read := func(n string) row {
|
||||
t.Helper()
|
||||
var r row
|
||||
if err := db.QueryRowContext(ctx,
|
||||
`SELECT owner_id, deleted_at, retire_requested_at,
|
||||
(SELECT count(*) FROM server_aliases WHERE server_name = s.name),
|
||||
(SELECT count(*) FROM server_allowlist WHERE server_name = s.name),
|
||||
(SELECT COALESCE(string_agg(reason || ':' || status || ':' || COALESCE(former_owner, ''), ','), '') FROM world_backups WHERE server_name = s.name),
|
||||
(SELECT COALESCE(string_agg(action, ',' ORDER BY id), '') FROM audit_logs WHERE server_name = s.name AND source = 'reaper')
|
||||
FROM servers s WHERE name = $1`, n).Scan(&r.owner, &r.deleted, &r.retir, &r.aliases, &r.allow, &r.archives, &r.audit); err != nil {
|
||||
t.Fatalf("read %s: %v", n, err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
g := read(given)
|
||||
if g.owner.Valid || g.deleted.Valid || g.retir.Valid || g.aliases != 1 || g.allow != 0 ||
|
||||
g.archives != "released:present:"+u.ID || g.audit != reaper.ActionReleaseWorld {
|
||||
t.Fatalf("given up: %+v; want released, still listed at its subdomain, its archive kept as the owner's", g)
|
||||
}
|
||||
d := read(doomed)
|
||||
if d.owner.Valid || !d.deleted.Valid || d.retir.Valid || d.aliases != 0 || d.allow != 0 ||
|
||||
d.archives != "released:present:"+u.ID || d.audit != reaper.ActionDeleteServer {
|
||||
t.Fatalf("deleted: %+v; want the row marked deleted, its subdomain freed, its archive kept", d)
|
||||
}
|
||||
b := read(bystander)
|
||||
if b.owner.String != u.ID || b.deleted.Valid || b.aliases != 1 || b.allow != 1 || b.archives != "" || b.audit != "" {
|
||||
t.Fatalf("the bystander changed: %+v", b)
|
||||
}
|
||||
|
||||
// Nothing is left for the next run.
|
||||
if _, err := r.RunOnce(ctx); err != nil {
|
||||
t.Fatalf("second RunOnce: %v", err)
|
||||
}
|
||||
if got := mineOnly(ar.archived) + " | " + mineOnly(cl.deletedServer); got != "/archives/doomed-new.tar.gz /archives/given-new.tar.gz | doomed" {
|
||||
t.Fatalf("the second run did more: %s", got)
|
||||
}
|
||||
|
||||
if err := repo.SeedServer(ctx, doomed, doomed+"-s", 200, 256, 2); err != nil {
|
||||
t.Fatalf("a new server under the deleted one's name: %v", err)
|
||||
}
|
||||
if at, del := retireColumns(t, doomed); at.Valid || del {
|
||||
t.Fatalf("the new server inherited a retirement: at=%v delete=%v", at, del)
|
||||
}
|
||||
if rec, err := repo.ServerBySubdomain(ctx, doomed+"-s"); err != nil || rec.Name != doomed || rec.OwnerID != "" || rec.Retire != nil {
|
||||
t.Fatalf("the new server = %+v, %v; want it unowned at the freed subdomain", rec, err)
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,8 @@ import (
|
||||
// backups: an owned world joined since its owner's newest intact scheduled
|
||||
// backup is due once that backup is older than the period, worlds without one
|
||||
// first. A previous owner's, a corrupt, a deleted, a pre-claim or a manual
|
||||
// backup is no restore point of the current owner's world.
|
||||
// backup is no restore point of the current owner's world. A world being given
|
||||
// up is never due: the reaper archives it anyway.
|
||||
func TestScheduledBackupCandidates(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
u := newUser(t, "user", "sched")
|
||||
@@ -75,6 +76,10 @@ func TestScheduledBackupCandidates(t *testing.T) {
|
||||
backup(manual, u.ID, "manual", "present", now.Add(-30*time.Minute), false)
|
||||
server("unowned", "", claimed, now.Add(-h), false)
|
||||
server("gone", u.ID, claimed, now.Add(-h), true)
|
||||
retiring := server("retiring", u.ID, claimed, now.Add(-h), false)
|
||||
if _, err := repo.RequestRetire(ctx, retiring, false); err != nil {
|
||||
t.Fatalf("RequestRetire: %v", err)
|
||||
}
|
||||
|
||||
got, err := repo.ScheduledBackupCandidates(ctx, now.Add(-24*h))
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user