diff --git a/docs/operations.md b/docs/operations.md index 6b9b7fb..1729540 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -48,6 +48,28 @@ and gains no failover. A multi-node shape would need, at least, storage that can pod to another node and leader election in felis-operator (controller-runtime's `LeaderElection`) so a second replica can stand by. +### While felis-api restarts + +An installer rerun that changes felis-api, a node restart or a crashed pod takes the API +away until its new pod is ready: about 12 s on the reference VM (`kubectl rollout +restart` to Available). Its Deployment keeps one replica with the Recreate strategy, so +the old pod is gone before the new one starts. Two pods at once would be wrong for +felis-api: the uploads volume is ReadWriteOnce, a chunked upload is serialized inside the +process, and the build reconciler, restore settler, registry pruner, upload reapers and +audit retention run in-process without leader election, so each would run twice. During +the window: + +- Players already on a server stay there; game servers keep running. +- A player leaving the login gate or joining a server by its address is admitted when + felis-api confirmed their link within the last 10 minutes; anyone else is told login + verification is temporarily unavailable. +- The login gate retries a new login for up to 60 s and tells the player it is retrying, + so a restart shorter than that only delays the login. +- Wakes, stops, `/link` and the panel wait for the API. +- A Velocity restart in the window routes on + `/opt/felis/velocity/plugins/felis-link/last-servers.json`, the last server list the + API answered with, until a refresh succeeds (every 15 s). + ## 2. Sizing ### What the platform itself uses diff --git a/plugins/shared/src/main/java/best/lolicon/felis/link/FelisApiClient.java b/plugins/shared/src/main/java/best/lolicon/felis/link/FelisApiClient.java index ed6f4d6..37e3082 100644 --- a/plugins/shared/src/main/java/best/lolicon/felis/link/FelisApiClient.java +++ b/plugins/shared/src/main/java/best/lolicon/felis/link/FelisApiClient.java @@ -7,7 +7,6 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -57,17 +56,7 @@ public final class FelisApiClient { /** listServers returns the lifecycle view of every MinecraftServer (GET /servers). */ public List listServers() throws LinkException { - Map obj = getObject("/api/v1/servers", 200); - Object arr = obj.get("servers"); - List out = new ArrayList<>(); - if (arr instanceof List) { - for (Object e : (List) arr) { - if (e instanceof Map) { - out.add(ServerView.fromJson((Map) e)); - } - } - } - return out; + return ServerView.listFrom(getObject("/api/v1/servers", 200)); } /** serverStatus reads one server's current lifecycle view (internal status). */ diff --git a/plugins/shared/src/main/java/best/lolicon/felis/link/ServerView.java b/plugins/shared/src/main/java/best/lolicon/felis/link/ServerView.java index 08f4914..a8d1e18 100644 --- a/plugins/shared/src/main/java/best/lolicon/felis/link/ServerView.java +++ b/plugins/shared/src/main/java/best/lolicon/felis/link/ServerView.java @@ -1,5 +1,8 @@ package best.lolicon.felis.link; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; import java.util.Map; /** @@ -62,6 +65,66 @@ public final class ServerView { intval(o, "playersMax")); } + /** + * listFromJson reads the {@code {"servers":[...]}} envelope {@code GET /servers} + * answers with, which is also the shape of the proxy's saved server list. Entries + * that are not objects are skipped; text that is not such an envelope throws + * IllegalArgumentException. + */ + public static List listFromJson(String text) { + Object root = Json.parse(text); + if (!(root instanceof Map)) { + throw new IllegalArgumentException("server list: not a JSON object"); + } + return listFrom((Map) root); + } + + static List listFrom(Map envelope) { + Object arr = envelope.get("servers"); + List out = new ArrayList<>(); + if (arr instanceof List) { + for (Object e : (List) arr) { + if (e instanceof Map) { + out.add(fromJson((Map) e)); + } + } + } + return out; + } + + /** listToJson renders views in the envelope {@link #listFromJson(String)} reads back. */ + public static String listToJson(Collection views) { + StringBuilder b = new StringBuilder("{\"servers\":["); + boolean first = true; + for (ServerView v : views) { + if (!first) { + b.append(','); + } + first = false; + b.append('{'); + field(b, "name", v.name, true); + field(b, "subdomain", v.subdomain, false); + field(b, "phase", v.phase, false); + b.append(",\"ready\":").append(v.ready); + field(b, "autostartPolicy", v.autostartPolicy, false); + field(b, "desiredState", v.desiredState, false); + field(b, "endpointMode", v.endpointMode, false); + field(b, "endpointAddress", v.endpointAddress, false); + b.append(",\"playersOnline\":").append(v.playersOnline); + b.append(",\"playersMax\":").append(v.playersMax); + b.append('}'); + } + return b.append("]}").toString(); + } + + // field appends "key":"value", or "key":null for an absent value. + private static void field(StringBuilder b, String key, String value, boolean first) { + if (!first) { + b.append(','); + } + b.append(Json.quote(key)).append(':').append(value == null ? "null" : Json.quote(value)); + } + public String name() { return name; } diff --git a/plugins/test.sh b/plugins/test.sh index 0376dbf..20f3d2f 100644 --- a/plugins/test.sh +++ b/plugins/test.sh @@ -10,7 +10,9 @@ # round-trips every frame kind (spec §12), no server name can re-aim a # felis-api request path, only the lobby and the login gate may drive # felis:control (and each only with its own frames), the link-status outage -# fallback fails closed outside its window, /invite prompts cannot double-fire +# fallback fails closed outside its window, a proxy restarted during an API +# outage routes on the last saved server list (and only until a fetch +# succeeds), /invite prompts cannot double-fire # or outlive their TTL, the invite card really is a green/red clickable # prompt, and the op-login approval card names the account and leaves its # name for the admin to type. InviteCardTest and OpApprovalCardTest need the @@ -81,6 +83,14 @@ java -cp "$work/policy-classes" best.lolicon.felis.velocity.ControlPolicyTest echo "==> LinkGateTest (outage fallback + frame budget, velocity)" java -cp "$work/policy-classes" best.lolicon.felis.velocity.LinkGateTest +echo "==> ServerListSourceTest (saved server list for a restart during an outage, velocity)" +mkdir -p "$work/list-classes" +javac -d "$work/list-classes" \ + plugins/shared/src/main/java/best/lolicon/felis/link/*.java \ + plugins/velocity/src/main/java/best/lolicon/felis/velocity/ServerListSource.java \ + plugins/velocity/test/best/lolicon/felis/velocity/ServerListSourceTest.java +java -cp "$work/list-classes" best.lolicon.felis.velocity.ServerListSourceTest + echo "==> InviteBookTest (/invite prompt store, velocity)" mkdir -p "$work/velocity-classes" javac -d "$work/velocity-classes" \ diff --git a/plugins/velocity/src/main/java/best/lolicon/felis/velocity/FelisVelocityPlugin.java b/plugins/velocity/src/main/java/best/lolicon/felis/velocity/FelisVelocityPlugin.java index 822b455..e6e3fec 100644 --- a/plugins/velocity/src/main/java/best/lolicon/felis/velocity/FelisVelocityPlugin.java +++ b/plugins/velocity/src/main/java/best/lolicon/felis/velocity/FelisVelocityPlugin.java @@ -70,6 +70,8 @@ import java.util.regex.Pattern; ) public final class FelisVelocityPlugin { private static final Duration REGISTRATION_REFRESH = Duration.ofSeconds(15); + // The last server list felis-api answered with, for a restart during an API outage. + private static final String SERVER_LIST_FILE = "last-servers.json"; private static final Duration WAIT_POLL = Duration.ofSeconds(2); private static final Duration INVITE_TTL = Duration.ofSeconds(120); // Long enough that spraying cards at a room is tedious, short enough that showing three @@ -91,6 +93,7 @@ public final class FelisVelocityPlugin { private LinkClient linkClient; private FelisApiClient apiClient; private ServerRegistry registry; + private ServerListSource serverList; private WaitingRouter router; private boolean onlineMode; private boolean routingActive; @@ -133,6 +136,7 @@ public final class FelisVelocityPlugin { this.apiClient = new FelisApiClient(config.linkConfig()); this.registry = new ServerRegistry(proxy, logger, config.rootDomain()); + this.serverList = new ServerListSource(apiClient::listServers, dataDirectory.resolve(SERVER_LIST_FILE)); this.router = new WaitingRouter(proxy, logger, apiClient, registry, this, config.loginServer(), config.lobbyServer()); MotdResponder motd = new MotdResponder(registry); @@ -213,14 +217,23 @@ public final class FelisVelocityPlugin { if (router != null) { router.pruneLinks(); } - try { - List servers = apiClient.listServers(); - registry.refresh(servers); - } catch (LinkException e) { + ServerListSource.Result r = serverList.next(); + if (r.servers != null) { + registry.refresh(r.servers); + } + if (r.restored) { + logger.warn("Felis: felis-api is unreachable at startup (status={}): {}; routing to the {} backends " + + "in the saved server list until it answers.", + r.failure.statusCode(), r.failure.getMessage(), r.servers.size()); + } else if (r.failure != null) { // Keep existing registrations on a control-plane blip (spec §11): a // transient failure must never deregister live backends. logger.warn("Felis: server list refresh failed (status={}): {}; keeping current registrations.", - e.statusCode(), e.getMessage()); + r.failure.statusCode(), r.failure.getMessage()); + } + if (r.fileError != null) { + logger.warn("Felis: saved server list {}: {}", r.failure == null ? "not written" : "not usable", + r.fileError.toString()); } } diff --git a/plugins/velocity/src/main/java/best/lolicon/felis/velocity/ServerListSource.java b/plugins/velocity/src/main/java/best/lolicon/felis/velocity/ServerListSource.java new file mode 100644 index 0000000..8768774 --- /dev/null +++ b/plugins/velocity/src/main/java/best/lolicon/felis/velocity/ServerListSource.java @@ -0,0 +1,119 @@ +package best.lolicon.felis.velocity; + +import best.lolicon.felis.link.LinkException; +import best.lolicon.felis.link.ServerView; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; + +/** + * ServerListSource decides what the registry reconciles against on each refresh, and + * keeps the last list felis-api answered with on disk. + * + *

A fetch that succeeds always wins, and its list is saved when it differs from the + * one saved last. A fetch that fails keeps the current registrations — except before + * the first success since the proxy started. The registry is empty then (velocity.toml + * holds only a dead login placeholder), so a proxy restarted while felis-api is down — + * an upgrade window, a crashed API pod — would refuse every player until the API came + * back. In that one case the saved list is handed out once instead: the login gate, + * the lobby and the user backends go back to the addresses they last had. Those are + * Service ClusterIPs, which outlive pod restarts, and game servers keep running while + * the API is down, so a saved address is usually still the live one; the next + * successful fetch replaces the whole list either way. + * + *

The file is written next to felis-link.properties and replaced by rename, so a + * crash mid-write leaves the previous list. An unreadable or malformed file counts as + * no saved list. + */ +final class ServerListSource { + + /** Fetch is the one felis-api call the source wraps (FelisApiClient::listServers). */ + interface Fetch { + List listServers() throws LinkException; + } + + /** Result is one refresh's outcome. */ + static final class Result { + /** The list to reconcile against, or null to keep the current registrations. */ + final List servers; + /** True when servers came from the saved list. */ + final boolean restored; + /** The fetch failure, or null when the fetch succeeded. */ + final LinkException failure; + /** Why the saved list could not be written or read, or null. */ + final IOException fileError; + + Result(List servers, boolean restored, LinkException failure, IOException fileError) { + this.servers = servers; + this.restored = restored; + this.failure = failure; + this.fileError = fileError; + } + } + + private final Fetch api; + private final Path file; + + private boolean fetched; + private boolean fellBack; + private String lastSaved; + + ServerListSource(Fetch api, Path file) { + this.api = api; + this.file = file; + } + + /** next fetches the server list and says what the registry should do with it. */ + synchronized Result next() { + List servers; + try { + servers = api.listServers(); + } catch (LinkException e) { + if (fetched || fellBack) { + return new Result(null, false, e, null); + } + fellBack = true; + try { + List saved = load(); + return new Result(saved, saved != null, e, null); + } catch (IOException io) { + return new Result(null, false, e, io); + } catch (IllegalArgumentException bad) { + return new Result(null, false, e, new IOException("the saved server list is malformed", bad)); + } + } + fetched = true; + try { + save(servers); + return new Result(servers, false, null, null); + } catch (IOException io) { + return new Result(servers, false, null, io); + } + } + + private List load() throws IOException { + String text; + try { + text = Files.readString(file, StandardCharsets.UTF_8); + } catch (NoSuchFileException absent) { + return null; + } + return ServerView.listFromJson(text); + } + + private void save(List servers) throws IOException { + String text = ServerView.listToJson(servers); + if (text.equals(lastSaved)) { + return; + } + Path tmp = file.resolveSibling(file.getFileName() + ".tmp"); + Files.writeString(tmp, text, StandardCharsets.UTF_8); + Files.move(tmp, file, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + lastSaved = text; + } +} diff --git a/plugins/velocity/src/main/java/best/lolicon/felis/velocity/ServerRegistry.java b/plugins/velocity/src/main/java/best/lolicon/felis/velocity/ServerRegistry.java index a091b61..2b66383 100644 --- a/plugins/velocity/src/main/java/best/lolicon/felis/velocity/ServerRegistry.java +++ b/plugins/velocity/src/main/java/best/lolicon/felis/velocity/ServerRegistry.java @@ -23,8 +23,8 @@ import java.util.concurrent.ConcurrentHashMap; * the felis lifecycle views keyed by name + subdomain, and the registration of * each server's backend address as a Velocity {@link RegisteredServer}. * - *

{@link #refresh(Collection)} is called only on a successful fetch - * from felis-api, so a name's absence is a genuine removal. On an API failure the + *

{@link #refresh(Collection)} is called on a successful fetch from + * felis-api, so a name's absence is a genuine removal. On an API failure the * caller skips the refresh entirely and the previous registrations survive * untouched (spec §11 keep-old-on-failure) — a transient control-plane blip must * never deregister live backends out from under connected players. @@ -32,7 +32,9 @@ import java.util.concurrent.ConcurrentHashMap; *

Every API-reported backend, including the system login and lobby, lives in * this registry. The generated {@code velocity.toml} contains a deliberately dead * login placeholder only so Velocity can validate {@code try = ["login"]}; the - * first successful refresh replaces that placeholder with the live ClusterIP. + * first successful refresh replaces that placeholder with the live ClusterIP. A + * proxy that starts while felis-api is down refreshes once from the last list the + * API answered with instead ({@link ServerListSource}). */ final class ServerRegistry { private static final int DEFAULT_PORT = 25565; diff --git a/plugins/velocity/test/best/lolicon/felis/velocity/ServerListSourceTest.java b/plugins/velocity/test/best/lolicon/felis/velocity/ServerListSourceTest.java new file mode 100644 index 0000000..3c0b3b5 --- /dev/null +++ b/plugins/velocity/test/best/lolicon/felis/velocity/ServerListSourceTest.java @@ -0,0 +1,191 @@ +package best.lolicon.felis.velocity; + +import best.lolicon.felis.link.LinkException; +import best.lolicon.felis.link.ServerView; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; + +/** + * ServerListSourceTest covers what the proxy routes on while felis-api is down: a + * successful fetch is saved and always wins, a failure keeps the current + * registrations, and only before the first success does the saved list come back — + * field for field, once. Framework free: a failed assertion throws and the process + * exits non-zero. + * + *

Run: {@code javac -d shared/src/main/java/best/lolicon/felis/link/*.java + * velocity/src/main/java/best/lolicon/felis/velocity/ServerListSource.java + * velocity/test/best/lolicon/felis/velocity/ServerListSourceTest.java && java -cp + * best.lolicon.felis.velocity.ServerListSourceTest}. + */ +public final class ServerListSourceTest { + + private static int checks; + + // What the stub felis-api does next: answer with this list, or fail when null. + private static volatile List answer; + + private static final ServerView LOGIN = new ServerView("login", null, "Running", true, + null, "Running", "ClusterIP", "10.43.0.17:25565", 0, 0); + private static final ServerView SURVIVAL = new ServerView("survival", "Survival", "Stopped", false, + "Anyone", "Stopped", "ClusterIP", "10.43.9.2:25565", 3, 20); + + public static void main(String[] args) throws Exception { + Path dir = Files.createTempDirectory("felis-server-list"); + try { + restartDuringAnOutageRoutesOnTheSavedList(dir.resolve("a")); + failuresAfterAFetchKeepTheCurrentRegistrations(dir.resolve("b")); + theSavedListIsHandedOutOnce(dir.resolve("c")); + noSavedListMeansNothingToRestore(dir.resolve("d")); + aMalformedSavedListIsReportedAndIgnored(dir.resolve("e")); + theApiEnvelopeParses(); + } finally { + try (Stream walk = Files.walk(dir)) { + walk.sorted(Comparator.reverseOrder()).forEach(p -> p.toFile().delete()); + } + } + System.out.println("ServerListSourceTest OK (" + checks + " checks)"); + } + + private static Path fresh(Path dir) throws Exception { + Files.createDirectories(dir); + return dir.resolve("last-servers.json"); + } + + private static void restartDuringAnOutageRoutesOnTheSavedList(Path dir) throws Exception { + Path file = fresh(dir); + answer = Arrays.asList(LOGIN, SURVIVAL); + ServerListSource.Result live = new ServerListSource(ServerListSourceTest::fetch, file).next(); + assertEq("live list returned", 2, live.servers.size()); + assertEq("live list is not restored", false, live.restored); + assertEq("live fetch has no failure", null, live.failure); + assertEq("saved without error", null, live.fileError); + assertEq("no temp file left behind", List.of("last-servers.json"), names(dir)); + + // A new proxy process (a new source) while felis-api is down. + answer = null; + ServerListSource.Result r = new ServerListSource(ServerListSourceTest::fetch, file).next(); + assertEq("restored", true, r.restored); + assertEq("failure is reported", 503, r.failure.statusCode()); + assertEq("two backends", 2, r.servers.size()); + ServerView login = r.servers.get(0); + assertEq("login name", "login", login.name()); + assertEq("login subdomain stays absent", null, login.subdomain()); + assertEq("login address", "10.43.0.17:25565", login.endpointAddress()); + assertEq("login ready", true, login.ready()); + ServerView s = r.servers.get(1); + assertEq("survival name", "survival", s.name()); + assertEq("survival subdomain", "Survival", s.subdomain()); + assertEq("survival phase", "Stopped", s.phase()); + assertEq("survival ready", false, s.ready()); + assertEq("survival policy", "Anyone", s.autostartPolicy()); + assertEq("survival desired", "Stopped", s.desiredState()); + assertEq("survival mode", "ClusterIP", s.endpointMode()); + assertEq("survival address", "10.43.9.2:25565", s.endpointAddress()); + assertEq("survival online", 3, s.playersOnline()); + assertEq("survival max", 20, s.playersMax()); + } + + private static void failuresAfterAFetchKeepTheCurrentRegistrations(Path dir) throws Exception { + Path file = fresh(dir); + ServerListSource src = new ServerListSource(ServerListSourceTest::fetch, file); + answer = List.of(LOGIN); + src.next(); + answer = null; + ServerListSource.Result r = src.next(); + assertEq("keep current registrations", null, r.servers); + assertEq("not restored", false, r.restored); + assertEq("failure reported", 503, r.failure.statusCode()); + + // A later success replaces the saved list, including a removal. + answer = List.of(SURVIVAL); + src.next(); + answer = null; + ServerListSource.Result restarted = new ServerListSource(ServerListSourceTest::fetch, file).next(); + assertEq("newest list saved", 1, restarted.servers.size()); + assertEq("newest list content", "survival", restarted.servers.get(0).name()); + } + + private static void theSavedListIsHandedOutOnce(Path dir) throws Exception { + Path file = fresh(dir); + answer = List.of(LOGIN); + new ServerListSource(ServerListSourceTest::fetch, file).next(); + answer = null; + ServerListSource src = new ServerListSource(ServerListSourceTest::fetch, file); + assertEq("first failure restores", true, src.next().restored); + ServerListSource.Result again = src.next(); + assertEq("second failure keeps registrations", null, again.servers); + assertEq("second failure is not a restore", false, again.restored); + } + + private static void noSavedListMeansNothingToRestore(Path dir) throws Exception { + Path file = fresh(dir); + answer = null; + ServerListSource.Result r = new ServerListSource(ServerListSourceTest::fetch, file).next(); + assertEq("nothing to route on", null, r.servers); + assertEq("not restored", false, r.restored); + assertEq("a missing file is no error", null, r.fileError); + } + + private static void aMalformedSavedListIsReportedAndIgnored(Path dir) throws Exception { + Path file = fresh(dir); + Files.writeString(file, "{\"servers\":[{\"name\":\"login\"", StandardCharsets.UTF_8); + answer = null; + ServerListSource.Result r = new ServerListSource(ServerListSourceTest::fetch, file).next(); + assertEq("nothing to route on", null, r.servers); + assertEq("not restored", false, r.restored); + assertEq("malformed file reported", true, r.fileError != null); + + // The next success overwrites it. + answer = List.of(LOGIN); + ServerListSource src = new ServerListSource(ServerListSourceTest::fetch, file); + src.next(); + assertEq("rewritten", true, Files.readString(file, StandardCharsets.UTF_8).contains("\"10.43.0.17:25565\"")); + } + + // The exact shape GET /api/v1/servers answers with, extra fields included. + private static void theApiEnvelopeParses() { + List v = ServerView.listFromJson("{\"servers\":[{\"name\":\"lobby\",\"subdomain\":\"\"," + + "\"phase\":\"Running\",\"ready\":true,\"desiredState\":\"Running\",\"endpointMode\":\"ClusterIP\"," + + "\"endpointAddress\":\"10.43.1.5:25565\",\"playersOnline\":1,\"playersMax\":100," + + "\"owner\":\"x\"},\"junk\"]}"); + assertEq("non-object entries skipped", 1, v.size()); + assertEq("lobby name", "lobby", v.get(0).name()); + assertEq("lobby max", 100, v.get(0).playersMax()); + try { + ServerView.listFromJson("[]"); + } catch (IllegalArgumentException expected) { + checks++; + return; + } + throw new AssertionError("a bare array is not a server list"); + } + + // ---- harness ---- + + private static List fetch() throws LinkException { + List a = answer; + if (a == null) { + throw new LinkException(503, "unavailable", "stub outage"); + } + return a; + } + + private static List names(Path dir) throws Exception { + try (Stream s = Files.list(dir)) { + return s.map(p -> p.getFileName().toString()).sorted().toList(); + } + } + + private static void assertEq(String what, Object want, Object got) { + if (want == null ? got != null : !want.equals(got)) { + throw new AssertionError(what + ": want " + want + ", got " + got); + } + checks++; + } +}