fix(velocity): 保存最近一次服务器列表，控制面不可用时重启代理仍能路由登录

docs/operations.md

@@ -48,6 +48,28 @@ and gains no failover. A multi-node shape would need, at least, storage that can
 pod to another node and leader election in felis-operator (controller-runtime's
 `LeaderElection`) so a second replica can stand by.
 
+### While felis-api restarts
+
+An installer rerun that changes felis-api, a node restart or a crashed pod takes the API
+away until its new pod is ready: about 12 s on the reference VM (`kubectl rollout
+restart` to Available). Its Deployment keeps one replica with the Recreate strategy, so
+the old pod is gone before the new one starts. Two pods at once would be wrong for
+felis-api: the uploads volume is ReadWriteOnce, a chunked upload is serialized inside the
+process, and the build reconciler, restore settler, registry pruner, upload reapers and
+audit retention run in-process without leader election, so each would run twice. During
+the window:
+
+- Players already on a server stay there; game servers keep running.
+- A player leaving the login gate or joining a server by its address is admitted when
+  felis-api confirmed their link within the last 10 minutes; anyone else is told login
+  verification is temporarily unavailable.
+- The login gate retries a new login for up to 60 s and tells the player it is retrying,
+  so a restart shorter than that only delays the login.
+- Wakes, stops, `/link` and the panel wait for the API.
+- A Velocity restart in the window routes on
+  `/opt/felis/velocity/plugins/felis-link/last-servers.json`, the last server list the
+  API answered with, until a refresh succeeds (every 15 s).
+
 ## 2. Sizing
 
 ### What the platform itself uses

plugins/shared/src/main/java/best/lolicon/felis/link/FelisApiClient.java

@@ -7,7 +7,6 @@ import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -57,17 +56,7 @@ public final class FelisApiClient {
 
     /** listServers returns the lifecycle view of every MinecraftServer (GET /servers). */
     public List<ServerView> listServers() throws LinkException {
-        Map<?, ?> obj = getObject("/api/v1/servers", 200);
+        return ServerView.listFrom(getObject("/api/v1/servers", 200));
-        Object arr = obj.get("servers");
-        List<ServerView> out = new ArrayList<>();
-        if (arr instanceof List) {
-            for (Object e : (List<?>) arr) {
-                if (e instanceof Map) {
-                    out.add(ServerView.fromJson((Map<?, ?>) e));
-                }
-            }
-        }
-        return out;
     }
 
     /** serverStatus reads one server's current lifecycle view (internal status). */

plugins/shared/src/main/java/best/lolicon/felis/link/ServerView.java

@@ -1,5 +1,8 @@
 package best.lolicon.felis.link;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -62,6 +65,66 @@ public final class ServerView {
                 intval(o, "playersMax"));
     }
 
+    /**
+     * listFromJson reads the {@code {"servers":[...]}} envelope {@code GET /servers}
+     * answers with, which is also the shape of the proxy's saved server list. Entries
+     * that are not objects are skipped; text that is not such an envelope throws
+     * IllegalArgumentException.
+     */
+    public static List<ServerView> listFromJson(String text) {
+        Object root = Json.parse(text);
+        if (!(root instanceof Map)) {
+            throw new IllegalArgumentException("server list: not a JSON object");
+        }
+        return listFrom((Map<?, ?>) root);
+    }
+
+    static List<ServerView> listFrom(Map<?, ?> envelope) {
+        Object arr = envelope.get("servers");
+        List<ServerView> out = new ArrayList<>();
+        if (arr instanceof List) {
+            for (Object e : (List<?>) arr) {
+                if (e instanceof Map) {
+                    out.add(fromJson((Map<?, ?>) e));
+                }
+            }
+        }
+        return out;
+    }
+
+    /** listToJson renders views in the envelope {@link #listFromJson(String)} reads back. */
+    public static String listToJson(Collection<ServerView> views) {
+        StringBuilder b = new StringBuilder("{\"servers\":[");
+        boolean first = true;
+        for (ServerView v : views) {
+            if (!first) {
+                b.append(',');
+            }
+            first = false;
+            b.append('{');
+            field(b, "name", v.name, true);
+            field(b, "subdomain", v.subdomain, false);
+            field(b, "phase", v.phase, false);
+            b.append(",\"ready\":").append(v.ready);
+            field(b, "autostartPolicy", v.autostartPolicy, false);
+            field(b, "desiredState", v.desiredState, false);
+            field(b, "endpointMode", v.endpointMode, false);
+            field(b, "endpointAddress", v.endpointAddress, false);
+            b.append(",\"playersOnline\":").append(v.playersOnline);
+            b.append(",\"playersMax\":").append(v.playersMax);
+            b.append('}');
+        }
+        return b.append("]}").toString();
+    }
+
+    // field appends "key":"value", or "key":null for an absent value.
+    private static void field(StringBuilder b, String key, String value, boolean first) {
+        if (!first) {
+            b.append(',');
+        }
+        b.append(Json.quote(key)).append(':').append(value == null ? "null" : Json.quote(value));
+    }
+
     public String name() {
         return name;
     }

plugins/test.sh

@@ -10,7 +10,9 @@
 #      round-trips every frame kind (spec §12), no server name can re-aim a
 #      felis-api request path, only the lobby and the login gate may drive
 #      felis:control (and each only with its own frames), the link-status outage
-#      fallback fails closed outside its window, /invite prompts cannot double-fire
+#      fallback fails closed outside its window, a proxy restarted during an API
+#      outage routes on the last saved server list (and only until a fetch
+#      succeeds), /invite prompts cannot double-fire
 #      or outlive their TTL, the invite card really is a green/red clickable
 #      prompt, and the op-login approval card names the account and leaves its
 #      name for the admin to type. InviteCardTest and OpApprovalCardTest need the
@@ -81,6 +83,14 @@ java -cp "$work/policy-classes" best.lolicon.felis.velocity.ControlPolicyTest
 echo "==> LinkGateTest (outage fallback + frame budget, velocity)"
 java -cp "$work/policy-classes" best.lolicon.felis.velocity.LinkGateTest
 
+echo "==> ServerListSourceTest (saved server list for a restart during an outage, velocity)"
+mkdir -p "$work/list-classes"
+javac -d "$work/list-classes" \
+  plugins/shared/src/main/java/best/lolicon/felis/link/*.java \
+  plugins/velocity/src/main/java/best/lolicon/felis/velocity/ServerListSource.java \
+  plugins/velocity/test/best/lolicon/felis/velocity/ServerListSourceTest.java
+java -cp "$work/list-classes" best.lolicon.felis.velocity.ServerListSourceTest
+
 echo "==> InviteBookTest (/invite prompt store, velocity)"
 mkdir -p "$work/velocity-classes"
 javac -d "$work/velocity-classes" \

plugins/velocity/src/main/java/best/lolicon/felis/velocity/FelisVelocityPlugin.java

@@ -70,6 +70,8 @@ import java.util.regex.Pattern;
 )
 public final class FelisVelocityPlugin {
     private static final Duration REGISTRATION_REFRESH = Duration.ofSeconds(15);
+    // The last server list felis-api answered with, for a restart during an API outage.
+    private static final String SERVER_LIST_FILE = "last-servers.json";
     private static final Duration WAIT_POLL = Duration.ofSeconds(2);
     private static final Duration INVITE_TTL = Duration.ofSeconds(120);
     // Long enough that spraying cards at a room is tedious, short enough that showing three
@@ -91,6 +93,7 @@ public final class FelisVelocityPlugin {
     private LinkClient linkClient;
     private FelisApiClient apiClient;
     private ServerRegistry registry;
+    private ServerListSource serverList;
     private WaitingRouter router;
     private boolean onlineMode;
     private boolean routingActive;
@@ -133,6 +136,7 @@ public final class FelisVelocityPlugin {
 
         this.apiClient = new FelisApiClient(config.linkConfig());
         this.registry = new ServerRegistry(proxy, logger, config.rootDomain());
+        this.serverList = new ServerListSource(apiClient::listServers, dataDirectory.resolve(SERVER_LIST_FILE));
         this.router = new WaitingRouter(proxy, logger, apiClient, registry, this,
                 config.loginServer(), config.lobbyServer());
         MotdResponder motd = new MotdResponder(registry);
@@ -213,14 +217,23 @@ public final class FelisVelocityPlugin {
         if (router != null) {
             router.pruneLinks();
         }
-        try {
+        ServerListSource.Result r = serverList.next();
-            List<ServerView> servers = apiClient.listServers();
+        if (r.servers != null) {
-            registry.refresh(servers);
+            registry.refresh(r.servers);
-        } catch (LinkException e) {
+        }
+        if (r.restored) {
+            logger.warn("Felis: felis-api is unreachable at startup (status={}): {}; routing to the {} backends "
+                            + "in the saved server list until it answers.",
+                    r.failure.statusCode(), r.failure.getMessage(), r.servers.size());
+        } else if (r.failure != null) {
             // Keep existing registrations on a control-plane blip (spec §11): a
             // transient failure must never deregister live backends.
             logger.warn("Felis: server list refresh failed (status={}): {}; keeping current registrations.",
-                    e.statusCode(), e.getMessage());
+                    r.failure.statusCode(), r.failure.getMessage());
+        }
+        if (r.fileError != null) {
+            logger.warn("Felis: saved server list {}: {}", r.failure == null ? "not written" : "not usable",
+                    r.fileError.toString());
         }
     }
 

plugins/velocity/src/main/java/best/lolicon/felis/velocity/ServerListSource.java

@@ -0,0 +1,119 @@
+package best.lolicon.felis.velocity;
+
+import best.lolicon.felis.link.LinkException;
+import best.lolicon.felis.link.ServerView;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.List;
+
+/**
+ * ServerListSource decides what the registry reconciles against on each refresh, and
+ * keeps the last list felis-api answered with on disk.
+ *
+ * <p>A fetch that succeeds always wins, and its list is saved when it differs from the
+ * one saved last. A fetch that fails keeps the current registrations — except before
+ * the first success since the proxy started. The registry is empty then (velocity.toml
+ * holds only a dead login placeholder), so a proxy restarted while felis-api is down —
+ * an upgrade window, a crashed API pod — would refuse every player until the API came
+ * back. In that one case the saved list is handed out once instead: the login gate,
+ * the lobby and the user backends go back to the addresses they last had. Those are
+ * Service ClusterIPs, which outlive pod restarts, and game servers keep running while
+ * the API is down, so a saved address is usually still the live one; the next
+ * successful fetch replaces the whole list either way.
+ *
+ * <p>The file is written next to felis-link.properties and replaced by rename, so a
+ * crash mid-write leaves the previous list. An unreadable or malformed file counts as
+ * no saved list.
+ */
+final class ServerListSource {
+
+    /** Fetch is the one felis-api call the source wraps (FelisApiClient::listServers). */
+    interface Fetch {
+        List<ServerView> listServers() throws LinkException;
+    }
+
+    /** Result is one refresh's outcome. */
+    static final class Result {
+        /** The list to reconcile against, or null to keep the current registrations. */
+        final List<ServerView> servers;
+        /** True when servers came from the saved list. */
+        final boolean restored;
+        /** The fetch failure, or null when the fetch succeeded. */
+        final LinkException failure;
+        /** Why the saved list could not be written or read, or null. */
+        final IOException fileError;
+
+        Result(List<ServerView> servers, boolean restored, LinkException failure, IOException fileError) {
+            this.servers = servers;
+            this.restored = restored;
+            this.failure = failure;
+            this.fileError = fileError;
+        }
+    }
+
+    private final Fetch api;
+    private final Path file;
+
+    private boolean fetched;
+    private boolean fellBack;
+    private String lastSaved;
+
+    ServerListSource(Fetch api, Path file) {
+        this.api = api;
+        this.file = file;
+    }
+
+    /** next fetches the server list and says what the registry should do with it. */
+    synchronized Result next() {
+        List<ServerView> servers;
+        try {
+            servers = api.listServers();
+        } catch (LinkException e) {
+            if (fetched || fellBack) {
+                return new Result(null, false, e, null);
+            }
+            fellBack = true;
+            try {
+                List<ServerView> saved = load();
+                return new Result(saved, saved != null, e, null);
+            } catch (IOException io) {
+                return new Result(null, false, e, io);
+            } catch (IllegalArgumentException bad) {
+                return new Result(null, false, e, new IOException("the saved server list is malformed", bad));
+            }
+        }
+        fetched = true;
+        try {
+            save(servers);
+            return new Result(servers, false, null, null);
+        } catch (IOException io) {
+            return new Result(servers, false, null, io);
+        }
+    }
+
+    private List<ServerView> load() throws IOException {
+        String text;
+        try {
+            text = Files.readString(file, StandardCharsets.UTF_8);
+        } catch (NoSuchFileException absent) {
+            return null;
+        }
+        return ServerView.listFromJson(text);
+    }
+
+    private void save(List<ServerView> servers) throws IOException {
+        String text = ServerView.listToJson(servers);
+        if (text.equals(lastSaved)) {
+            return;
+        }
+        Path tmp = file.resolveSibling(file.getFileName() + ".tmp");
+        Files.writeString(tmp, text, StandardCharsets.UTF_8);
+        Files.move(tmp, file, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
+        lastSaved = text;
+    }
+}

plugins/velocity/src/main/java/best/lolicon/felis/velocity/ServerRegistry.java

@@ -23,8 +23,8 @@ import java.util.concurrent.ConcurrentHashMap;
  * the felis lifecycle views keyed by name + subdomain, and the registration of
  * each server's backend address as a Velocity {@link RegisteredServer}.
  *
- * <p>{@link #refresh(Collection)} is called only on a <em>successful</em> fetch
+ * <p>{@link #refresh(Collection)} is called on a <em>successful</em> fetch from
- * from felis-api, so a name's absence is a genuine removal. On an API failure the
+ * felis-api, so a name's absence is a genuine removal. On an API failure the
  * caller skips the refresh entirely and the previous registrations survive
  * untouched (spec §11 keep-old-on-failure) — a transient control-plane blip must
  * never deregister live backends out from under connected players.
@@ -32,7 +32,9 @@ import java.util.concurrent.ConcurrentHashMap;
  * <p>Every API-reported backend, including the system login and lobby, lives in
  * this registry. The generated {@code velocity.toml} contains a deliberately dead
  * login placeholder only so Velocity can validate {@code try = ["login"]}; the
- * first successful refresh replaces that placeholder with the live ClusterIP.
+ * first successful refresh replaces that placeholder with the live ClusterIP. A
+ * proxy that starts while felis-api is down refreshes once from the last list the
+ * API answered with instead ({@link ServerListSource}).
  */
 final class ServerRegistry {
     private static final int DEFAULT_PORT = 25565;

plugins/velocity/test/best/lolicon/felis/velocity/ServerListSourceTest.java

@@ -0,0 +1,191 @@
+package best.lolicon.felis.velocity;
+
+import best.lolicon.felis.link.LinkException;
+import best.lolicon.felis.link.ServerView;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Stream;
+
+/**
+ * ServerListSourceTest covers what the proxy routes on while felis-api is down: a
+ * successful fetch is saved and always wins, a failure keeps the current
+ * registrations, and only before the first success does the saved list come back —
+ * field for field, once. Framework free: a failed assertion throws and the process
+ * exits non-zero.
+ *
+ * <p>Run: {@code javac -d <out> shared/src/main/java/best/lolicon/felis/link/*.java
+ * velocity/src/main/java/best/lolicon/felis/velocity/ServerListSource.java
+ * velocity/test/best/lolicon/felis/velocity/ServerListSourceTest.java && java -cp <out>
+ * best.lolicon.felis.velocity.ServerListSourceTest}.
+ */
+public final class ServerListSourceTest {
+
+    private static int checks;
+
+    // What the stub felis-api does next: answer with this list, or fail when null.
+    private static volatile List<ServerView> answer;
+
+    private static final ServerView LOGIN = new ServerView("login", null, "Running", true,
+            null, "Running", "ClusterIP", "10.43.0.17:25565", 0, 0);
+    private static final ServerView SURVIVAL = new ServerView("survival", "Survival", "Stopped", false,
+            "Anyone", "Stopped", "ClusterIP", "10.43.9.2:25565", 3, 20);
+
+    public static void main(String[] args) throws Exception {
+        Path dir = Files.createTempDirectory("felis-server-list");
+        try {
+            restartDuringAnOutageRoutesOnTheSavedList(dir.resolve("a"));
+            failuresAfterAFetchKeepTheCurrentRegistrations(dir.resolve("b"));
+            theSavedListIsHandedOutOnce(dir.resolve("c"));
+            noSavedListMeansNothingToRestore(dir.resolve("d"));
+            aMalformedSavedListIsReportedAndIgnored(dir.resolve("e"));
+            theApiEnvelopeParses();
+        } finally {
+            try (Stream<Path> walk = Files.walk(dir)) {
+                walk.sorted(Comparator.reverseOrder()).forEach(p -> p.toFile().delete());
+            }
+        }
+        System.out.println("ServerListSourceTest OK (" + checks + " checks)");
+    }
+
+    private static Path fresh(Path dir) throws Exception {
+        Files.createDirectories(dir);
+        return dir.resolve("last-servers.json");
+    }
+
+    private static void restartDuringAnOutageRoutesOnTheSavedList(Path dir) throws Exception {
+        Path file = fresh(dir);
+        answer = Arrays.asList(LOGIN, SURVIVAL);
+        ServerListSource.Result live = new ServerListSource(ServerListSourceTest::fetch, file).next();
+        assertEq("live list returned", 2, live.servers.size());
+        assertEq("live list is not restored", false, live.restored);
+        assertEq("live fetch has no failure", null, live.failure);
+        assertEq("saved without error", null, live.fileError);
+        assertEq("no temp file left behind", List.of("last-servers.json"), names(dir));
+
+        // A new proxy process (a new source) while felis-api is down.
+        answer = null;
+        ServerListSource.Result r = new ServerListSource(ServerListSourceTest::fetch, file).next();
+        assertEq("restored", true, r.restored);
+        assertEq("failure is reported", 503, r.failure.statusCode());
+        assertEq("two backends", 2, r.servers.size());
+        ServerView login = r.servers.get(0);
+        assertEq("login name", "login", login.name());
+        assertEq("login subdomain stays absent", null, login.subdomain());
+        assertEq("login address", "10.43.0.17:25565", login.endpointAddress());
+        assertEq("login ready", true, login.ready());
+        ServerView s = r.servers.get(1);
+        assertEq("survival name", "survival", s.name());
+        assertEq("survival subdomain", "Survival", s.subdomain());
+        assertEq("survival phase", "Stopped", s.phase());
+        assertEq("survival ready", false, s.ready());
+        assertEq("survival policy", "Anyone", s.autostartPolicy());
+        assertEq("survival desired", "Stopped", s.desiredState());
+        assertEq("survival mode", "ClusterIP", s.endpointMode());
+        assertEq("survival address", "10.43.9.2:25565", s.endpointAddress());
+        assertEq("survival online", 3, s.playersOnline());
+        assertEq("survival max", 20, s.playersMax());
+    }
+
+    private static void failuresAfterAFetchKeepTheCurrentRegistrations(Path dir) throws Exception {
+        Path file = fresh(dir);
+        ServerListSource src = new ServerListSource(ServerListSourceTest::fetch, file);
+        answer = List.of(LOGIN);
+        src.next();
+        answer = null;
+        ServerListSource.Result r = src.next();
+        assertEq("keep current registrations", null, r.servers);
+        assertEq("not restored", false, r.restored);
+        assertEq("failure reported", 503, r.failure.statusCode());
+
+        // A later success replaces the saved list, including a removal.
+        answer = List.of(SURVIVAL);
+        src.next();
+        answer = null;
+        ServerListSource.Result restarted = new ServerListSource(ServerListSourceTest::fetch, file).next();
+        assertEq("newest list saved", 1, restarted.servers.size());
+        assertEq("newest list content", "survival", restarted.servers.get(0).name());
+    }
+
+    private static void theSavedListIsHandedOutOnce(Path dir) throws Exception {
+        Path file = fresh(dir);
+        answer = List.of(LOGIN);
+        new ServerListSource(ServerListSourceTest::fetch, file).next();
+        answer = null;
+        ServerListSource src = new ServerListSource(ServerListSourceTest::fetch, file);
+        assertEq("first failure restores", true, src.next().restored);
+        ServerListSource.Result again = src.next();
+        assertEq("second failure keeps registrations", null, again.servers);
+        assertEq("second failure is not a restore", false, again.restored);
+    }
+
+    private static void noSavedListMeansNothingToRestore(Path dir) throws Exception {
+        Path file = fresh(dir);
+        answer = null;
+        ServerListSource.Result r = new ServerListSource(ServerListSourceTest::fetch, file).next();
+        assertEq("nothing to route on", null, r.servers);
+        assertEq("not restored", false, r.restored);
+        assertEq("a missing file is no error", null, r.fileError);
+    }
+
+    private static void aMalformedSavedListIsReportedAndIgnored(Path dir) throws Exception {
+        Path file = fresh(dir);
+        Files.writeString(file, "{\"servers\":[{\"name\":\"login\"", StandardCharsets.UTF_8);
+        answer = null;
+        ServerListSource.Result r = new ServerListSource(ServerListSourceTest::fetch, file).next();
+        assertEq("nothing to route on", null, r.servers);
+        assertEq("not restored", false, r.restored);
+        assertEq("malformed file reported", true, r.fileError != null);
+
+        // The next success overwrites it.
+        answer = List.of(LOGIN);
+        ServerListSource src = new ServerListSource(ServerListSourceTest::fetch, file);
+        src.next();
+        assertEq("rewritten", true, Files.readString(file, StandardCharsets.UTF_8).contains("\"10.43.0.17:25565\""));
+    }
+
+    // The exact shape GET /api/v1/servers answers with, extra fields included.
+    private static void theApiEnvelopeParses() {
+        List<ServerView> v = ServerView.listFromJson("{\"servers\":[{\"name\":\"lobby\",\"subdomain\":\"\","
+                + "\"phase\":\"Running\",\"ready\":true,\"desiredState\":\"Running\",\"endpointMode\":\"ClusterIP\","
+                + "\"endpointAddress\":\"10.43.1.5:25565\",\"playersOnline\":1,\"playersMax\":100,"
+                + "\"owner\":\"x\"},\"junk\"]}");
+        assertEq("non-object entries skipped", 1, v.size());
+        assertEq("lobby name", "lobby", v.get(0).name());
+        assertEq("lobby max", 100, v.get(0).playersMax());
+        try {
+            ServerView.listFromJson("[]");
+        } catch (IllegalArgumentException expected) {
+            checks++;
+            return;
+        }
+        throw new AssertionError("a bare array is not a server list");
+    }
+
+    // ---- harness ----
+
+    private static List<ServerView> fetch() throws LinkException {
+        List<ServerView> a = answer;
+        if (a == null) {
+            throw new LinkException(503, "unavailable", "stub outage");
+        }
+        return a;
+    }
+
+    private static List<String> names(Path dir) throws Exception {
+        try (Stream<Path> s = Files.list(dir)) {
+            return s.map(p -> p.getFileName().toString()).sorted().toList();
+        }
+    }
+
+    private static void assertEq(String what, Object want, Object got) {
+        if (want == null ? got != null : !want.equals(got)) {
+            throw new AssertionError(what + ": want " + want + ", got " + got);
+        }
+        checks++;
+    }
+}