feat(plugins): 插件侧计数器与周期健康日志，Limbo/刷新失败按故障周期升级日志

plugins/velocity/src/main/java/best/lolicon/felis/velocity/FelisVelocityPlugin.java

@@ -5,6 +5,7 @@ import best.lolicon.felis.link.LinkClient;
 import best.lolicon.felis.link.LinkCode;
 import best.lolicon.felis.link.LinkException;
 import best.lolicon.felis.link.OpLoginView;
+import best.lolicon.felis.link.OutageTracker;
 import best.lolicon.felis.link.ServerView;
 
 import com.google.inject.Inject;
@@ -87,6 +88,10 @@ public final class FelisVelocityPlugin {
     private static final int API_THREADS = 8;
     private static final int API_QUEUE = 64;
     private static final long BUSY_LOG_INTERVAL_MILLIS = 60_000L;
+    // One health line per window (ProxyStats.line), and a reminder at most this often
+    // while the server-list refresh keeps failing.
+    private static final Duration STATS_INTERVAL = Duration.ofMinutes(10);
+    private static final long REFRESH_OUTAGE_REPEAT_MILLIS = 5 * 60_000L;
 
     private final ProxyServer proxy;
     private final Logger logger;
@@ -99,6 +104,9 @@ public final class FelisVelocityPlugin {
     private final FrameBudget commandBudget = new FrameBudget(5, 0.2, System::currentTimeMillis);
     private final BoundedExecutor apiCalls;
     private final AtomicLong lastBusyLog = new AtomicLong();
+    private final ProxyStats stats = new ProxyStats();
+    private final OutageTracker refreshOutage =
+            new OutageTracker(REFRESH_OUTAGE_REPEAT_MILLIS, System::currentTimeMillis);
 
     private FelisVelocityConfig config;
     private LinkClient linkClient;
@@ -169,6 +177,7 @@ public final class FelisVelocityPlugin {
         refreshRegistrations();
         repeating(REGISTRATION_REFRESH, this::refreshRegistrations);
         repeating(WAIT_POLL, router::tick);
+        repeating(STATS_INTERVAL, this::logStats);
 
         this.routingActive = true;
         logger.info("Felis routing ready: rootDomain={}, login={}, lobby={}. /link, /felis and /invite registered.",
@@ -205,6 +214,7 @@ public final class FelisVelocityPlugin {
         if (apiCalls.submit(task)) {
             return true;
         }
+        stats.count(ProxyStats.Event.POOL_REFUSED);
         long now = System.currentTimeMillis();
         long last = lastBusyLog.get();
         if (now - last >= BUSY_LOG_INTERVAL_MILLIS && lastBusyLog.compareAndSet(last, now)) {
@@ -214,6 +224,20 @@ public final class FelisVelocityPlugin {
         return false;
     }
 
+    /** stats is the proxy's failure counters, for the health line and /felis. */
+    ProxyStats stats() {
+        return stats;
+    }
+
+    // logStats writes the periodic health line; an idle window writes nothing.
+    private void logStats() {
+        String line = ProxyStats.line(STATS_INTERVAL.toMinutes(), apiClient.stats().window(), stats.window(),
+                router.waitingCount());
+        if (line != null) {
+            logger.info(line);
+        }
+    }
+
     /** async for a task a player is waiting on: a refusal is told to them at once. */
     void async(Player player, Runnable task) {
         if (!async(task)) {
@@ -263,15 +287,30 @@ public final class FelisVelocityPlugin {
         if (r.servers != null) {
             registry.refresh(r.servers);
         }
-        if (r.restored) {
-            logger.warn("Felis: felis-api is unreachable at startup (status={}): {}; routing to the {} backends "
-                            + "in the saved server list until it answers.",
-                    r.failure.statusCode(), r.failure.getMessage(), r.servers.size());
-        } else if (r.failure != null) {
-            // Keep existing registrations on a control-plane blip (spec §11): a
-            // transient failure must never deregister live backends.
-            logger.warn("Felis: server list refresh failed (status={}): {}; keeping current registrations.",
-                    r.failure.statusCode(), r.failure.getMessage());
+        if (r.failure == null) {
+            if (refreshOutage.success() == OutageTracker.Report.RECOVERED) {
+                logger.info("Felis: server list refresh recovered after {} failed attempts over {} s.",
+                        refreshOutage.lastOutageFailures(), refreshOutage.lastOutageMillis() / 1000);
+            }
+        } else {
+            stats.count(ProxyStats.Event.REFRESH_FAILED);
+            OutageTracker.Report report = refreshOutage.failure();
+            if (r.restored) {
+                logger.warn("Felis: felis-api is unreachable at startup (status={}): {}; routing to the {} backends "
+                                + "in the saved server list until it answers.",
+                        r.failure.statusCode(), r.failure.getMessage(), r.servers.size());
+            } else if (report == OutageTracker.Report.DOWN) {
+                // Keep existing registrations on a control-plane blip (spec §11): a
+                // transient failure must never deregister live backends.
+                logger.warn("Felis: server list refresh failed (status={}): {}; keeping current registrations. "
+                                + "Repeats are summarized every {} min until it recovers.",
+                        r.failure.statusCode(), r.failure.getMessage(), REFRESH_OUTAGE_REPEAT_MILLIS / 60_000L);
+            } else if (report == OutageTracker.Report.STILL_DOWN) {
+                logger.warn("Felis: server list refresh still failing: {} failed attempts over {} s, last (status={}): {}; "
+                                + "keeping current registrations.",
+                        refreshOutage.failures(), refreshOutage.downForMillis() / 1000,
+                        r.failure.statusCode(), r.failure.getMessage());
+            }
         }
         if (r.fileError != null) {
             logger.warn("Felis: saved server list {}: {}", r.failure == null ? "not written" : "not usable",
@@ -492,6 +531,17 @@ public final class FelisVelocityPlugin {
         source.sendMessage(field("login", config.loginServer()));
         source.sendMessage(field("lobby", config.lobbyServer()));
         source.sendMessage(field("servers", String.valueOf(registry.all().size())));
+        if (!(source instanceof Player)) {
+            // Operator health, console only: felis-api as this proxy sees it, and the
+            // proxy-side failures since start.
+            source.sendMessage(field("felis-api", apiClient.stats().total().summary()));
+            source.sendMessage(field("waiting", String.valueOf(router.waitingCount())));
+            StringBuilder failures = new StringBuilder();
+            for (ProxyStats.Event e : ProxyStats.Event.values()) {
+                failures.append(failures.length() == 0 ? "" : ", ").append(e.label).append('=').append(stats.total(e));
+            }
+            source.sendMessage(field("since start", failures.toString()));
+        }
         source.sendMessage(Component.text(
                 zh ? "  /felis help 查看命令" : "  /felis help for commands", NamedTextColor.GRAY));
     }

plugins/velocity/src/main/java/best/lolicon/felis/velocity/ProxyStats.java

@@ -0,0 +1,71 @@
+package best.lolicon.felis.velocity;
+
+import best.lolicon.felis.link.ApiStats;
+
+import java.util.concurrent.atomic.AtomicLongArray;
+
+/**
+ * ProxyStats counts the proxy-side failures that each already get a warning line, so
+ * an operator can see their rate without grepping: calls the bounded pool refused,
+ * join-events that failed or were dropped (each one leaves the reaper blind to a real
+ * join and skips an allowlist append), transfers that did not connect, and server-list
+ * refreshes that failed. {@link #window()} returns what changed since the previous
+ * window; {@link #line} folds that and felis-api's own window into the periodic log
+ * line.
+ */
+final class ProxyStats {
+    enum Event {
+        POOL_REFUSED("busy refusals"),
+        JOIN_EVENT_FAILED("join-events failed"),
+        JOIN_EVENT_DROPPED("join-events dropped"),
+        TRANSFER_FAILED("transfers failed"),
+        REFRESH_FAILED("server-list refreshes failed");
+
+        final String label;
+
+        Event(String label) {
+            this.label = label;
+        }
+    }
+
+    private static final Event[] EVENTS = Event.values();
+    private final AtomicLongArray counts = new AtomicLongArray(EVENTS.length);
+    private final long[] lastWindow = new long[EVENTS.length];
+
+    void count(Event e) {
+        counts.incrementAndGet(e.ordinal());
+    }
+
+    long total(Event e) {
+        return counts.get(e.ordinal());
+    }
+
+    /** window returns each event's count since the previous call, indexed by ordinal. */
+    synchronized long[] window() {
+        long[] delta = new long[EVENTS.length];
+        for (int i = 0; i < EVENTS.length; i++) {
+            long now = counts.get(i);
+            delta[i] = now - lastWindow[i];
+            lastWindow[i] = now;
+        }
+        return delta;
+    }
+
+    /**
+     * line is the periodic health line for one window, or null when the window saw no
+     * felis-api call, no failure and nobody waiting (an idle proxy logs nothing).
+     */
+    static String line(long minutes, ApiStats.Snapshot api, long[] events, int waiting) {
+        boolean any = api.calls > 0 || waiting > 0;
+        StringBuilder sb = new StringBuilder();
+        for (Event e : EVENTS) {
+            long n = events[e.ordinal()];
+            any |= n > 0;
+            sb.append(", ").append(e.label).append('=').append(n);
+        }
+        if (!any) {
+            return null;
+        }
+        return "Felis: last " + minutes + " min: felis-api " + api.summary() + sb + ", waiting now=" + waiting;
+    }
+}

plugins/velocity/src/main/java/best/lolicon/felis/velocity/WaitingRouter.java

@@ -349,15 +349,22 @@ public final class WaitingRouter {
             } catch (LinkException e) {
                 // A lost join-event leaves the reaper blind to real activity and skips
                 // the allowlist append, so it is an operator-visible failure.
+                plugin.stats().count(ProxyStats.Event.JOIN_EVENT_FAILED);
                 log.warn("Felis: join-event for {} on {} failed (status={}): {}",
                         id, name, e.statusCode(), e.getMessage());
             }
         });
         if (!taken) {
+            plugin.stats().count(ProxyStats.Event.JOIN_EVENT_DROPPED);
             log.warn("Felis: join-event for {} on {} dropped: the felis-api call queue is full", id, name);
         }
     }
 
+    /** waitingCount is how many players are parked for a backend right now. */
+    int waitingCount() {
+        return waiting.size();
+    }
+
     /** tick drains the waiting queue; the plugin schedules it on the async pool. */
     void tick() {
         if (waiting.isEmpty() || !ticking.compareAndSet(false, true)) {
@@ -556,6 +563,7 @@ public final class WaitingRouter {
     private void transfer(Player player, String serverName, RegisteredServer backend) {
         player.createConnectionRequest(backend).connect().whenComplete((result, err) -> {
             if (err != null || (result != null && !result.isSuccessful())) {
+                plugin.stats().count(ProxyStats.Event.TRANSFER_FAILED);
                 log.warn("Felis: transfer of {} to {} failed: {}", player.getUniqueId(), serverName,
                         err != null ? err.toString() : result.getStatus());
                 player.sendMessage(Component.text(

plugins/velocity/test/best/lolicon/felis/velocity/ProxyStatsTest.java

@@ -0,0 +1,92 @@
+package best.lolicon.felis.velocity;
+
+import best.lolicon.felis.link.ApiStats;
+import best.lolicon.felis.link.FelisApiClient;
+import best.lolicon.felis.link.LinkConfig;
+import best.lolicon.felis.link.LinkException;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.util.UUID;
+
+/**
+ * ProxyStatsTest checks the proxy's failure counters: windows never overlap, totals
+ * keep growing, and the periodic line names every counter with its window value and
+ * stays silent for an idle window. Framework free: a failed assertion throws.
+ *
+ * <p>Run: {@code javac -d <out> shared/src/main/java/best/lolicon/felis/link/*.java
+ * velocity/src/main/java/best/lolicon/felis/velocity/ProxyStats.java
+ * velocity/test/best/lolicon/felis/velocity/ProxyStatsTest.java && java -cp <out>
+ * best.lolicon.felis.velocity.ProxyStatsTest}.
+ */
+public final class ProxyStatsTest {
+
+    private static int checks;
+
+    public static void main(String[] args) throws IOException {
+        ProxyStats s = new ProxyStats();
+        s.count(ProxyStats.Event.JOIN_EVENT_FAILED);
+        s.count(ProxyStats.Event.JOIN_EVENT_FAILED);
+        s.count(ProxyStats.Event.TRANSFER_FAILED);
+        s.count(ProxyStats.Event.REFRESH_FAILED);
+        s.count(ProxyStats.Event.REFRESH_FAILED);
+        s.count(ProxyStats.Event.REFRESH_FAILED);
+
+        long[] w1 = s.window();
+        assertEq("w1 pool refused", 0L, w1[ProxyStats.Event.POOL_REFUSED.ordinal()]);
+        assertEq("w1 join failed", 2L, w1[ProxyStats.Event.JOIN_EVENT_FAILED.ordinal()]);
+        assertEq("w1 join dropped", 0L, w1[ProxyStats.Event.JOIN_EVENT_DROPPED.ordinal()]);
+        assertEq("w1 transfer failed", 1L, w1[ProxyStats.Event.TRANSFER_FAILED.ordinal()]);
+        assertEq("w1 refresh failed", 3L, w1[ProxyStats.Event.REFRESH_FAILED.ordinal()]);
+
+        s.count(ProxyStats.Event.POOL_REFUSED);
+        s.count(ProxyStats.Event.JOIN_EVENT_FAILED);
+        long[] w2 = s.window();
+        assertEq("w2 pool refused", 1L, w2[ProxyStats.Event.POOL_REFUSED.ordinal()]);
+        assertEq("w2 join failed (only the new one)", 1L, w2[ProxyStats.Event.JOIN_EVENT_FAILED.ordinal()]);
+        assertEq("w2 refresh failed (none new)", 0L, w2[ProxyStats.Event.REFRESH_FAILED.ordinal()]);
+        assertEq("total join failed", 3L, s.total(ProxyStats.Event.JOIN_EVENT_FAILED));
+        assertEq("total refresh failed", 3L, s.total(ProxyStats.Event.REFRESH_FAILED));
+
+        long[] idle = s.window();
+        ApiStats noCalls = new ApiStats();
+        assertEq("idle window logs nothing", null, ProxyStats.line(10, noCalls.window(), idle, 0));
+        assertEq("someone waiting is worth a line",
+                "Felis: last 10 min: felis-api calls=0 (no answer=0, 4xx=0, 5xx=0, retried=0), avg=0 ms, max=0 ms"
+                        + ", busy refusals=0, join-events failed=0, join-events dropped=0, transfers failed=0"
+                        + ", server-list refreshes failed=0, waiting now=2",
+                ProxyStats.line(10, noCalls.window(), idle, 2));
+        assertEq("a failure alone is worth a line",
+                "Felis: last 10 min: felis-api calls=0 (no answer=0, 4xx=0, 5xx=0, retried=0), avg=0 ms, max=0 ms"
+                        + ", busy refusals=1, join-events failed=1, join-events dropped=0, transfers failed=0"
+                        + ", server-list refreshes failed=0, waiting now=0",
+                ProxyStats.line(10, noCalls.window(), w2, 0));
+
+        // One felis-api call and nothing else is still worth a line: a single POST to a
+        // port nobody listens on (POST is never retried, so exactly one attempt).
+        int closed;
+        try (ServerSocket probe = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) {
+            closed = probe.getLocalPort();
+        }
+        FelisApiClient api = new FelisApiClient(new LinkConfig("http://127.0.0.1:" + closed, "t"));
+        try {
+            api.reportJoin("lobby", UUID.fromString("00000000-0000-0000-0000-000000000001"));
+            throw new AssertionError("reportJoin to a closed port succeeded");
+        } catch (LinkException expected) {
+            // the call failed without an answer, which is what we want counted
+        }
+        String one = ProxyStats.line(10, api.stats().window(), s.window(), 0);
+        assertEq("one call alone is worth a line", true, one != null && one.startsWith(
+                "Felis: last 10 min: felis-api calls=1 (no answer=1, 4xx=0, 5xx=0, retried=0), avg="));
+
+        System.out.println("ProxyStatsTest OK (" + checks + " checks)");
+    }
+
+    private static void assertEq(String what, Object want, Object got) {
+        if (want == null ? got != null : !want.equals(got)) {
+            throw new AssertionError(what + ": got " + got + ", want " + want);
+        }
+        checks++;
+    }
+}