feat(plugins): 插件侧计数器与周期健康日志，Limbo/刷新失败按故障周期升级日志

plugins/shared/src/main/java/best/lolicon/felis/link/ApiStats.java

@@ -0,0 +1,103 @@
+package best.lolicon.felis.link;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * ApiStats counts what a {@link FelisApiClient} sent and how it came back, so the
+ * plugin can report felis-api's health as its callers see it: calls, the ones that
+ * never got an answer (refused, reset, timed out), 4xx and 5xx answers, retries, and
+ * latency. Every attempt is one call, a retried GET counting twice. Counters only
+ * grow; {@link #window()} hands out what changed since the previous window, which is
+ * what a periodic log line wants.
+ */
+public final class ApiStats {
+    private final AtomicLong calls = new AtomicLong();
+    private final AtomicLong transport = new AtomicLong();
+    private final AtomicLong clientErrors = new AtomicLong();
+    private final AtomicLong serverErrors = new AtomicLong();
+    private final AtomicLong retries = new AtomicLong();
+    private final AtomicLong totalMillis = new AtomicLong();
+    private final AtomicLong windowMaxMillis = new AtomicLong();
+    private final AtomicLong maxMillis = new AtomicLong();
+    private Snapshot lastWindow = new Snapshot(0, 0, 0, 0, 0, 0, 0);
+
+    /** record notes one attempt: its HTTP status (0 when no answer came) and how long it took. */
+    void record(int status, long millis) {
+        calls.incrementAndGet();
+        if (status == 0) {
+            transport.incrementAndGet();
+        } else if (status >= 500) {
+            serverErrors.incrementAndGet();
+        } else if (status >= 400) {
+            clientErrors.incrementAndGet();
+        }
+        totalMillis.addAndGet(millis);
+        windowMaxMillis.accumulateAndGet(millis, Math::max);
+        maxMillis.accumulateAndGet(millis, Math::max);
+    }
+
+    void retried() {
+        retries.incrementAndGet();
+    }
+
+    /** total is every count since the client was made, with the slowest call ever. */
+    public Snapshot total() {
+        return new Snapshot(calls.get(), transport.get(), clientErrors.get(), serverErrors.get(),
+                retries.get(), totalMillis.get(), maxMillis.get());
+    }
+
+    /**
+     * window returns the counts since the previous call (the first call: since the
+     * client was made), with the slowest call in that span, and starts a new window, so
+     * consecutive windows never overlap.
+     */
+    public synchronized Snapshot window() {
+        long max = windowMaxMillis.getAndSet(0);
+        Snapshot now = new Snapshot(calls.get(), transport.get(), clientErrors.get(), serverErrors.get(),
+                retries.get(), totalMillis.get(), max);
+        Snapshot delta = new Snapshot(
+                now.calls - lastWindow.calls,
+                now.transport - lastWindow.transport,
+                now.clientErrors - lastWindow.clientErrors,
+                now.serverErrors - lastWindow.serverErrors,
+                now.retries - lastWindow.retries,
+                now.totalMillis - lastWindow.totalMillis,
+                max);
+        lastWindow = now;
+        return delta;
+    }
+
+    /** Snapshot is one immutable reading of the counters. */
+    public static final class Snapshot {
+        public final long calls;
+        public final long transport;
+        public final long clientErrors;
+        public final long serverErrors;
+        public final long retries;
+        public final long totalMillis;
+        public final long maxMillis;
+
+        Snapshot(long calls, long transport, long clientErrors, long serverErrors,
+                 long retries, long totalMillis, long maxMillis) {
+            this.calls = calls;
+            this.transport = transport;
+            this.clientErrors = clientErrors;
+            this.serverErrors = serverErrors;
+            this.retries = retries;
+            this.totalMillis = totalMillis;
+            this.maxMillis = maxMillis;
+        }
+
+        /** avgMillis is the mean call time, 0 with no calls. */
+        public long avgMillis() {
+            return calls == 0 ? 0 : totalMillis / calls;
+        }
+
+        /** summary is the one-line form the plugins log and print. */
+        public String summary() {
+            return "calls=" + calls + " (no answer=" + transport + ", 4xx=" + clientErrors
+                    + ", 5xx=" + serverErrors + ", retried=" + retries + "), avg=" + avgMillis()
+                    + " ms, max=" + maxMillis + " ms";
+        }
+    }
+}

plugins/shared/src/main/java/best/lolicon/felis/link/FelisApiClient.java

@@ -54,6 +54,7 @@ public final class FelisApiClient {
 
     private final LinkConfig config;
     private final HttpClient http;
+    private final ApiStats stats = new ApiStats();
 
     public FelisApiClient(LinkConfig config) {
         this.config = Objects.requireNonNull(config, "config");
@@ -62,6 +63,11 @@ public final class FelisApiClient {
                 .build();
     }
 
+    /** stats counts this client's calls and their outcomes, for the plugin's health line. */
+    public ApiStats stats() {
+        return stats;
+    }
+
     /** listServers returns the lifecycle view of every MinecraftServer (GET /servers). */
     public List<ServerView> listServers() throws LinkException {
         return ServerView.listFrom(getObject("/api/v1/servers", 200));
@@ -297,6 +303,7 @@ public final class FelisApiClient {
         } catch (InterruptedException e) {
             throw interrupted(e);
         }
+        stats.retried();
         return expectObject(send(req), expect);
     }
 
@@ -338,8 +345,17 @@ public final class FelisApiClient {
         }
     }
 
+    // exchange is the one place a request goes out, so every attempt is counted once.
     private HttpResponse<String> exchange(HttpRequest req) throws IOException, InterruptedException {
-        return http.send(req, HttpResponse.BodyHandlers.ofString());
+        long start = System.nanoTime();
+        int status = 0;
+        try {
+            HttpResponse<String> res = http.send(req, HttpResponse.BodyHandlers.ofString());
+            status = res.statusCode();
+            return res;
+        } finally {
+            stats.record(status, (System.nanoTime() - start) / 1_000_000);
+        }
     }
 
     // A refused connection arrives as a ConnectException with no message; the class

plugins/shared/src/main/java/best/lolicon/felis/link/OutageTracker.java

@@ -0,0 +1,75 @@
+package best.lolicon.felis.link;
+
+import java.util.function.LongSupplier;
+
+/**
+ * OutageTracker decides which failures of a repeated felis-api call are worth a log
+ * line. A poll that runs every few seconds for every waiting player would bury the
+ * log if each failure were a warning, and hides the outage entirely if each is a
+ * debug line. Instead: the first failure after a success is {@link Report#DOWN}
+ * (log a warning), later failures are {@link Report#STILL_DOWN} at most once per
+ * {@code repeatMillis} (a reminder with the running count), the first success after
+ * failures is {@link Report#RECOVERED} (log how long it lasted), and everything
+ * else is {@link Report#NONE}.
+ */
+public final class OutageTracker {
+    public enum Report { NONE, DOWN, STILL_DOWN, RECOVERED }
+
+    private final LongSupplier clock;
+    private final long repeatMillis;
+    private long downSince = -1;
+    private long lastReport;
+    private long failures;
+    private long lastOutageFailures;
+    private long lastOutageMillis;
+
+    public OutageTracker(long repeatMillis, LongSupplier clock) {
+        this.repeatMillis = repeatMillis;
+        this.clock = clock;
+    }
+
+    public synchronized Report failure() {
+        long now = clock.getAsLong();
+        failures++;
+        if (downSince < 0) {
+            downSince = now;
+            lastReport = now;
+            return Report.DOWN;
+        }
+        if (now - lastReport >= repeatMillis) {
+            lastReport = now;
+            return Report.STILL_DOWN;
+        }
+        return Report.NONE;
+    }
+
+    public synchronized Report success() {
+        if (downSince < 0) {
+            return Report.NONE;
+        }
+        lastOutageFailures = failures;
+        lastOutageMillis = clock.getAsLong() - downSince;
+        downSince = -1;
+        failures = 0;
+        return Report.RECOVERED;
+    }
+
+    /** failures is how many calls have failed in the current outage (0 when up). */
+    public synchronized long failures() {
+        return failures;
+    }
+
+    /** downForMillis is how long the current outage has lasted (0 when up). */
+    public synchronized long downForMillis() {
+        return downSince < 0 ? 0 : clock.getAsLong() - downSince;
+    }
+
+    /** lastOutageFailures and lastOutageMillis describe the outage a RECOVERED report just closed. */
+    public synchronized long lastOutageFailures() {
+        return lastOutageFailures;
+    }
+
+    public synchronized long lastOutageMillis() {
+        return lastOutageMillis;
+    }
+}

plugins/shared/test/best/lolicon/felis/link/FelisApiClientRetryTest.java

@@ -20,7 +20,8 @@ import java.util.concurrent.atomic.AtomicInteger;
  * felis-api that counts the requests each path receives: a GET is tried once more
  * after a connection that dropped mid-reply or a 502/503/504, never a third time; a POST is
  * never repeated (the first attempt may have landed); a GET that ran past the
- * request timeout is cut off at that timeout and not repeated.
+ * request timeout is cut off at that timeout and not repeated. It then checks the
+ * client's {@link ApiStats} against the requests the stub saw.
  *
  * <p>Run: {@code javac -d <out> shared/src/main/java/best/lolicon/felis/link/*.java
  * shared/test/best/lolicon/felis/link/FelisApiClientRetryTest.java && java -cp <out>
@@ -49,6 +50,8 @@ public final class FelisApiClientRetryTest {
             getIsNotRetriedOnAClientError(api);
             postIsNeverRetried(api);
             timedOutGetIsCutOffAndNotRetried(api);
+            statsCountEveryAttempt(api);
+            statsArithmetic();
         } finally {
             stub.stop(0);
             pool.shutdownNow();
@@ -149,6 +152,61 @@ public final class FelisApiClientRetryTest {
         assertEq("slow requests", 1, count("GET /api/v1/internal/servers/slow/status"));
     }
 
+    // The calls above, attempt by attempt: flaky 503+200, cut dropped+200, down 502+502,
+    // gone 404, busy 503 (POST), slow timed out. Nine attempts, three of them retries;
+    // no answer twice (the cut reply, the timeout), one 4xx, four 5xx.
+    private static void statsCountEveryAttempt(FelisApiClient api) throws LinkException {
+        ApiStats.Snapshot t = api.stats().total();
+        int sent = hits.values().stream().mapToInt(AtomicInteger::get).sum();
+        assertEq("stats calls = requests the stub saw", (long) sent, t.calls);
+        assertEq("stats calls", 9L, t.calls);
+        assertEq("stats no answer", 2L, t.transport);
+        assertEq("stats 4xx", 1L, t.clientErrors);
+        assertEq("stats 5xx", 4L, t.serverErrors);
+        assertEq("stats retried", 3L, t.retries);
+        // The slowest attempt is the one cut off by the 400 ms request timeout.
+        if (t.maxMillis < 390 || t.maxMillis >= 1200) {
+            throw new AssertionError("stats max: " + t.maxMillis + " ms, want the ~400 ms timeout");
+        }
+        checks++;
+
+        ApiStats.Snapshot w1 = api.stats().window();
+        assertEq("first window = totals so far", 9L, w1.calls);
+        assertEq("first window 5xx", 4L, w1.serverErrors);
+        assertEq("first window max", t.maxMillis, w1.maxMillis);
+        ApiStats.Snapshot w2 = api.stats().window();
+        assertEq("empty window calls", 0L, w2.calls);
+        assertEq("empty window max", 0L, w2.maxMillis);
+        assertEq("empty window avg", 0L, w2.avgMillis());
+        assertEq("total max outlives the windows", t.maxMillis, api.stats().total().maxMillis);
+        api.serverStatus("flaky"); // answers 200 from now on
+        ApiStats.Snapshot w3 = api.stats().window();
+        assertEq("next window calls", 1L, w3.calls);
+        assertEq("next window 5xx", 0L, w3.serverErrors);
+        assertEq("next window retried", 0L, w3.retries);
+        assertEq("totals keep growing", 10L, api.stats().total().calls);
+    }
+
+    // The status classes at their edges and the latency figures, on fixed inputs.
+    private static void statsArithmetic() {
+        ApiStats s = new ApiStats();
+        s.record(200, 10);
+        s.record(399, 40);
+        s.record(400, 30);
+        s.record(499, 5);
+        s.record(500, 20);
+        s.record(0, 15);
+        ApiStats.Snapshot t = s.total();
+        assertEq("edge calls", 6L, t.calls);
+        assertEq("edge 4xx (400, 499)", 2L, t.clientErrors);
+        assertEq("edge 5xx (500)", 1L, t.serverErrors);
+        assertEq("edge no answer", 1L, t.transport);
+        assertEq("avg of 10+40+30+5+20+15 over 6", 20L, t.avgMillis());
+        assertEq("max is the slowest, not the last", 40L, t.maxMillis);
+        assertEq("summary", "calls=6 (no answer=1, 4xx=2, 5xx=1, retried=0), avg=20 ms, max=40 ms",
+                t.summary());
+    }
+
     // ---- harness ----
 
     interface Call {

plugins/shared/test/best/lolicon/felis/link/OutageTrackerTest.java

@@ -0,0 +1,62 @@
+package best.lolicon.felis.link;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * OutageTrackerTest walks an {@link OutageTracker} through an outage on an injected
+ * clock: one DOWN on the first failure, STILL_DOWN no more than once per repeat
+ * interval, RECOVERED once with the outage's length and failure count, and silence
+ * while all is well.
+ *
+ * <p>Run: {@code javac -d <out> shared/src/main/java/best/lolicon/felis/link/*.java
+ * shared/test/best/lolicon/felis/link/OutageTrackerTest.java && java -cp <out>
+ * best.lolicon.felis.link.OutageTrackerTest}.
+ */
+public final class OutageTrackerTest {
+
+    private static int checks;
+
+    public static void main(String[] args) {
+        AtomicLong now = new AtomicLong(10_000L);
+        OutageTracker t = new OutageTracker(60_000L, now::get);
+
+        assertEq("success while up", OutageTracker.Report.NONE, t.success());
+        assertEq("failures while up", 0L, t.failures());
+
+        assertEq("first failure", OutageTracker.Report.DOWN, t.failure());
+        now.addAndGet(2_000L);
+        assertEq("second failure 2 s later", OutageTracker.Report.NONE, t.failure());
+        now.addAndGet(57_999L); // 59.999 s after DOWN
+        assertEq("just under the repeat interval", OutageTracker.Report.NONE, t.failure());
+        now.addAndGet(1L); // 60 s after DOWN
+        assertEq("repeat interval reached", OutageTracker.Report.STILL_DOWN, t.failure());
+        now.addAndGet(30_000L);
+        assertEq("half an interval after the reminder", OutageTracker.Report.NONE, t.failure());
+        now.addAndGet(30_000L);
+        assertEq("an interval after the reminder", OutageTracker.Report.STILL_DOWN, t.failure());
+        assertEq("failures in the outage", 6L, t.failures());
+        assertEq("outage length so far", 120_000L, t.downForMillis());
+
+        now.addAndGet(5_000L);
+        assertEq("first success", OutageTracker.Report.RECOVERED, t.success());
+        assertEq("closed outage failures", 6L, t.lastOutageFailures());
+        assertEq("closed outage length", 125_000L, t.lastOutageMillis());
+        assertEq("failures reset", 0L, t.failures());
+        assertEq("down-for reset", 0L, t.downForMillis());
+        assertEq("second success", OutageTracker.Report.NONE, t.success());
+
+        // A new outage starts its own count and is announced again at once.
+        now.addAndGet(1_000L);
+        assertEq("next outage", OutageTracker.Report.DOWN, t.failure());
+        assertEq("next outage failures", 1L, t.failures());
+
+        System.out.println("OutageTrackerTest OK (" + checks + " checks)");
+    }
+
+    private static void assertEq(String what, Object want, Object got) {
+        if (!want.equals(got)) {
+            throw new AssertionError(what + ": got " + got + ", want " + want);
+        }
+        checks++;
+    }
+}