feat(velocity): felis-api 调用改走有界线程池、定时任务防重叠,超时可配置,幂等 GET 带抖动重试一次

This commit is contained in:
Lemon-miaow committed 2026-09-26 00:03:19 +08:00
1 parent 001f060027
commit fcf5c305ea
16 files changed
+872 -37

No files matched your search

+2
View File
@@ -58,6 +58,8 @@ Configuration (deployment inputs, never compiled in; env wins over a
| `FELIS_LOBBY_SERVER` | Velocity server name to transfer to | `lobby` | | `FELIS_LOBBY_SERVER` | Velocity server name to transfer to | `lobby` |
| `FELIS_LOGIN_TIMEOUT_SECONDS` | login window (clamped 30–3600) | `600` | | `FELIS_LOGIN_TIMEOUT_SECONDS` | login window (clamped 30–3600) | `600` |
| `FELIS_HEALTH_PORT` | readiness port | `8080` | | `FELIS_HEALTH_PORT` | readiness port | `8080` |
| `FELIS_API_CONNECT_TIMEOUT_SECONDS` | felis-api connect timeout (1–120) | `10` |
| `FELIS_API_REQUEST_TIMEOUT_SECONDS` | felis-api call timeout (1–120) | `10` |
If the API config **or** the root domain is absent the login flow stays **OFF** and If the API config **or** the root domain is absent the login flow stays **OFF** and
the plugin runs readiness-only (the same "load un-crippled" fail-safe the other the plugin runs readiness-only (the same "load un-crippled" fail-safe the other
+18 -1
View File
@@ -120,7 +120,15 @@ so there is nothing to shade and each jar is self-contained.
- `LinkConfigLoader` — reads `FELIS_API_BASE_URL` / `FELIS_SERVICE_TOKEN` (env - `LinkConfigLoader` — reads `FELIS_API_BASE_URL` / `FELIS_SERVICE_TOKEN` (env
wins) or a `felis-link.properties` file written as a commented template on wins) or a `felis-link.properties` file written as a commented template on
first run. **The API URL and service token are deployment inputs and are never first run. **The API URL and service token are deployment inputs and are never
compiled in.** compiled in.** Two optional keys bound each call, in whole seconds from 1 to 120
(default 10): `connect-timeout-seconds` / `FELIS_API_CONNECT_TIMEOUT_SECONDS`
for opening the connection, `request-timeout-seconds` /
`FELIS_API_REQUEST_TIMEOUT_SECONDS` for the whole call. Anything else fails the
load with the key named.
- `FelisApiClient` — the routing client. A GET that failed on a dropped
connection or a 502/503/504 is tried once more after a 100–400 ms jittered
pause; a GET that timed out, and every POST (wake, claim, join-event,
approvals), is never repeated.
Threading: the command runs on the server thread; the HTTP call is dispatched to Threading: the command runs on the server thread; the HTTP call is dispatched to
a daemon single-thread executor and the reply is hopped back onto the server a daemon single-thread executor and the reply is hopped back onto the server
@@ -175,6 +183,15 @@ Velocity-only config keys (read from the same `felis-link.properties` / env as
| `login-server` | `FELIS_LOGIN_SERVER` | The system auth gate every fresh connection must pass. Defaults to `login`. | | `login-server` | `FELIS_LOGIN_SERVER` | The system auth gate every fresh connection must pass. Defaults to `login`. |
| `lobby-server` | `FELIS_LOBBY_SERVER` | The distinct post-auth holding server used while a backend wakes. Defaults to `lobby`; it must not equal `login-server`. | | `lobby-server` | `FELIS_LOBBY_SERVER` | The distinct post-auth holding server used while a backend wakes. Defaults to `lobby`; it must not equal `login-server`. |
Load bounds on the proxy: felis-api calls run on a pool of 8 threads with 64
waiting slots. Past that a call is refused at once — the player reads "busy", a
lobby frame gets a `busy` error, and a dropped join-event is logged at warn —
rather than piling up threads while felis-api is slow. The registration refresh
(15 s) and the waiting-queue poll (2 s) skip a run that falls due while the last
one is still going. Acting commands (`/link`, `/felis claim`, migrate, op
approve) share a per-player budget of 5 then one per 5 s; felis:control frames
from the lobby are metered per player and menu status answers are cached.
## Lobby menu (§12) ## Lobby menu (§12)
The `paper/` module is the lobby's player-facing face for §27 scenario 10 The `paper/` module is the lobby's player-facing face for §27 scenario 10
@@ -384,6 +384,9 @@ public final class FelisPaperPlugin extends JavaPlugin implements Listener, Plug
case "invalid_server_name": case "invalid_server_name":
return zh ? "这台服务器已不存在。" return zh ? "这台服务器已不存在。"
: "That server no longer exists."; : "That server no longer exists.";
case "busy":
return zh ? "Felis 现在很忙,请过一会儿再试。"
: "Felis is busy right now — try again in a moment.";
case "transport_error": case "transport_error":
case "interrupted": case "interrupted":
return zh ? "Felis 暂时不可用,请稍后再试。" return zh ? "Felis 暂时不可用,请稍后再试。"
@@ -6,11 +6,14 @@ import java.net.URLEncoder;
import java.net.http.HttpClient; import java.net.http.HttpClient;
import java.net.http.HttpRequest; import java.net.http.HttpRequest;
import java.net.http.HttpResponse; import java.net.http.HttpResponse;
import java.net.http.HttpTimeoutException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import java.util.regex.Pattern; import java.util.regex.Pattern;
/** /**
@@ -43,6 +46,11 @@ import java.util.regex.Pattern;
public final class FelisApiClient { public final class FelisApiClient {
// Mirrors internal/naming.serverNameRE plus its no-leading/trailing-dash rule. // Mirrors internal/naming.serverNameRE plus its no-leading/trailing-dash rule.
private static final Pattern SERVER_NAME = Pattern.compile("^[a-z0-9][a-z0-9-]{1,30}[a-z0-9]$"); private static final Pattern SERVER_NAME = Pattern.compile("^[a-z0-9][a-z0-9-]{1,30}[a-z0-9]$");
// Answers a second GET can outlive: felis-api restarting behind its Service, or a
// gateway with no ready endpoint.
private static final Set<Integer> RETRY_STATUSES = Set.of(502, 503, 504);
private static final int RETRY_PAUSE_MIN_MILLIS = 100;
private static final int RETRY_PAUSE_JITTER_MILLIS = 300;
private final LinkConfig config; private final LinkConfig config;
private final HttpClient http; private final HttpClient http;
@@ -50,7 +58,7 @@ public final class FelisApiClient {
public FelisApiClient(LinkConfig config) { public FelisApiClient(LinkConfig config) {
this.config = Objects.requireNonNull(config, "config"); this.config = Objects.requireNonNull(config, "config");
this.http = HttpClient.newBuilder() this.http = HttpClient.newBuilder()
.connectTimeout(config.timeout()) .connectTimeout(config.connectTimeout())
.build(); .build();
} }
@@ -261,8 +269,34 @@ public final class FelisApiClient {
return URLEncoder.encode(value, StandardCharsets.UTF_8).replace("+", "%20"); return URLEncoder.encode(value, StandardCharsets.UTF_8).replace("+", "%20");
} }
// getObject tries a GET a second time, after a jittered 100-400 ms pause, when the
// first attempt failed in a way a retry can fix: the connection was refused or
// reset (felis-api restarting, its pod moving) or the answer was 502/503/504. GETs
// only read, so repeating one is safe; POSTs (wake, claim, join-event, approvals)
// are never repeated, because the first attempt may have landed. A GET that timed
// out is not repeated either: felis-api is then slow rather than gone, and a second
// full wait would hold the caller twice as long while adding load to an API that
// is already behind. The jitter keeps a proxy's queued callers from retrying in
// one burst.
private Map<?, ?> getObject(String path, int expect) throws LinkException { private Map<?, ?> getObject(String path, int expect) throws LinkException {
HttpRequest req = base(path).GET().build(); HttpRequest req = base(path).GET().build();
try {
HttpResponse<String> res = exchange(req);
if (!RETRY_STATUSES.contains(res.statusCode())) {
return expectObject(res, expect);
}
} catch (HttpTimeoutException e) {
throw transportError(e);
} catch (IOException e) {
// refused or reset: retried below
} catch (InterruptedException e) {
throw interrupted(e);
}
try {
Thread.sleep(RETRY_PAUSE_MIN_MILLIS + ThreadLocalRandom.current().nextInt(RETRY_PAUSE_JITTER_MILLIS + 1));
} catch (InterruptedException e) {
throw interrupted(e);
}
return expectObject(send(req), expect); return expectObject(send(req), expect);
} }
@@ -289,23 +323,37 @@ public final class FelisApiClient {
} }
return HttpRequest.newBuilder() return HttpRequest.newBuilder()
.uri(uri) .uri(uri)
.timeout(config.timeout()) .timeout(config.requestTimeout())
.header("Authorization", "Bearer " + config.serviceToken()) .header("Authorization", "Bearer " + config.serviceToken())
.header("Accept", "application/json"); .header("Accept", "application/json");
} }
private HttpResponse<String> send(HttpRequest req) throws LinkException { private HttpResponse<String> send(HttpRequest req) throws LinkException {
try { try {
return http.send(req, HttpResponse.BodyHandlers.ofString()); return exchange(req);
} catch (IOException e) { } catch (IOException e) {
throw new LinkException(0, "transport_error", throw transportError(e);
"could not reach felis-api: " + e.getMessage(), e);
} catch (InterruptedException e) { } catch (InterruptedException e) {
Thread.currentThread().interrupt(); throw interrupted(e);
throw new LinkException(0, "interrupted", "felis-api request interrupted", e);
} }
} }
private HttpResponse<String> exchange(HttpRequest req) throws IOException, InterruptedException {
return http.send(req, HttpResponse.BodyHandlers.ofString());
}
// A refused connection arrives as a ConnectException with no message; the class
// name keeps the log line from reading "could not reach felis-api: null".
private static LinkException transportError(IOException e) {
String why = e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage();
return new LinkException(0, "transport_error", "could not reach felis-api: " + why, e);
}
private static LinkException interrupted(InterruptedException e) {
Thread.currentThread().interrupt();
return new LinkException(0, "interrupted", "felis-api request interrupted", e);
}
private Map<?, ?> expectObject(HttpResponse<String> res, int expect) throws LinkException { private Map<?, ?> expectObject(HttpResponse<String> res, int expect) throws LinkException {
int status = res.statusCode(); int status = res.statusCode();
if (status != expect) { if (status != expect) {
@@ -40,7 +40,7 @@ public final class LinkClient {
public LinkClient(LinkConfig config) { public LinkClient(LinkConfig config) {
this.config = Objects.requireNonNull(config, "config"); this.config = Objects.requireNonNull(config, "config");
this.http = HttpClient.newBuilder() this.http = HttpClient.newBuilder()
.connectTimeout(config.timeout()) .connectTimeout(config.connectTimeout())
.build(); .build();
} }
@@ -50,7 +50,7 @@ public final class LinkClient {
String body = "{\"mc_uuid\":\"" + mcUuid + "\"}"; String body = "{\"mc_uuid\":\"" + mcUuid + "\"}";
HttpRequest req = HttpRequest.newBuilder() HttpRequest req = HttpRequest.newBuilder()
.uri(URI.create(config.apiBaseUrl() + PATH)) .uri(URI.create(config.apiBaseUrl() + PATH))
.timeout(config.timeout()) .timeout(config.requestTimeout())
.header("Authorization", "Bearer " + config.serviceToken()) .header("Authorization", "Bearer " + config.serviceToken())
.header("Content-Type", "application/json") .header("Content-Type", "application/json")
.header("Accept", "application/json") .header("Accept", "application/json")
@@ -10,29 +10,39 @@ import java.util.Objects;
* and are <em>never</em> compiled in. Keeping them out of source is what lets the * and are <em>never</em> compiled in. Keeping them out of source is what lets the
* tree stay domain- and credential-free; each platform's config loader is * tree stay domain- and credential-free; each platform's config loader is
* responsible for sourcing them. * responsible for sourcing them.
*
* <p>Two timeouts bound each call: {@code connectTimeout} for opening the TCP
* connection and {@code requestTimeout} for the whole exchange once it is open. A
* felis-api that is gone fails fast on the first; one that is slow is cut off by the
* second, so a blocked call cannot hold a plugin thread indefinitely.
*/ */
public final class LinkConfig { public final class LinkConfig {
private final String apiBaseUrl; private final String apiBaseUrl;
private final String serviceToken; private final String serviceToken;
private final Duration timeout; private final Duration connectTimeout;
private final Duration requestTimeout;
public LinkConfig(String apiBaseUrl, String serviceToken, Duration timeout) { public LinkConfig(String apiBaseUrl, String serviceToken, Duration connectTimeout, Duration requestTimeout) {
this.apiBaseUrl = stripTrailingSlash(Objects.requireNonNull(apiBaseUrl, "apiBaseUrl")); this.apiBaseUrl = stripTrailingSlash(Objects.requireNonNull(apiBaseUrl, "apiBaseUrl"));
this.serviceToken = Objects.requireNonNull(serviceToken, "serviceToken"); this.serviceToken = Objects.requireNonNull(serviceToken, "serviceToken");
this.timeout = Objects.requireNonNull(timeout, "timeout"); this.connectTimeout = Objects.requireNonNull(connectTimeout, "connectTimeout");
this.requestTimeout = Objects.requireNonNull(requestTimeout, "requestTimeout");
if (this.apiBaseUrl.isEmpty()) { if (this.apiBaseUrl.isEmpty()) {
throw new IllegalArgumentException("apiBaseUrl is empty"); throw new IllegalArgumentException("apiBaseUrl is empty");
} }
if (this.serviceToken.isEmpty()) { if (this.serviceToken.isEmpty()) {
throw new IllegalArgumentException("serviceToken is empty"); throw new IllegalArgumentException("serviceToken is empty");
} }
if (this.timeout.isZero() || this.timeout.isNegative()) { if (this.connectTimeout.isZero() || this.connectTimeout.isNegative()) {
throw new IllegalArgumentException("timeout must be positive"); throw new IllegalArgumentException("connectTimeout must be positive");
}
if (this.requestTimeout.isZero() || this.requestTimeout.isNegative()) {
throw new IllegalArgumentException("requestTimeout must be positive");
} }
} }
public LinkConfig(String apiBaseUrl, String serviceToken) { public LinkConfig(String apiBaseUrl, String serviceToken) {
this(apiBaseUrl, serviceToken, Duration.ofSeconds(10)); this(apiBaseUrl, serviceToken, Duration.ofSeconds(10), Duration.ofSeconds(10));
} }
public String apiBaseUrl() { public String apiBaseUrl() {
@@ -43,8 +53,12 @@ public final class LinkConfig {
return serviceToken; return serviceToken;
} }
public Duration timeout() { public Duration connectTimeout() {
return timeout; return connectTimeout;
}
public Duration requestTimeout() {
return requestTimeout;
} }
private static String stripTrailingSlash(String u) { private static String stripTrailingSlash(String u) {
@@ -8,6 +8,7 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.Duration; import java.time.Duration;
import java.util.Properties; import java.util.Properties;
import java.util.function.UnaryOperator;
/** /**
* LinkConfigLoader resolves a {@link LinkConfig} the same way on every platform: * LinkConfigLoader resolves a {@link LinkConfig} the same way on every platform:
@@ -18,12 +19,25 @@ import java.util.Properties;
* the source tree stays domain- and credential-free. On first run it writes a * the source tree stays domain- and credential-free. On first run it writes a
* commented template and then reports the values as missing, so an operator gets * commented template and then reports the values as missing, so an operator gets
* a file to fill in rather than a silent half-configured plugin. * a file to fill in rather than a silent half-configured plugin.
*
* <p>The two call timeouts ({@code connect-timeout-seconds},
* {@code request-timeout-seconds}, or {@code FELIS_API_CONNECT_TIMEOUT_SECONDS} /
* {@code FELIS_API_REQUEST_TIMEOUT_SECONDS}) are optional and default to
* {@value #DEFAULT_TIMEOUT_SECONDS} s. A value that is not a whole number of seconds
* from 1 to {@value #MAX_TIMEOUT_SECONDS} fails the load: a typo quietly falling back
* to the default would hide the setting the operator meant to change.
*/ */
public final class LinkConfigLoader { public final class LinkConfigLoader {
public static final String ENV_URL = "FELIS_API_BASE_URL"; public static final String ENV_URL = "FELIS_API_BASE_URL";
public static final String ENV_TOKEN = "FELIS_SERVICE_TOKEN"; public static final String ENV_TOKEN = "FELIS_SERVICE_TOKEN";
private static final String KEY_URL = "api-base-url"; private static final String KEY_URL = "api-base-url";
public static final String ENV_CONNECT_TIMEOUT = "FELIS_API_CONNECT_TIMEOUT_SECONDS";
public static final String ENV_REQUEST_TIMEOUT = "FELIS_API_REQUEST_TIMEOUT_SECONDS";
static final int DEFAULT_TIMEOUT_SECONDS = 10;
static final int MAX_TIMEOUT_SECONDS = 120;
private static final String KEY_TOKEN = "service-token"; private static final String KEY_TOKEN = "service-token";
private static final String KEY_CONNECT_TIMEOUT = "connect-timeout-seconds";
private static final String KEY_REQUEST_TIMEOUT = "request-timeout-seconds";
private LinkConfigLoader() { private LinkConfigLoader() {
} }
@@ -33,9 +47,15 @@ public final class LinkConfigLoader {
* if the file does not yet exist. * if the file does not yet exist.
* *
* @throws IOException if the file cannot be read/created, or if neither the * @throws IOException if the file cannot be read/created, or if neither the
* environment nor the file supplies both required values. * environment nor the file supplies both required values, or if a
* timeout is not a whole number of seconds in range.
*/ */
public static LinkConfig load(Path propertiesFile) throws IOException { public static LinkConfig load(Path propertiesFile) throws IOException {
return load(propertiesFile, System::getenv);
}
// load with the environment passed in, so the precedence rules are testable.
static LinkConfig load(Path propertiesFile, UnaryOperator<String> env) throws IOException {
Properties props = new Properties(); Properties props = new Properties();
if (Files.exists(propertiesFile)) { if (Files.exists(propertiesFile)) {
try (InputStream in = Files.newInputStream(propertiesFile)) { try (InputStream in = Files.newInputStream(propertiesFile)) {
@@ -45,14 +65,36 @@ public final class LinkConfigLoader {
writeTemplate(propertiesFile); writeTemplate(propertiesFile);
} }
String url = firstNonBlank(System.getenv(ENV_URL), props.getProperty(KEY_URL)); String url = firstNonBlank(env.apply(ENV_URL), props.getProperty(KEY_URL));
String token = firstNonBlank(System.getenv(ENV_TOKEN), props.getProperty(KEY_TOKEN)); String token = firstNonBlank(env.apply(ENV_TOKEN), props.getProperty(KEY_TOKEN));
if (isBlank(url) || isBlank(token)) { if (isBlank(url) || isBlank(token)) {
throw new IOException("set " + ENV_URL + "/" + ENV_TOKEN throw new IOException("set " + ENV_URL + "/" + ENV_TOKEN
+ " or fill in " + propertiesFile + " (" + KEY_URL + ", " + KEY_TOKEN + ")"); + " or fill in " + propertiesFile + " (" + KEY_URL + ", " + KEY_TOKEN + ")");
} }
return new LinkConfig(url, token, Duration.ofSeconds(10)); Duration connect = seconds(KEY_CONNECT_TIMEOUT, ENV_CONNECT_TIMEOUT,
firstNonBlank(env.apply(ENV_CONNECT_TIMEOUT), props.getProperty(KEY_CONNECT_TIMEOUT)));
Duration request = seconds(KEY_REQUEST_TIMEOUT, ENV_REQUEST_TIMEOUT,
firstNonBlank(env.apply(ENV_REQUEST_TIMEOUT), props.getProperty(KEY_REQUEST_TIMEOUT)));
return new LinkConfig(url, token, connect, request);
}
private static Duration seconds(String key, String envName, String raw) throws IOException {
if (isBlank(raw)) {
return Duration.ofSeconds(DEFAULT_TIMEOUT_SECONDS);
}
String v = raw.trim();
int n;
try {
n = Integer.parseInt(v);
} catch (NumberFormatException e) {
n = 0;
}
if (n < 1 || n > MAX_TIMEOUT_SECONDS) {
throw new IOException(key + " (" + envName + ") must be a whole number of seconds from 1 to "
+ MAX_TIMEOUT_SECONDS + ", got \"" + v + "\"");
}
return Duration.ofSeconds(n);
} }
private static void writeTemplate(Path file) throws IOException { private static void writeTemplate(Path file) throws IOException {
@@ -72,7 +114,14 @@ public final class LinkConfigLoader {
+ KEY_URL + "=\n" + KEY_URL + "=\n"
+ "#\n" + "#\n"
+ "# " + KEY_TOKEN + ": the internal service token (keep this secret).\n" + "# " + KEY_TOKEN + ": the internal service token (keep this secret).\n"
+ KEY_TOKEN + "=\n"; + KEY_TOKEN + "=\n"
+ "#\n"
+ "# Optional call timeouts in whole seconds (1-" + MAX_TIMEOUT_SECONDS + ", default "
+ DEFAULT_TIMEOUT_SECONDS + "): " + KEY_CONNECT_TIMEOUT + " bounds opening the\n"
+ "# connection, " + KEY_REQUEST_TIMEOUT + " the whole call once it is open\n"
+ "# (" + ENV_CONNECT_TIMEOUT + ", " + ENV_REQUEST_TIMEOUT + ").\n"
+ "#" + KEY_CONNECT_TIMEOUT + "=" + DEFAULT_TIMEOUT_SECONDS + "\n"
+ "#" + KEY_REQUEST_TIMEOUT + "=" + DEFAULT_TIMEOUT_SECONDS + "\n";
try (OutputStream out = Files.newOutputStream(file)) { try (OutputStream out = Files.newOutputStream(file)) {
out.write(template.getBytes(StandardCharsets.UTF_8)); out.write(template.getBytes(StandardCharsets.UTF_8));
} }
@@ -0,0 +1,191 @@
package best.lolicon.felis.link;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpServer;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
/**
* FelisApiClientRetryTest pins which failures FelisApiClient repeats, against a stub
* felis-api that counts the requests each path receives: a GET is tried once more
* after a connection that dropped mid-reply or a 502/503/504, never a third time; a POST is
* never repeated (the first attempt may have landed); a GET that ran past the
* request timeout is cut off at that timeout and not repeated.
*
* <p>Run: {@code javac -d <out> shared/src/main/java/best/lolicon/felis/link/*.java
* shared/test/best/lolicon/felis/link/FelisApiClientRetryTest.java && java -cp <out>
* best.lolicon.felis.link.FelisApiClientRetryTest}.
*/
public final class FelisApiClientRetryTest {
private static final String READY = "{\"name\":\"alpha\",\"phase\":\"Running\",\"ready\":true}";
private static final Map<String, AtomicInteger> hits = new ConcurrentHashMap<>();
private static int checks;
public static void main(String[] args) throws Exception {
HttpServer stub = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
// The slow handler sleeps; the others must not queue behind it.
ExecutorService pool = Executors.newCachedThreadPool();
stub.setExecutor(pool);
stub.createContext("/", FelisApiClientRetryTest::handle);
stub.start();
try {
String base = "http://127.0.0.1:" + stub.getAddress().getPort();
FelisApiClient api = new FelisApiClient(new LinkConfig(
base, "test-token", Duration.ofSeconds(2), Duration.ofMillis(400)));
getRetriesOnceAfter503(api);
getRetriesOnceAfterACutReply(api);
getGivesUpAfterTheSecondTry(api);
getIsNotRetriedOnAClientError(api);
postIsNeverRetried(api);
timedOutGetIsCutOffAndNotRetried(api);
} finally {
stub.stop(0);
pool.shutdownNow();
}
System.out.println("FelisApiClientRetryTest OK (" + checks + " checks)");
}
// Each server name is one scripted behaviour; the n-th request to it picks the reply.
private static void handle(HttpExchange ex) throws IOException {
String path = ex.getRequestURI().getRawPath();
ex.getRequestBody().readAllBytes();
int n = hits.computeIfAbsent(ex.getRequestMethod() + " " + path, k -> new AtomicInteger()).incrementAndGet();
switch (path) {
case "/api/v1/internal/servers/flaky/status":
reply(ex, n == 1 ? 503 : 200, n == 1 ? error("unavailable") : READY);
return;
case "/api/v1/internal/servers/cut/status":
if (n == 1) {
// Headers promise 1000 bytes, the connection drops after 10: felis-api
// dying mid-reply. (A drop before any response byte is not a usable
// probe: the JDK client resends such a GET on its own.)
ex.getResponseHeaders().set("Content-Type", "application/json");
ex.sendResponseHeaders(200, 1000);
ex.getResponseBody().write("{\"name\":\"a".getBytes(StandardCharsets.UTF_8));
ex.getResponseBody().flush();
ex.close();
return;
}
reply(ex, 200, READY);
return;
case "/api/v1/internal/servers/down/status":
reply(ex, 502, error("bad_gateway"));
return;
case "/api/v1/internal/servers/gone/status":
reply(ex, 404, error("not_found"));
return;
case "/api/v1/internal/servers/busy/wake":
reply(ex, 503, error("at_capacity"));
return;
case "/api/v1/internal/servers/slow/status":
try {
Thread.sleep(1500);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
reply(ex, 200, READY);
return;
default:
reply(ex, 500, error("unexpected"));
}
}
private static void getRetriesOnceAfter503(FelisApiClient api) throws LinkException {
ServerView v = api.serverStatus("flaky");
assertEq("flaky phase", "Running", v.phase());
assertEq("flaky requests", 2, count("GET /api/v1/internal/servers/flaky/status"));
}
private static void getRetriesOnceAfterACutReply(FelisApiClient api) throws LinkException {
ServerView v = api.serverStatus("cut");
assertEq("cut phase", "Running", v.phase());
assertEq("cut requests", 2, count("GET /api/v1/internal/servers/cut/status"));
}
private static void getGivesUpAfterTheSecondTry(FelisApiClient api) {
LinkException e = expectFailure("down", () -> api.serverStatus("down"));
assertEq("down status", 502, e.statusCode());
assertEq("down code", "bad_gateway", e.errorCode());
assertEq("down requests", 2, count("GET /api/v1/internal/servers/down/status"));
}
private static void getIsNotRetriedOnAClientError(FelisApiClient api) {
LinkException e = expectFailure("gone", () -> api.serverStatus("gone"));
assertEq("gone status", 404, e.statusCode());
assertEq("gone requests", 1, count("GET /api/v1/internal/servers/gone/status"));
}
private static void postIsNeverRetried(FelisApiClient api) {
UUID id = UUID.fromString("00000000-0000-0000-0000-00000000000a");
LinkException e = expectFailure("busy", () -> api.wake("busy", id));
assertEq("busy status", 503, e.statusCode());
assertEq("busy code", "at_capacity", e.errorCode());
assertEq("busy requests", 1, count("POST /api/v1/internal/servers/busy/wake"));
}
private static void timedOutGetIsCutOffAndNotRetried(FelisApiClient api) {
long start = System.nanoTime();
LinkException e = expectFailure("slow", () -> api.serverStatus("slow"));
long tookMillis = (System.nanoTime() - start) / 1_000_000;
assertEq("slow status", 0, e.statusCode());
assertEq("slow code", "transport_error", e.errorCode());
// 400 ms request timeout against a 1500 ms handler: well under the handler's
// time, which a missing or ignored request timeout would have to wait out.
if (tookMillis >= 1200) {
throw new AssertionError("slow: took " + tookMillis + " ms, want the 400 ms request timeout");
}
checks++;
assertEq("slow requests", 1, count("GET /api/v1/internal/servers/slow/status"));
}
// ---- harness ----
interface Call {
void run() throws LinkException;
}
private static LinkException expectFailure(String what, Call call) {
try {
call.run();
} catch (LinkException e) {
return e;
}
throw new AssertionError(what + ": call succeeded, want a LinkException");
}
private static int count(String key) {
AtomicInteger n = hits.get(key);
return n == null ? 0 : n.get();
}
private static String error(String code) {
return "{\"error\":{\"code\":\"" + code + "\",\"message\":\"stub\"}}";
}
private static void reply(HttpExchange ex, int status, String body) throws IOException {
byte[] b = body.getBytes(StandardCharsets.UTF_8);
ex.getResponseHeaders().set("Content-Type", "application/json");
ex.sendResponseHeaders(status, b.length);
try (OutputStream os = ex.getResponseBody()) {
os.write(b);
}
}
private static void assertEq(String what, Object want, Object got) {
if (!want.equals(got)) {
throw new AssertionError(what + ": got " + got + ", want " + want);
}
checks++;
}
}
@@ -0,0 +1,166 @@
package best.lolicon.felis.link;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.Stream;
/**
* LinkConfigLoaderTest checks how felis-link.properties and the environment combine:
* the environment wins, both required values must come from somewhere, a first run
* leaves a template and still refuses to start, and the call timeouts default to
* 10 s, take the operator's value, and refuse anything that is not a whole number
* of seconds from 1 to 120.
*
* <p>Run: {@code javac -d <out> shared/src/main/java/best/lolicon/felis/link/*.java
* shared/test/best/lolicon/felis/link/LinkConfigLoaderTest.java && java -cp <out>
* best.lolicon.felis.link.LinkConfigLoaderTest}.
*/
public final class LinkConfigLoaderTest {
private static int checks;
private static Path dir;
public static void main(String[] args) throws Exception {
dir = Files.createTempDirectory("felis-link-test");
try {
fileAloneIsEnough();
environmentWinsOverTheFile();
missingTokenIsRefused();
firstRunWritesATemplateAndRefuses();
timeoutsComeFromFileOrEnvironment();
badTimeoutsAreRefused();
} finally {
try (Stream<Path> walk = Files.walk(dir)) {
walk.sorted(Comparator.reverseOrder()).forEach(p -> p.toFile().delete());
}
}
System.out.println("LinkConfigLoaderTest OK (" + checks + " checks)");
}
private static void fileAloneIsEnough() throws IOException {
Path f = write("plain.properties",
"api-base-url=http://10.43.0.10:8081/\nservice-token=file-token\n");
LinkConfig c = LinkConfigLoader.load(f, env());
assertEq("file url (trailing slash stripped)", "http://10.43.0.10:8081", c.apiBaseUrl());
assertEq("file token", "file-token", c.serviceToken());
assertEq("default connect timeout", Duration.ofSeconds(10), c.connectTimeout());
assertEq("default request timeout", Duration.ofSeconds(10), c.requestTimeout());
}
private static void environmentWinsOverTheFile() throws IOException {
Path f = write("both.properties",
"api-base-url=http://file:8081\nservice-token=file-token\n");
LinkConfig c = LinkConfigLoader.load(f, env(
"FELIS_API_BASE_URL", "http://env:8081",
"FELIS_SERVICE_TOKEN", "env-token"));
assertEq("env url", "http://env:8081", c.apiBaseUrl());
assertEq("env token", "env-token", c.serviceToken());
// A blank variable is unset, not an override to empty.
LinkConfig blank = LinkConfigLoader.load(f, env("FELIS_SERVICE_TOKEN", " "));
assertEq("blank env token falls back", "file-token", blank.serviceToken());
}
private static void missingTokenIsRefused() throws IOException {
Path f = write("no-token.properties", "api-base-url=http://10.43.0.10:8081\n");
IOException e = expectRefused("no token", f, env());
assertContains("no token message", e.getMessage(), "service-token");
}
private static void firstRunWritesATemplateAndRefuses() throws IOException {
Path f = dir.resolve("fresh/felis-link.properties");
expectRefused("first run", f, env());
String template = new String(Files.readAllBytes(f), StandardCharsets.UTF_8);
assertContains("template url key", template, "\napi-base-url=\n");
assertContains("template token key", template, "\nservice-token=\n");
assertContains("template connect timeout", template, "\n#connect-timeout-seconds=10\n");
assertContains("template request timeout", template, "\n#request-timeout-seconds=10\n");
// The template's commented timeouts leave the defaults in force once filled in.
Files.write(f, (template.replace("\napi-base-url=\n", "\napi-base-url=http://x:8081\n")
.replace("\nservice-token=\n", "\nservice-token=t\n")).getBytes(StandardCharsets.UTF_8));
LinkConfig c = LinkConfigLoader.load(f, env());
assertEq("filled template connect timeout", Duration.ofSeconds(10), c.connectTimeout());
assertEq("filled template request timeout", Duration.ofSeconds(10), c.requestTimeout());
}
private static void timeoutsComeFromFileOrEnvironment() throws IOException {
Path f = write("timeouts.properties", "api-base-url=http://x:8081\nservice-token=t\n"
+ "connect-timeout-seconds=3\nrequest-timeout-seconds= 25 \n");
LinkConfig c = LinkConfigLoader.load(f, env());
assertEq("file connect timeout", Duration.ofSeconds(3), c.connectTimeout());
assertEq("file request timeout", Duration.ofSeconds(25), c.requestTimeout());
LinkConfig e = LinkConfigLoader.load(f, env(
"FELIS_API_CONNECT_TIMEOUT_SECONDS", "1",
"FELIS_API_REQUEST_TIMEOUT_SECONDS", "120"));
assertEq("env connect timeout", Duration.ofSeconds(1), e.connectTimeout());
assertEq("env request timeout", Duration.ofSeconds(120), e.requestTimeout());
}
private static void badTimeoutsAreRefused() throws IOException {
String base = "api-base-url=http://x:8081\nservice-token=t\n";
String[][] bad = {
{"connect-timeout-seconds", "0"},
{"connect-timeout-seconds", "-5"},
{"connect-timeout-seconds", "ten"},
{"request-timeout-seconds", "121"},
{"request-timeout-seconds", "2.5"},
{"request-timeout-seconds", "10s"},
};
for (String[] kv : bad) {
Path f = write("bad.properties", base + kv[0] + "=" + kv[1] + "\n");
IOException e = expectRefused(kv[0] + "=" + kv[1], f, env());
assertContains(kv[0] + "=" + kv[1] + " names the key", e.getMessage(), kv[0]);
assertContains(kv[0] + "=" + kv[1] + " quotes the value", e.getMessage(), "\"" + kv[1] + "\"");
}
Path f = write("env-bad.properties", base);
IOException e = expectRefused("env request timeout 0", f, env("FELIS_API_REQUEST_TIMEOUT_SECONDS", "0"));
assertContains("env bad names the variable", e.getMessage(), "FELIS_API_REQUEST_TIMEOUT_SECONDS");
}
// ---- harness ----
private static java.util.function.UnaryOperator<String> env(String... kv) {
Map<String, String> m = new HashMap<>();
for (int i = 0; i < kv.length; i += 2) {
m.put(kv[i], kv[i + 1]);
}
return m::get;
}
private static Path write(String name, String body) throws IOException {
Path f = dir.resolve(name);
Files.write(f, body.getBytes(StandardCharsets.UTF_8));
return f;
}
private static IOException expectRefused(String what, Path f, java.util.function.UnaryOperator<String> env) {
try {
LinkConfigLoader.load(f, env);
} catch (IOException e) {
checks++;
return e;
}
throw new AssertionError(what + ": loaded, want an IOException");
}
private static void assertContains(String what, String got, String want) {
if (got == null || !got.contains(want)) {
throw new AssertionError(what + ": " + got + " does not contain " + want);
}
checks++;
}
private static void assertEq(String what, Object want, Object got) {
if (!want.equals(got)) {
throw new AssertionError(what + ": got " + got + ", want " + want);
}
checks++;
}
}
+25 -1
View File
@@ -10,7 +10,11 @@
# velocity/test. They check what "compiles" cannot: the felis:control codec # velocity/test. They check what "compiles" cannot: the felis:control codec
# round-trips every frame kind (spec §12), no server name can re-aim a # round-trips every frame kind (spec §12), no server name can re-aim a
# felis-api request path, only the lobby and the login gate may drive # felis-api request path, only the lobby and the login gate may drive
# felis:control (and each only with its own frames), the link-status outage # felis:control (and each only with its own frames), a GET is retried once
# after a dropped reply or a 502/503/504 and a POST never, the request timeout
# cuts a slow call off, felis-link.properties timeouts are validated, the
# felis-api call pool refuses instead of growing and a repeating task never
# overlaps itself, the link-status outage
# fallback fails closed outside its window, a proxy restarted during an API # fallback fails closed outside its window, a proxy restarted during an API
# outage routes on the last saved server list (and only until a fetch # outage routes on the last saved server list (and only until a fetch
# succeeds), /invite prompts cannot double-fire # succeeds), /invite prompts cannot double-fire
@@ -71,6 +75,18 @@ javac -d "$work/shared-classes" \
plugins/shared/test/best/lolicon/felis/link/FelisApiClientTest.java plugins/shared/test/best/lolicon/felis/link/FelisApiClientTest.java
java -cp "$work/shared-classes" best.lolicon.felis.link.FelisApiClientTest java -cp "$work/shared-classes" best.lolicon.felis.link.FelisApiClientTest
echo "==> FelisApiClientRetryTest (which failures are retried, request timeout, shared)"
javac -d "$work/shared-classes" \
plugins/shared/src/main/java/best/lolicon/felis/link/*.java \
plugins/shared/test/best/lolicon/felis/link/FelisApiClientRetryTest.java
java -cp "$work/shared-classes" best.lolicon.felis.link.FelisApiClientRetryTest
echo "==> LinkConfigLoaderTest (env/file precedence, template, timeouts, shared)"
javac -d "$work/shared-classes" \
plugins/shared/src/main/java/best/lolicon/felis/link/*.java \
plugins/shared/test/best/lolicon/felis/link/LinkConfigLoaderTest.java
java -cp "$work/shared-classes" best.lolicon.felis.link.LinkConfigLoaderTest
echo "==> ControlPolicyTest (who may send what on felis:control, velocity)" echo "==> ControlPolicyTest (who may send what on felis:control, velocity)"
mkdir -p "$work/policy-classes" mkdir -p "$work/policy-classes"
javac -d "$work/policy-classes" \ javac -d "$work/policy-classes" \
@@ -93,6 +109,14 @@ javac -d "$work/list-classes" \
plugins/velocity/test/best/lolicon/felis/velocity/ServerListSourceTest.java plugins/velocity/test/best/lolicon/felis/velocity/ServerListSourceTest.java
java -cp "$work/list-classes" best.lolicon.felis.velocity.ServerListSourceTest java -cp "$work/list-classes" best.lolicon.felis.velocity.ServerListSourceTest
echo "==> ApiPoolTest (bounded felis-api pool, non-overlapping repeats, velocity)"
mkdir -p "$work/pool-classes"
javac -d "$work/pool-classes" \
plugins/velocity/src/main/java/best/lolicon/felis/velocity/BoundedExecutor.java \
plugins/velocity/src/main/java/best/lolicon/felis/velocity/SkipIfRunning.java \
plugins/velocity/test/best/lolicon/felis/velocity/ApiPoolTest.java
java -cp "$work/pool-classes" best.lolicon.felis.velocity.ApiPoolTest
echo "==> InviteBookTest (/invite prompt store, velocity)" echo "==> InviteBookTest (/invite prompt store, velocity)"
mkdir -p "$work/velocity-classes" mkdir -p "$work/velocity-classes"
javac -d "$work/velocity-classes" \ javac -d "$work/velocity-classes" \
@@ -0,0 +1,68 @@
package best.lolicon.felis.velocity;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
/**
* BoundedExecutor runs the plugin's felis-api calls on a fixed number of threads with
* a bounded wait queue. Velocity's scheduler starts a thread for every task it is
* handed, so while felis-api was slow a burst of joins, menu clicks and commands
* stacked blocking calls without limit, and each of them landed on the API the moment
* it recovered. Here at most {@code threads} calls run and at most {@code queue} wait;
* past that {@link #submit} refuses at once, so the caller can say "busy" instead of
* leaving the player waiting on a call that would run minutes later.
*
* <p>A task that throws is handed to {@code onError} and the worker carries on, so one
* bad task never shrinks the pool or disappears without a log line.
*/
final class BoundedExecutor {
private final ThreadPoolExecutor pool;
private final Consumer<Throwable> onError;
BoundedExecutor(String name, int threads, int queue, Consumer<Throwable> onError) {
this.onError = onError;
AtomicInteger n = new AtomicInteger();
this.pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<>(queue),
r -> {
Thread t = new Thread(r, name + "-" + n.incrementAndGet());
t.setDaemon(true);
return t;
},
new ThreadPoolExecutor.AbortPolicy());
}
/** submit queues the task, or returns false when every thread is busy and the queue is full, or after shutdown. */
boolean submit(Runnable task) {
try {
pool.execute(() -> {
try {
task.run();
} catch (Throwable t) {
onError.accept(t);
}
});
return true;
} catch (RejectedExecutionException e) {
return false;
}
}
/** running is how many tasks are executing now; waiting how many are queued. For the busy log line. */
int running() {
return pool.getActiveCount();
}
int waiting() {
return pool.getQueue().size();
}
/** shutdown stops taking tasks and interrupts the running ones (the proxy is going down). */
void shutdown() {
pool.shutdownNow();
}
}
@@ -237,7 +237,7 @@ public final class ControlChannel implements WaitingRouter.MenuTransferListener
send(source, cached.frame); send(source, cached.frame);
return; return;
} }
plugin.async(() -> { boolean taken = plugin.async(() -> {
try { try {
MenuStatus s = api.menuStatus(server); MenuStatus s = api.menuStatus(server);
ControlFrame frame = ControlFrame.statusUpdate( ControlFrame frame = ControlFrame.statusUpdate(
@@ -248,6 +248,9 @@ public final class ControlChannel implements WaitingRouter.MenuTransferListener
send(source, errorFrame(e, server)); send(source, errorFrame(e, server));
} }
}); });
if (!taken) {
send(source, busyFrame(server));
}
} }
// A WakeRequest is the menu's Join/Wake button on a server the player owns: wake // A WakeRequest is the menu's Join/Wake button on a server the player owns: wake
@@ -262,7 +265,7 @@ public final class ControlChannel implements WaitingRouter.MenuTransferListener
// autostartPolicy gate). A claim refusal answers with Error and never wakes. // autostartPolicy gate). A claim refusal answers with Error and never wakes.
private void handleClaim(ServerConnection source, Player player, String server) { private void handleClaim(ServerConnection source, Player player, String server) {
UUID id = player.getUniqueId(); UUID id = player.getUniqueId();
plugin.async(() -> { boolean taken = plugin.async(() -> {
try { try {
api.claim(server, id); api.claim(server, id);
} catch (LinkException e) { } catch (LinkException e) {
@@ -274,6 +277,9 @@ public final class ControlChannel implements WaitingRouter.MenuTransferListener
// hop for the wake, which is fine from here. // hop for the wake, which is fine from here.
router.enqueueFromMenu(player, server); router.enqueueFromMenu(player, server);
}); });
if (!taken) {
send(source, busyFrame(server));
}
} }
/** /**
@@ -305,6 +311,12 @@ public final class ControlChannel implements WaitingRouter.MenuTransferListener
return ControlFrame.error(code, message, server); return ControlFrame.error(code, message, server);
} }
// busyFrame answers a frame whose felis-api call the bounded pool refused.
private static ControlFrame busyFrame(String server) {
return ControlFrame.error("busy",
"Felis 现在很忙,请稍后再试 / Felis is busy right now — please try again in a moment.", server);
}
private static final class CachedStatus { private static final class CachedStatus {
final ControlFrame frame; final ControlFrame frame;
final long atMillis; final long atMillis;
@@ -18,6 +18,7 @@ import com.velocitypowered.api.command.CommandSource;
import com.velocitypowered.api.event.Subscribe; import com.velocitypowered.api.event.Subscribe;
import com.velocitypowered.api.event.connection.DisconnectEvent; import com.velocitypowered.api.event.connection.DisconnectEvent;
import com.velocitypowered.api.event.proxy.ProxyInitializeEvent; import com.velocitypowered.api.event.proxy.ProxyInitializeEvent;
import com.velocitypowered.api.event.proxy.ProxyShutdownEvent;
import com.velocitypowered.api.plugin.Plugin; import com.velocitypowered.api.plugin.Plugin;
import com.velocitypowered.api.plugin.annotation.DataDirectory; import com.velocitypowered.api.plugin.annotation.DataDirectory;
import com.velocitypowered.api.proxy.Player; import com.velocitypowered.api.proxy.Player;
@@ -34,6 +35,7 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Optional; import java.util.Optional;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern; import java.util.regex.Pattern;
/** /**
@@ -78,6 +80,13 @@ public final class FelisVelocityPlugin {
// friends around one after another is not. Sub-TTL on purpose: a sender may hold several // friends around one after another is not. Sub-TTL on purpose: a sender may hold several
// live invites, they just cannot post them all in one breath. // live invites, they just cannot post them all in one breath.
private static final Duration INVITE_COOLDOWN = Duration.ofSeconds(30); private static final Duration INVITE_COOLDOWN = Duration.ofSeconds(30);
// felis-api calls in flight, and waiting, at most. A healthy call takes milliseconds,
// so this is far above what one proxy's players produce; it only binds while
// felis-api is slow, where 8 x the 10 s request timeout already means the 64th
// waiter hears back after about 80 s. Refusing past that beats a reply minutes late.
private static final int API_THREADS = 8;
private static final int API_QUEUE = 64;
private static final long BUSY_LOG_INTERVAL_MILLIS = 60_000L;
private final ProxyServer proxy; private final ProxyServer proxy;
private final Logger logger; private final Logger logger;
@@ -88,6 +97,8 @@ public final class FelisVelocityPlugin {
// typing, and a bound on a client macro that would otherwise mint link codes or // typing, and a bound on a client macro that would otherwise mint link codes or
// fire claims as fast as it can send chat. // fire claims as fast as it can send chat.
private final FrameBudget commandBudget = new FrameBudget(5, 0.2, System::currentTimeMillis); private final FrameBudget commandBudget = new FrameBudget(5, 0.2, System::currentTimeMillis);
private final BoundedExecutor apiCalls;
private final AtomicLong lastBusyLog = new AtomicLong();
private FelisVelocityConfig config; private FelisVelocityConfig config;
private LinkClient linkClient; private LinkClient linkClient;
@@ -103,6 +114,8 @@ public final class FelisVelocityPlugin {
this.proxy = proxy; this.proxy = proxy;
this.logger = logger; this.logger = logger;
this.dataDirectory = dataDirectory; this.dataDirectory = dataDirectory;
this.apiCalls = new BoundedExecutor("felis-api", API_THREADS, API_QUEUE,
t -> logger.error("Felis: a felis-api task failed", t));
} }
@Subscribe @Subscribe
@@ -167,6 +180,11 @@ public final class FelisVelocityPlugin {
commandBudget.forget(event.getPlayer().getUniqueId()); commandBudget.forget(event.getPlayer().getUniqueId());
} }
@Subscribe
public void onProxyShutdown(ProxyShutdownEvent event) {
apiCalls.shutdown();
}
// withinBudget spends one command token for the player, or tells them to slow down. // withinBudget spends one command token for the player, or tells them to slow down.
private boolean withinBudget(Player player) { private boolean withinBudget(Player player) {
if (commandBudget.tryTake(player.getUniqueId())) { if (commandBudget.tryTake(player.getUniqueId())) {
@@ -178,13 +196,37 @@ public final class FelisVelocityPlugin {
return false; return false;
} }
/** async runs a task on Velocity's scheduler so felis-api I/O never blocks the proxy thread. */ /**
void async(Runnable task) { * async runs a felis-api task off the proxy thread, on the bounded pool. It returns
proxy.getScheduler().buildTask(this, task).schedule(); * false, without running the task, when API_THREADS calls are already in flight and
* API_QUEUE more are waiting; the caller then answers "busy" in its own terms.
*/
boolean async(Runnable task) {
if (apiCalls.submit(task)) {
return true;
}
long now = System.currentTimeMillis();
long last = lastBusyLog.get();
if (now - last >= BUSY_LOG_INTERVAL_MILLIS && lastBusyLog.compareAndSet(last, now)) {
logger.warn("Felis: felis-api is not keeping up ({} calls running, {} waiting); refusing new calls "
+ "until the queue drains.", apiCalls.running(), apiCalls.waiting());
}
return false;
} }
/** async for a task a player is waiting on: a refusal is told to them at once. */
void async(Player player, Runnable task) {
if (!async(task)) {
player.sendMessage(Component.text(
zh(player) ? "服务器现在很忙,请过一会儿再试。"
: "The network is busy right now — try again in a moment.",
NamedTextColor.YELLOW));
}
}
// A run that falls due while the previous one is still going is skipped (SkipIfRunning).
private void repeating(Duration interval, Runnable task) { private void repeating(Duration interval, Runnable task) {
proxy.getScheduler().buildTask(this, task).delay(interval).repeat(interval).schedule(); proxy.getScheduler().buildTask(this, new SkipIfRunning(task)).delay(interval).repeat(interval).schedule();
} }
/** /**
@@ -263,7 +305,7 @@ public final class FelisVelocityPlugin {
boolean zh = zh(player); boolean zh = zh(player);
player.sendMessage(Component.text( player.sendMessage(Component.text(
zh ? "正在获取绑定码……" : "Requesting a link code…", NamedTextColor.GRAY)); zh ? "正在获取绑定码……" : "Requesting a link code…", NamedTextColor.GRAY));
async(() -> { async(player, () -> {
try { try {
LinkCode code = linkClient.requestCode(player.getUniqueId()); LinkCode code = linkClient.requestCode(player.getUniqueId());
player.sendMessage(Component.text( player.sendMessage(Component.text(
@@ -583,7 +625,7 @@ public final class FelisVelocityPlugin {
UUID uuid = player.getUniqueId(); UUID uuid = player.getUniqueId();
player.sendMessage(Component.text( player.sendMessage(Component.text(
zh ? "正在认领「" + name + "」……" : "Claiming « " + name + " »…", NamedTextColor.GRAY)); zh ? "正在认领「" + name + "」……" : "Claiming « " + name + " »…", NamedTextColor.GRAY));
async(() -> { async(player, () -> {
try { try {
apiClient.claim(name, uuid); apiClient.claim(name, uuid);
player.sendMessage(Component.text( player.sendMessage(Component.text(
@@ -619,7 +661,7 @@ public final class FelisVelocityPlugin {
String who = player.getUsername(); String who = player.getUsername();
player.sendMessage(Component.text( player.sendMessage(Component.text(
zh ? "正在发起账户迁移……" : "Starting account migration…", NamedTextColor.GRAY)); zh ? "正在发起账户迁移……" : "Starting account migration…", NamedTextColor.GRAY));
async(() -> { async(player, () -> {
try { try {
apiClient.migrateStart(uuid); apiClient.migrateStart(uuid);
String panelHost = config.panelHostname(); String panelHost = config.panelHostname();
@@ -718,7 +760,7 @@ public final class FelisVelocityPlugin {
UUID approver = player.getUniqueId(); UUID approver = player.getUniqueId();
String who = player.getUsername(); String who = player.getUsername();
if (accountArg == null) { if (accountArg == null) {
async(() -> { async(player, () -> {
try { try {
OpLoginView req = apiClient.opLoginShow(code, approver); OpLoginView req = apiClient.opLoginShow(code, approver);
long age = req.createdAt() == null ? -1 long age = req.createdAt() == null ? -1
@@ -735,7 +777,7 @@ public final class FelisVelocityPlugin {
String account = accountArg.trim(); String account = accountArg.trim();
player.sendMessage(Component.text( player.sendMessage(Component.text(
zh ? "正在批准管理员登录……" : "Approving operator sign-in…", NamedTextColor.GRAY)); zh ? "正在批准管理员登录……" : "Approving operator sign-in…", NamedTextColor.GRAY));
async(() -> { async(player, () -> {
try { try {
OpLoginView done = apiClient.opLoginApprove(code, approver, account); OpLoginView done = apiClient.opLoginApprove(code, approver, account);
player.sendMessage(Component.text( player.sendMessage(Component.text(
@@ -0,0 +1,32 @@
package best.lolicon.felis.velocity;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* SkipIfRunning wraps a repeating task so that a run falling due while the previous
* one is still going is skipped rather than started beside it. Velocity's
* {@code repeat()} fires on schedule whatever the last run is doing, so a felis-api
* call slower than the interval (a 10 s request timeout against the 15 s registration
* refresh, or several calls in one pass) stacked runs that each held a thread and
* sent the same requests again, adding load exactly when the API was struggling.
*/
final class SkipIfRunning implements Runnable {
private final Runnable task;
private final AtomicBoolean running = new AtomicBoolean();
SkipIfRunning(Runnable task) {
this.task = task;
}
@Override
public void run() {
if (!running.compareAndSet(false, true)) {
return;
}
try {
task.run();
} finally {
running.set(false);
}
}
}
@@ -343,7 +343,7 @@ public final class WaitingRouter {
|| name.equalsIgnoreCase(lobbyServer)) { || name.equalsIgnoreCase(lobbyServer)) {
return; // system/static servers do not affect user-server activity return; // system/static servers do not affect user-server activity
} }
plugin.async(() -> { boolean taken = plugin.async(() -> {
try { try {
api.reportJoin(name, id); api.reportJoin(name, id);
} catch (LinkException e) { } catch (LinkException e) {
@@ -353,6 +353,9 @@ public final class WaitingRouter {
id, name, e.statusCode(), e.getMessage()); id, name, e.statusCode(), e.getMessage());
} }
}); });
if (!taken) {
log.warn("Felis: join-event for {} on {} dropped: the felis-api call queue is full", id, name);
}
} }
/** tick drains the waiting queue; the plugin schedules it on the async pool. */ /** tick drains the waiting queue; the plugin schedules it on the async pool. */
@@ -451,7 +454,7 @@ public final class WaitingRouter {
: "You're already on « " + serverName + " ».", NamedTextColor.YELLOW)); : "You're already on « " + serverName + " ».", NamedTextColor.YELLOW));
return; return;
} }
plugin.async(() -> { plugin.async(player, () -> {
try { try {
if (!linked(id)) { if (!linked(id)) {
player.sendMessage(Component.text( player.sendMessage(Component.text(
@@ -0,0 +1,164 @@
package best.lolicon.felis.velocity;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/**
* ApiPoolTest covers the two guards around the proxy's felis-api calls:
* {@link BoundedExecutor} (a fixed number of calls in flight, a bounded queue, refusal
* past it, a throwing task logged and survived) and {@link SkipIfRunning} (a repeating
* task never overlaps itself, and a run that throws does not block later runs).
* Framework free: a failed assertion throws and the process exits non-zero.
*
* <p>Run: {@code javac -d <out> velocity/src/main/java/best/lolicon/felis/velocity/{BoundedExecutor,SkipIfRunning}.java
* velocity/test/best/lolicon/felis/velocity/ApiPoolTest.java && java -cp <out>
* best.lolicon.felis.velocity.ApiPoolTest}.
*/
public final class ApiPoolTest {
private static int checks;
public static void main(String[] args) throws Exception {
fullPoolRefusesInsteadOfGrowing();
throwingTaskIsReportedAndThePoolLives();
shutdownRefuses();
repeatingTaskNeverOverlaps();
throwingRunDoesNotWedgeTheGuard();
System.out.println("ApiPoolTest OK (" + checks + " checks)");
}
// Two threads, two queue slots: four tasks are taken (two running, two waiting), the
// fifth is refused; once the blockers finish, every accepted task runs.
private static void fullPoolRefusesInsteadOfGrowing() throws Exception {
BoundedExecutor pool = new BoundedExecutor("t", 2, 2, t -> { });
CountDownLatch started = new CountDownLatch(2);
CountDownLatch release = new CountDownLatch(1);
AtomicInteger ran = new AtomicInteger();
Runnable blocker = () -> {
started.countDown();
await(release);
ran.incrementAndGet();
};
assertEq("first taken", true, pool.submit(blocker));
assertEq("second taken", true, pool.submit(blocker));
assertEq("both blockers running", true, started.await(5, TimeUnit.SECONDS));
assertEq("running while blocked", 2, pool.running());
assertEq("third queued", true, pool.submit(ran::incrementAndGet));
assertEq("fourth queued", true, pool.submit(ran::incrementAndGet));
assertEq("waiting while blocked", 2, pool.waiting());
assertEq("fifth refused", false, pool.submit(ran::incrementAndGet));
assertEq("nothing finished yet", 0, ran.get());
release.countDown();
CountDownLatch drained = new CountDownLatch(1);
waitFor(() -> ran.get() == 4, "accepted tasks all run");
assertEq("refused task never ran", 4, ran.get());
// Room again once drained.
assertEq("taken after drain", true, pool.submit(drained::countDown));
assertEq("post-drain task ran", true, drained.await(5, TimeUnit.SECONDS));
pool.shutdown();
}
private static void throwingTaskIsReportedAndThePoolLives() throws Exception {
List<Throwable> errors = new CopyOnWriteArrayList<>();
BoundedExecutor pool = new BoundedExecutor("t", 1, 1, errors::add);
pool.submit(() -> {
throw new IllegalStateException("boom");
});
CountDownLatch next = new CountDownLatch(1);
waitFor(() -> errors.size() == 1, "error reported");
assertEq("reported message", "boom", errors.get(0).getMessage());
assertEq("taken after a throw", true, pool.submit(next::countDown));
assertEq("task after a throw ran", true, next.await(5, TimeUnit.SECONDS));
pool.shutdown();
}
private static void shutdownRefuses() {
BoundedExecutor pool = new BoundedExecutor("t", 1, 1, t -> { });
pool.shutdown();
assertEq("refused after shutdown", false, pool.submit(() -> { }));
}
// A second run that falls due while the first is inside the task is skipped, not
// started beside it; the next run after the first finishes goes ahead.
private static void repeatingTaskNeverOverlaps() throws Exception {
CountDownLatch inside = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
AtomicInteger entered = new AtomicInteger();
AtomicInteger concurrent = new AtomicInteger();
AtomicInteger maxConcurrent = new AtomicInteger();
SkipIfRunning guarded = new SkipIfRunning(() -> {
int now = concurrent.incrementAndGet();
maxConcurrent.accumulateAndGet(now, Math::max);
entered.incrementAndGet();
if (entered.get() == 1) {
inside.countDown();
await(release);
}
concurrent.decrementAndGet();
});
Thread first = new Thread(guarded);
first.start();
assertEq("first run inside", true, inside.await(5, TimeUnit.SECONDS));
guarded.run(); // due while the first is still running
guarded.run();
assertEq("overlapping runs skipped", 1, entered.get());
release.countDown();
first.join(5000);
guarded.run();
assertEq("run after the first finished", 2, entered.get());
assertEq("never two at once", 1, maxConcurrent.get());
}
private static void throwingRunDoesNotWedgeTheGuard() {
AtomicInteger calls = new AtomicInteger();
SkipIfRunning guarded = new SkipIfRunning(() -> {
if (calls.incrementAndGet() == 1) {
throw new IllegalStateException("felis-api down");
}
});
try {
guarded.run();
throw new AssertionError("the first run's exception was swallowed");
} catch (IllegalStateException e) {
assertEq("first run's exception propagates", "felis-api down", e.getMessage());
}
guarded.run();
assertEq("runs after a throw", 2, calls.get());
}
// ---- harness ----
interface Cond {
boolean ok();
}
private static void waitFor(Cond cond, String what) throws InterruptedException {
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
while (!cond.ok()) {
if (System.nanoTime() > deadline) {
throw new AssertionError(what + ": timed out");
}
Thread.sleep(5);
}
checks++;
}
private static void await(CountDownLatch l) {
try {
l.await(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
private static void assertEq(String what, Object want, Object got) {
if (!want.equals(got)) {
throw new AssertionError(what + ": got " + got + ", want " + want);
}
checks++;
}
}