feat: manage distributed node operations from Owner panel
This commit is contained in:
33 files changed
+2417
-26
No files matched your search
@@ -1019,6 +1019,10 @@ async function handlePublic(ctx: RequestContext): Promise<boolean> {
|
||||
|
||||
async function handleSession(ctx: SessionContext): Promise<boolean> {
|
||||
switch (route(ctx)) {
|
||||
case "GET settings/node-control":
|
||||
if (!isOwner(ctx.account.role)) sendError(ctx.res, 403, "forbidden", "Owner account required");
|
||||
else sendJSON(ctx.res, 200, { available: false, tasks: [] });
|
||||
return true;
|
||||
case "GET settings/wake-policy":
|
||||
case "PUT settings/wake-policy": {
|
||||
if (!isOwner(ctx.account.role)) {
|
||||
|
||||
@@ -84,3 +84,32 @@ test("Owner emergency stop stays secondary, confirms name, and distinguishes acc
|
||||
const links = await ownerNav.evaluateAll((elements) => elements.map((el) => el.getAttribute("href")));
|
||||
expect(links.at(-1)).toBe("/admin/platform");
|
||||
});
|
||||
|
||||
test("Owner executes node management and receives stage, failure logs and retry", async ({ page, signIn }) => {
|
||||
await signIn("owner");
|
||||
let task: Record<string, unknown> = {};
|
||||
let submitted: Record<string, unknown> | null = null;
|
||||
await page.route("**/api/v1/settings/node-control", (route) => route.fulfill({ json: { available: true, tasks: task.id ? [task] : [] } }));
|
||||
await page.route("**/api/v1/settings/node-control/tasks", async (route) => {
|
||||
submitted = route.request().postDataJSON();
|
||||
task = { id: "test-task", request: submitted, actor: "owner", state: "running", stage: "database_backup", startedAt: new Date().toISOString(), log: "[felis] database_backup" };
|
||||
await route.fulfill({ status: 202, json: task });
|
||||
});
|
||||
await page.route("**/api/v1/settings/node-control/tasks/test-task", (route) => route.fulfill({ json: task }));
|
||||
await page.route("**/api/v1/settings/node-control/tasks/test-task/retry", (route) => { task = { ...task, state: "running", error: undefined }; return route.fulfill({ status: 202, json: task }); });
|
||||
await page.goto("/admin/platform");
|
||||
await page.getByLabel(t("admin:node_control_ip")).fill("192.0.2.10");
|
||||
const submit = page.getByRole("button", { name: t("admin:node_control_submit"), exact: true });
|
||||
await expect(submit).toBeDisabled();
|
||||
await page.getByRole("switch").click();
|
||||
await submit.click();
|
||||
expect(submitted).toMatchObject({ action: "enable", externalIP: "192.0.2.10", confirmMaintenance: true });
|
||||
await expect(page.getByText(t("admin:node_control_stage_database_backup"), { exact: true })).toBeVisible();
|
||||
task = { ...task, state: "failed", error: "Database backup failed", log: "[felis] pg_dump failed: connection refused" };
|
||||
await page.getByRole("button", { name: t("admin:node_control_reload"), exact: true }).click();
|
||||
await expect(page.getByText("Database backup failed", { exact: true })).toBeVisible();
|
||||
await expect(page.getByLabel(t("admin:node_control_log"))).toContainText("pg_dump failed: connection refused");
|
||||
await page.getByRole("button", { name: t("admin:node_control_retry"), exact: true }).click();
|
||||
await expect(page.getByText(t("admin:node_control_state_running"), { exact: true })).toBeVisible();
|
||||
await expectFitsScreen(page);
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
// @vitest-environment jsdom
|
||||
import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { NodeControlPanel } from "./NodeControlPanel";
|
||||
import type { NodeControlTask } from "@/lib/types";
|
||||
const calls = vi.hoisted(() => ({ nodeTasks: vi.fn(), nodeTask: vi.fn(), startNodeTask: vi.fn(), retryNodeTask: vi.fn() }));
|
||||
vi.mock("@/lib/api", async (original) => ({ ...await original<typeof import("@/lib/api")>(), api: calls }));
|
||||
const failed: NodeControlTask = { id: "task-1", request: { action: "approve", name: "worker-01", sshTarget: "worker-01", confirmMaintenance: true }, actor: "owner", state: "failed", stage: "approval", startedAt: "2026-10-06T10:00:00Z", error: "Network isolation probe failed", log: "probe: controller port rejected" };
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks(); calls.nodeTasks.mockResolvedValue({ available: true, tasks: [] }); calls.nodeTask.mockResolvedValue(failed);
|
||||
});
|
||||
describe("host node management", () => {
|
||||
it("keeps enrollment form visible and disabled while availability is unknown", async () => {
|
||||
let resolve!: (value: { available: boolean; tasks: [] }) => void;
|
||||
calls.nodeTasks.mockReturnValue(new Promise((r) => { resolve = r; }));
|
||||
render(<NodeControlPanel distributed />);
|
||||
expect(screen.getByLabelText("Node name")).toHaveProperty("disabled", true);
|
||||
expect(screen.getByRole("button", { name: "Execute operation" })).toHaveProperty("disabled", true);
|
||||
await act(async () => resolve({ available: true, tasks: [] }));
|
||||
expect(screen.getByLabelText("Node name")).toHaveProperty("disabled", false);
|
||||
});
|
||||
it("updates the default action after the deployment mode loads", async () => {
|
||||
const view = render(<NodeControlPanel distributed={null} />);
|
||||
await waitFor(() => expect(calls.nodeTasks).toHaveBeenCalled());
|
||||
expect(screen.queryByLabelText("Node name")).toBeNull();
|
||||
view.rerender(<NodeControlPanel distributed />);
|
||||
await waitFor(() => expect(screen.getByLabelText("Node name")).toHaveProperty("disabled", false));
|
||||
expect(screen.getByRole("combobox", { name: "Operation" }).textContent).toContain("Enroll and approve worker");
|
||||
});
|
||||
it("requires maintenance acknowledgement and submits structured enrollment without credentials", async () => {
|
||||
const running = { ...failed, state: "running", request: { ...failed.request, action: "join" } };
|
||||
calls.startNodeTask.mockResolvedValue(running); calls.nodeTask.mockResolvedValue(running);
|
||||
render(<NodeControlPanel distributed />);
|
||||
await waitFor(() => expect(screen.getByLabelText("Node name")).toHaveProperty("disabled", false));
|
||||
fireEvent.change(screen.getByLabelText("Node name"), { target: { value: "worker-01" } });
|
||||
fireEvent.change(screen.getByLabelText("SSH target"), { target: { value: "[email protected]" } });
|
||||
fireEvent.change(screen.getByLabelText("Fixed node IP"), { target: { value: "192.0.2.10" } });
|
||||
expect(screen.getByRole("button", { name: "Execute operation" })).toHaveProperty("disabled", true);
|
||||
await userEvent.click(screen.getByRole("switch"));
|
||||
await userEvent.click(screen.getByRole("button", { name: "Execute operation" }));
|
||||
expect(calls.startNodeTask).toHaveBeenCalledWith({ action: "join", name: "worker-01", sshTarget: "[email protected]", externalIP: "192.0.2.10", peers: [], confirmMaintenance: true });
|
||||
await screen.findByRole("region", { name: "Task progress" });
|
||||
expect(screen.getByRole("button", { name: "Execute operation" })).toHaveProperty("disabled", true);
|
||||
});
|
||||
it("restores failed tasks with concrete errors and logs and retries as a new attempt", async () => {
|
||||
calls.nodeTasks.mockResolvedValue({ available: true, tasks: [failed] });
|
||||
calls.retryNodeTask.mockResolvedValue({ ...failed, id: "task-2", state: "running" });
|
||||
render(<NodeControlPanel distributed />);
|
||||
expect(await screen.findByText("Network isolation probe failed")).toBeTruthy();
|
||||
expect(screen.getByLabelText("Execution log").textContent).toContain("probe: controller port rejected");
|
||||
await userEvent.click(screen.getByRole("button", { name: "Retry operation" }));
|
||||
expect(calls.retryNodeTask).toHaveBeenCalledWith("task-1");
|
||||
await waitFor(() => expect(calls.nodeTask).toHaveBeenCalledWith("task-2"));
|
||||
});
|
||||
it("reports a host connection failure and disables changes", async () => {
|
||||
calls.nodeTasks.mockRejectedValue({ status: 503, code: "node_control_unavailable" });
|
||||
render(<NodeControlPanel distributed={false} />);
|
||||
expect(await screen.findByRole("alert")).toBeTruthy();
|
||||
expect(screen.getByRole("button", { name: "Execute operation" })).toHaveProperty("disabled", true);
|
||||
expect(calls.startNodeTask).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { Loader2, Play, RefreshCw } from "lucide-react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { MessageLine } from "@/components/MessageLine";
|
||||
import { isReauthCancelled, useReauth } from "@/components/ReauthDialog";
|
||||
import { api, humanizeError } from "@/lib/api";
|
||||
import { useAsync, usePolling } from "@/lib/hooks";
|
||||
import type { NodeControlRequest, NodeControlTask } from "@/lib/types";
|
||||
|
||||
export function NodeControlPanel({ distributed }: { distributed: boolean | null }) {
|
||||
const { t } = useTranslation("admin");
|
||||
const reauth = useReauth();
|
||||
const tasks = useAsync(api.nodeTasks, [], { coalesce: true });
|
||||
const [id, setId] = useState<string | null>(null);
|
||||
const [action, setAction] = useState<NodeControlRequest["action"]>(distributed ? "join" : "enable");
|
||||
const [name, setName] = useState("");
|
||||
const [ssh, setSSH] = useState("");
|
||||
const [ip, setIP] = useState("");
|
||||
const [peers, setPeers] = useState("");
|
||||
const [confirmed, setConfirmed] = useState(false);
|
||||
const [submitting, setSubmitting] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const progress = useAsync(() => id ? api.nodeTask(id) : Promise.resolve(null), [id], { coalesce: true });
|
||||
const task = progress.data;
|
||||
const running = tasks.data?.tasks.some((entry) => entry.state === "running") || task?.state === "running";
|
||||
const busy = submitting || !!running;
|
||||
const available = tasks.data?.available === true && !tasks.error && distributed !== null;
|
||||
useEffect(() => { if (distributed !== null) { setAction(distributed ? "join" : "enable"); setConfirmed(false); } }, [distributed]);
|
||||
useEffect(() => { if (!id && tasks.data?.tasks[0]) setId(tasks.data.tasks[0].id); }, [id, tasks.data]);
|
||||
usePolling(() => { tasks.reload(); if (id) progress.reload(); }, running || progress.error || tasks.error ? 3000 : 10000);
|
||||
async function run(retry = false) {
|
||||
if (!available || busy || (!retry && !confirmed)) return;
|
||||
setSubmitting(true); setError(null);
|
||||
try {
|
||||
const result: NodeControlTask = await reauth.guard(() => retry && id ? api.retryNodeTask(id) : api.startNodeTask({
|
||||
action, ...(action !== "enable" ? { name: name.trim(), sshTarget: ssh.trim() } : {}),
|
||||
...(action !== "approve" ? { externalIP: ip.trim(), peers: peers.split(/[\s,]+/).filter(Boolean) } : {}),
|
||||
confirmMaintenance: confirmed,
|
||||
}));
|
||||
setId(result.id); setConfirmed(false); tasks.reload();
|
||||
} catch (e) { if (!isReauthCancelled(e)) setError(humanizeError(e)); }
|
||||
finally { setSubmitting(false); }
|
||||
}
|
||||
const valid = confirmed && (action === "enable" || !!name.trim() && !!ssh.trim()) && (action === "approve" || !!ip.trim());
|
||||
return <Card>
|
||||
<CardHeader className="flex-row items-center justify-between gap-3"><CardTitle>{t("node_control_title")}</CardTitle><Button variant="outline" size="sm" disabled={tasks.loading} onClick={() => { tasks.reload(); progress.reload(); }}><RefreshCw />{t("node_control_reload")}</Button></CardHeader>
|
||||
<CardContent className="space-y-5">
|
||||
<p className="text-sm leading-relaxed text-muted-foreground">{t("node_control_description")}</p>
|
||||
{tasks.loading && !tasks.data && <p role="status" className="flex items-center gap-2 text-sm text-muted-foreground"><Loader2 className="h-4 w-4 animate-spin" />{t("node_control_loading")}</p>}
|
||||
{tasks.error != null && <MessageLine kind="error" message={humanizeError(tasks.error)} />}
|
||||
{tasks.data?.available === false && <p role="status" className="text-sm text-muted-foreground">{t("node_control_unavailable")}</p>}
|
||||
{running && task?.state !== "running" && <div className="flex flex-wrap items-center gap-3"><p role="status" className="text-sm text-muted-foreground">{t("node_control_other_running")}</p><Button variant="outline" size="sm" onClick={() => { const active = tasks.data?.tasks.find((entry) => entry.state === "running"); if (active) setId(active.id); }}>{t("node_control_show_running")}</Button></div>}
|
||||
<div className="grid gap-5 md:grid-cols-2">
|
||||
<div className="space-y-2"><Label htmlFor="node-action">{t("node_control_action")}</Label><Select value={action} disabled={!available || busy} onValueChange={(value) => { setAction(value as NodeControlRequest["action"]); setConfirmed(false); }}><SelectTrigger id="node-action"><SelectValue /></SelectTrigger><SelectContent>{!distributed && <SelectItem value="enable">{t("node_control_enable")}</SelectItem>}<SelectItem value="join" disabled={!distributed}>{t("node_control_join")}</SelectItem><SelectItem value="approve" disabled={!distributed}>{t("node_control_approve")}</SelectItem></SelectContent></Select></div>
|
||||
{action !== "enable" && <>
|
||||
<div className="space-y-2"><Label htmlFor="node-name">{t("node_control_name")}</Label><Input id="node-name" value={name} disabled={!available || busy} onChange={(e) => setName(e.target.value)} placeholder="worker-01" /></div>
|
||||
<div className="space-y-2"><Label htmlFor="node-ssh">{t("node_control_ssh")}</Label><Input id="node-ssh" value={ssh} disabled={!available || busy} onChange={(e) => setSSH(e.target.value)} placeholder="[email protected]" /><p className="text-xs text-muted-foreground">{t("node_control_ssh_hint")}</p></div>
|
||||
</>}
|
||||
{action !== "approve" && <>
|
||||
<div className="space-y-2"><Label htmlFor="node-ip">{t("node_control_ip")}</Label><Input id="node-ip" value={ip} disabled={!available || busy} onChange={(e) => setIP(e.target.value)} placeholder="192.168.1.20" /></div>
|
||||
<div className="space-y-2 md:col-span-2"><Label htmlFor="node-peers">{t("node_control_peers")}</Label><Input id="node-peers" value={peers} disabled={!available || busy} onChange={(e) => setPeers(e.target.value)} placeholder="192.168.1.21/32, 192.168.1.22/32" /><p className="text-xs text-muted-foreground">{t("node_control_peers_hint")}</p></div>
|
||||
</>}
|
||||
</div>
|
||||
<div className="flex items-start gap-3 rounded-lg border border-border p-4"><Switch id="node-confirm" checked={confirmed} disabled={!available || busy} onCheckedChange={setConfirmed} /><Label htmlFor="node-confirm" className="text-sm leading-relaxed">{t(action === "enable" ? "node_control_confirm_enable" : "node_control_confirm_worker")}</Label></div>
|
||||
{error && <MessageLine kind="error" message={error} />}
|
||||
<Button disabled={!available || busy || !valid} onClick={() => void run()}>{submitting ? <Loader2 className="animate-spin" /> : <Play />}{t("node_control_submit")}</Button>
|
||||
<details className="rounded-lg border border-border p-4"><summary className="cursor-pointer text-sm font-medium">{t("node_control_prerequisites")}</summary><p className="mt-3 text-sm leading-relaxed text-muted-foreground">{t("node_control_prerequisites_body")}</p><Button asChild variant="outline" size="sm" className="mt-3"><a href="https://github.com/FelisMC/Felis/blob/main/docs/distributed.md" target="_blank" rel="noreferrer">{t("platform_distribution_runbook")}</a></Button></details>
|
||||
{!!tasks.data?.tasks.length && <div className="space-y-2"><Label htmlFor="node-task">{t("node_control_history")}</Label><Select value={id ?? undefined} onValueChange={setId}><SelectTrigger id="node-task"><SelectValue /></SelectTrigger><SelectContent>{tasks.data.tasks.map((entry) => <SelectItem key={entry.id} value={entry.id}>{t(`node_control_${entry.request.action}`)} · {entry.request.name || t("node_control_controller")} · {new Date(entry.startedAt).toLocaleString()} · {t(`node_control_state_${entry.state}`)}</SelectItem>)}</SelectContent></Select></div>}
|
||||
{progress.error != null && <MessageLine kind="error" message={t("node_control_connection_lost", { error: humanizeError(progress.error) })} />}
|
||||
{id && progress.loading && !task && <p role="status" className="text-sm text-muted-foreground">{t("node_control_loading")}</p>}
|
||||
{task && <section aria-label={t("node_control_progress")} className="space-y-3 rounded-lg border border-border p-4">
|
||||
<div className="flex flex-wrap items-center gap-3"><Badge variant={task.state === "failed" ? "destructive" : task.state === "succeeded" ? "default" : "muted"}>{t(`node_control_state_${task.state === "running" && (progress.error || tasks.error) ? "unknown" : task.state}`)}</Badge><span className="text-sm">{t(`node_control_stage_${task.stage}`, { defaultValue: task.stage })}</span></div>
|
||||
<p className="text-xs text-muted-foreground">{t("node_control_task_id")}: <code>{task.id}</code></p>
|
||||
<p className="text-xs text-muted-foreground">{t("node_control_started", { time: new Date(task.startedAt).toLocaleString() })} · {t("node_control_timeout")}</p>
|
||||
{task.error && <MessageLine kind="error" message={task.error} />}
|
||||
<pre aria-label={t("node_control_log")} className="max-h-80 overflow-auto rounded-md bg-muted p-3 font-mono text-xs leading-relaxed whitespace-pre-wrap break-all">{task.log || t("node_control_log_pending")}</pre>
|
||||
{task.state === "failed" && <><p className="text-sm text-muted-foreground">{t("node_control_retry_hint")}</p><Button variant="outline" disabled={!available || busy} onClick={() => void run(true)}><RefreshCw />{t("node_control_retry")}</Button></>}
|
||||
{task.state === "succeeded" && task.request.action === "enable" && <Button variant="outline" onClick={() => window.location.reload()}>{t("node_control_refresh_deployment")}</Button>}
|
||||
</section>}
|
||||
</CardContent>
|
||||
{reauth.dialog}
|
||||
</Card>;
|
||||
}
|
||||
@@ -335,5 +335,57 @@
|
||||
"platform_distribution_approve": "Approve the node from the controller. The command verifies identity, network isolation, image pulling, and cross-node connectivity. Failed checks preserve isolation. Approval removes the specified cached image and must run before the node hosts game workloads.",
|
||||
"platform_distribution_runbook": "Deployment and acceptance runbook",
|
||||
"platform_distribution_management": "Online approved workers can be selected when creating a server. Existing servers must be fully stopped before migration. Successful migration leaves the server stopped and retains its source world volume.",
|
||||
"platform_distribution_servers": "Manage server placement and migration"
|
||||
"platform_distribution_servers": "Manage server placement and migration",
|
||||
"node_control_title": "Node operations",
|
||||
"node_control_description": "Enable distributed deployment, enroll workers and run admission checks. Tasks execute on the controller host; panel disconnection does not terminate a task.",
|
||||
"node_control_loading": "Loading host task status.",
|
||||
"node_control_unavailable": "Host node service is not installed. Update the controller installation to enable panel management.",
|
||||
"node_control_action": "Operation",
|
||||
"node_control_enable": "Enable distributed deployment",
|
||||
"node_control_join": "Enroll and approve worker",
|
||||
"node_control_approve": "Approve existing worker",
|
||||
"node_control_name": "Node name",
|
||||
"node_control_ssh": "SSH target",
|
||||
"node_control_ssh_hint": "Use an alias from the controller root SSH configuration or user@host. Noninteractive authentication and verified host fingerprints are required.",
|
||||
"node_control_ip": "Fixed node IP",
|
||||
"node_control_peers": "Additional peer addresses (optional)",
|
||||
"node_control_peers_hint": "Enter exact /32 or /128 addresses separated by commas. Existing nodes and the new node are included automatically.",
|
||||
"node_control_confirm_enable": "All game servers (including Login and Lobby) and maintenance tasks are stopped. Database and cluster backups and controller network changes are authorized. Panel access may be temporarily interrupted.",
|
||||
"node_control_confirm_worker": "Game and maintenance workloads on the target node are stopped. Node installation, peer firewall updates, image pulls and network isolation checks are authorized.",
|
||||
"node_control_submit": "Execute operation",
|
||||
"node_control_prerequisites": "Environment and SSH requirements",
|
||||
"node_control_prerequisites_body": "Controller and workers require matching architecture and k3s versions and fixed addresses. SSH accounts require root or sudo -n access. Configure keys and verify host fingerprints on the controller host before first use; unknown fingerprints are rejected. Existing worker SSH aliases must match node names for peer firewall updates. Enrollment requires a host without k3s. Nodes already joined after an interrupted enrollment must use “Approve existing worker”. Failed admission retains quarantine and does not start game servers.",
|
||||
"node_control_history": "Task history",
|
||||
"node_control_controller": "Controller",
|
||||
"node_control_state_running": "Running",
|
||||
"node_control_state_succeeded": "Completed",
|
||||
"node_control_state_failed": "Failed",
|
||||
"node_control_connection_lost": "Task status is temporarily unavailable: {{error}}. Polling resumes when connectivity is restored; the last confirmed status remains visible.",
|
||||
"node_control_progress": "Task progress",
|
||||
"node_control_task_id": "Task ID",
|
||||
"node_control_log": "Execution log",
|
||||
"node_control_log_pending": "No execution output yet.",
|
||||
"node_control_retry_hint": "Verify current host state and resolve the failure before retrying. Retry creates a new task and repeats checks. Already enrolled workers require approval instead of enrollment.",
|
||||
"node_control_retry": "Retry operation",
|
||||
"node_control_refresh_deployment": "Refresh deployment status",
|
||||
"node_control_stage_preflight": "Checking prerequisites",
|
||||
"node_control_stage_pause_operator": "Pausing operator reconciliation",
|
||||
"node_control_stage_database_backup": "Backing up database",
|
||||
"node_control_stage_cluster_backup": "Backing up cluster state",
|
||||
"node_control_stage_configure_controller": "Configuring controller identity",
|
||||
"node_control_stage_install_controller": "Deploying controller components and network",
|
||||
"node_control_stage_ssh_check": "Checking SSH and host environment",
|
||||
"node_control_stage_firewall": "Updating peer firewalls",
|
||||
"node_control_stage_bootstrap_token": "Creating temporary enrollment token",
|
||||
"node_control_stage_transfer": "Transferring installation files",
|
||||
"node_control_stage_install_worker": "Installing worker components",
|
||||
"node_control_stage_wait_ready": "Waiting for node readiness",
|
||||
"node_control_stage_approval": "Running node admission checks",
|
||||
"node_control_stage_complete": "Operation completed",
|
||||
"node_control_reload": "Refresh node tasks",
|
||||
"node_control_state_unknown": "State unconfirmed",
|
||||
"node_control_started": "Started: {{time}}",
|
||||
"node_control_timeout": "Operation timeout: 45 minutes",
|
||||
"node_control_other_running": "Another node task is running; new node operations are temporarily blocked.",
|
||||
"node_control_show_running": "View running task"
|
||||
}
|
||||
@@ -145,5 +145,7 @@
|
||||
"auth_sources_unavailable": "Authentication source management is unavailable.",
|
||||
"auth_sources_changed": "Authentication sources changed. Discard your edits and reload before saving.",
|
||||
"auth_source_tag_locked": "Saved source IDs cannot be renamed or removed. Disable the source instead.",
|
||||
"distributed_unavailable": "Distributed deployment is not configured. Configure the controller and worker nodes using the deployment runbook first."
|
||||
"distributed_unavailable": "Distributed deployment is not configured. Configure the controller and worker nodes using the deployment runbook first.",
|
||||
"node_operation_busy": "Node maintenance is running. Server starts and additional node tasks are temporarily blocked.",
|
||||
"node_control_unavailable": "Host node service is unavailable. Inspect felis-node-control.service on the controller."
|
||||
}
|
||||
@@ -331,5 +331,57 @@
|
||||
"platform_distribution_approve": "在主控批准节点。该命令验证节点身份、网络隔离、镜像拉取和跨节点连通性;任何检查失败均保留隔离状态。批准检查会删除指定缓存镜像,须在该节点尚无游戏任务时执行。",
|
||||
"platform_distribution_runbook": "查看完整部署与验收流程",
|
||||
"platform_distribution_management": "在线且已批准的 worker 可在创建服务器时选择。已有服务器须完全停止后才能迁移;迁移完成后仍保持停止,源世界卷保留。",
|
||||
"platform_distribution_servers": "管理服务器节点与迁移"
|
||||
"platform_distribution_servers": "管理服务器节点与迁移",
|
||||
"node_control_title": "节点操作",
|
||||
"node_control_description": "启用分布式部署、接入 worker 和执行节点批准。任务在主控宿主机执行;面板连接中断不代表任务终止。",
|
||||
"node_control_loading": "正在读取主机任务状态。",
|
||||
"node_control_unavailable": "主机节点执行服务尚未安装。更新主控安装后可启用面板管理。",
|
||||
"node_control_action": "操作类型",
|
||||
"node_control_enable": "启用分布式部署",
|
||||
"node_control_join": "接入并批准 worker",
|
||||
"node_control_approve": "批准已有 worker",
|
||||
"node_control_name": "节点名称",
|
||||
"node_control_ssh": "SSH 目标",
|
||||
"node_control_ssh_hint": "使用主控 root SSH 配置中的别名或 user@host。要求免交互认证及已核验的主机指纹。",
|
||||
"node_control_ip": "节点固定 IP",
|
||||
"node_control_peers": "额外对等节点地址(选填)",
|
||||
"node_control_peers_hint": "填写精确的 /32 或 /128 地址,以逗号分隔。系统自动包含现有节点及本次接入节点的地址。",
|
||||
"node_control_confirm_enable": "已停止全部游戏服务器(含登录空间与正式大厅)及维护任务,确认允许备份数据库与集群状态,并变更主控网络配置。期间面板可能暂时不可访问。",
|
||||
"node_control_confirm_worker": "已停止目标节点上的游戏与维护任务。确认允许安装节点组件、更新对等防火墙并执行镜像拉取及网络隔离验收。",
|
||||
"node_control_submit": "执行操作",
|
||||
"node_control_prerequisites": "环境与 SSH 要求",
|
||||
"node_control_prerequisites_body": "主控与 worker 需使用相同架构及 k3s 版本,节点使用固定地址。SSH 账户须为 root 或具备 sudo -n 权限。首次使用前,应在主控宿主机配置密钥并核验各主机指纹;系统不会自动接受未知指纹。已有 worker 的 SSH 别名应与节点名称一致,以便更新对等防火墙。接入仅用于尚未安装 k3s 的主机;中断后已加入集群的节点应使用“批准已有 worker”。批准失败时保留隔离状态,不自动启动游戏服务器。",
|
||||
"node_control_history": "任务记录",
|
||||
"node_control_controller": "主控",
|
||||
"node_control_state_running": "执行中",
|
||||
"node_control_state_succeeded": "已完成",
|
||||
"node_control_state_failed": "执行失败",
|
||||
"node_control_connection_lost": "任务状态暂时无法读取:{{error}}。连接恢复后自动继续查询;当前展示最后一次已确认的状态。",
|
||||
"node_control_progress": "任务进度",
|
||||
"node_control_task_id": "任务 ID",
|
||||
"node_control_log": "执行日志",
|
||||
"node_control_log_pending": "尚未产生执行日志。",
|
||||
"node_control_retry_hint": "重试前需核对主机当前状态并修复失败原因。重试创建新任务并重新执行检查;已接入的 worker 应改用批准操作。",
|
||||
"node_control_retry": "重试操作",
|
||||
"node_control_refresh_deployment": "刷新部署状态",
|
||||
"node_control_stage_preflight": "检查环境",
|
||||
"node_control_stage_pause_operator": "暂停控制器调谐",
|
||||
"node_control_stage_database_backup": "备份数据库",
|
||||
"node_control_stage_cluster_backup": "备份集群状态",
|
||||
"node_control_stage_configure_controller": "配置主控身份",
|
||||
"node_control_stage_install_controller": "部署主控组件与网络",
|
||||
"node_control_stage_ssh_check": "检查 SSH 与主机环境",
|
||||
"node_control_stage_firewall": "更新对等防火墙",
|
||||
"node_control_stage_bootstrap_token": "创建限时接入令牌",
|
||||
"node_control_stage_transfer": "传输节点安装文件",
|
||||
"node_control_stage_install_worker": "安装 worker 组件",
|
||||
"node_control_stage_wait_ready": "等待节点就绪",
|
||||
"node_control_stage_approval": "执行节点验收",
|
||||
"node_control_stage_complete": "操作完成",
|
||||
"node_control_reload": "刷新节点任务",
|
||||
"node_control_state_unknown": "状态未确认",
|
||||
"node_control_started": "开始时间:{{time}}",
|
||||
"node_control_timeout": "操作超时上限为 45 分钟",
|
||||
"node_control_other_running": "另一项节点任务正在执行,当前暂不允许新的节点操作。",
|
||||
"node_control_show_running": "查看执行中任务"
|
||||
}
|
||||
@@ -145,5 +145,7 @@
|
||||
"auth_sources_unavailable": "认证源管理暂不可用。",
|
||||
"auth_sources_changed": "认证源已被其他操作修改。请放弃当前更改并重新读取后再保存。",
|
||||
"auth_source_tag_locked": "已保存的认证源标识不能改名或移除,请使用停用。",
|
||||
"distributed_unavailable": "分布式部署未配置。请先按部署流程配置主控与 worker 节点。"
|
||||
"distributed_unavailable": "分布式部署未配置。请先按部署流程配置主控与 worker 节点。",
|
||||
"node_operation_busy": "节点维护正在执行,暂不允许启动服务器或创建另一项节点任务。",
|
||||
"node_control_unavailable": "主机节点执行服务不可用。请检查主控上的 felis-node-control.service。"
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
import type {
|
||||
NodeControlRequest,
|
||||
NodeControlTask,
|
||||
AccessResult,
|
||||
ExecutionNode,
|
||||
WorldMigration,
|
||||
@@ -610,6 +612,10 @@ export const api = rejectingSync({
|
||||
URL.revokeObjectURL(url);
|
||||
},
|
||||
|
||||
nodeTasks: () => request<{ available: boolean; tasks: NodeControlTask[] }>("GET", "/settings/node-control"),
|
||||
nodeTask: (id: string) => request<NodeControlTask>("GET", `/settings/node-control/tasks/${encodeURIComponent(id)}`),
|
||||
startNodeTask: (body: NodeControlRequest) => request<NodeControlTask>("POST", "/settings/node-control/tasks", body),
|
||||
retryNodeTask: (id: string) => request<NodeControlTask>("POST", `/settings/node-control/tasks/${encodeURIComponent(id)}/retry`),
|
||||
nodes: () => request<{ nodes: ExecutionNode[] }>("GET", "/nodes").then((r) => r.nodes),
|
||||
migration: (name: string) => request<WorldMigration>("GET", `/servers/${encodeURIComponent(name)}/migrations`),
|
||||
migrateServer: (name: string, targetNode: string) =>
|
||||
@@ -1253,6 +1259,10 @@ export function humanizeError(e: unknown): string {
|
||||
switch (err.code) {
|
||||
// Session doors (spec §B): every passwordless door 403s this when local
|
||||
// sessions are disabled on a Zero-Trust-only deployment.
|
||||
case "node_operation_busy":
|
||||
return t("node_operation_busy");
|
||||
case "node_control_unavailable":
|
||||
return t("node_control_unavailable");
|
||||
case "distributed_unavailable":
|
||||
return t("distributed_unavailable");
|
||||
case "local_auth_disabled":
|
||||
|
||||
@@ -1263,6 +1263,77 @@ export interface paths {
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/v1/settings/node-control": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** Read host node-service availability and persistent task history (Owner). */
|
||||
get: operations["nodeTasks"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/v1/settings/node-control/tasks": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Execute fixed host node operation (Owner, fresh reauthentication).
|
||||
* @description Tasks are serialized, audited and persisted on the controller. The host verifies stopped workloads and SSH trust before enrollment or admission. Bootstrap credentials never cross the external API. Server starts fail closed during node operations and when the configured host service cannot report its state.
|
||||
*/
|
||||
post: operations["startNodeTask"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/v1/settings/node-control/tasks/{id}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/** Read task stage, terminal error and bounded execution log (Owner). */
|
||||
get: operations["nodeTask"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/v1/settings/node-control/tasks/{id}/retry": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/** Create a new attempt from a failed task (Owner, fresh reauthentication). */
|
||||
post: operations["retryNodeTask"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/api/v1/settings/wake-policy": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
@@ -3128,6 +3199,35 @@ export interface components {
|
||||
attempt: number;
|
||||
error?: string;
|
||||
};
|
||||
NodeControlRequest: {
|
||||
/** @enum {string} */
|
||||
action: "enable" | "join" | "approve";
|
||||
/** @description Stable worker node name; required for join and approve. */
|
||||
name?: string;
|
||||
/** @description Verified root SSH alias or user@host; required for workers. */
|
||||
sshTarget?: string;
|
||||
/** @description Fixed node IP; required for enable and join. */
|
||||
externalIP?: string;
|
||||
/** @description Additional exact /32 or /128 peer addresses. */
|
||||
peers?: string[];
|
||||
/** @constant */
|
||||
confirmMaintenance: true;
|
||||
};
|
||||
NodeControlTask: {
|
||||
id: string;
|
||||
request: components["schemas"]["NodeControlRequest"];
|
||||
actor: string;
|
||||
/** @enum {string} */
|
||||
state: "running" | "succeeded" | "failed";
|
||||
stage: string;
|
||||
/** Format: date-time */
|
||||
startedAt: string;
|
||||
/** Format: date-time */
|
||||
finishedAt?: string;
|
||||
error?: string;
|
||||
/** @description Most recent 64 KiB of host execution output; absent in task lists. */
|
||||
log?: string;
|
||||
};
|
||||
/** @description Status projection of one server (internal/api/cluster.go ServerInfo). */
|
||||
ServerInfo: {
|
||||
/** @description Private runtime and startup diagnostics for authorized server managers. */
|
||||
@@ -6825,6 +6925,167 @@ export interface operations {
|
||||
401: components["responses"]["Unauthorized"];
|
||||
};
|
||||
};
|
||||
nodeTasks: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Availability and most recent task records; no credentials or logs. */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": {
|
||||
available: boolean;
|
||||
tasks: components["schemas"]["NodeControlTask"][];
|
||||
};
|
||||
};
|
||||
};
|
||||
400: components["responses"]["BadRequest"];
|
||||
401: components["responses"]["Unauthorized"];
|
||||
403: components["responses"]["Forbidden"];
|
||||
/** @description Another node task is running or this task cannot be retried. */
|
||||
409: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
/** @description Host node execution service is unavailable. */
|
||||
503: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
};
|
||||
};
|
||||
startNodeTask: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["NodeControlRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Persistent task created. */
|
||||
202: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["NodeControlTask"];
|
||||
};
|
||||
};
|
||||
400: components["responses"]["BadRequest"];
|
||||
401: components["responses"]["Unauthorized"];
|
||||
403: components["responses"]["Forbidden"];
|
||||
/** @description Another node task is running or this task cannot be retried. */
|
||||
409: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
/** @description Host node execution service is unavailable. */
|
||||
503: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
};
|
||||
};
|
||||
nodeTask: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path: {
|
||||
id: string;
|
||||
};
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Current task and recent output. */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["NodeControlTask"];
|
||||
};
|
||||
};
|
||||
400: components["responses"]["BadRequest"];
|
||||
401: components["responses"]["Unauthorized"];
|
||||
403: components["responses"]["Forbidden"];
|
||||
404: components["responses"]["NotFound"];
|
||||
/** @description Another node task is running or this task cannot be retried. */
|
||||
409: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
/** @description Host node execution service is unavailable. */
|
||||
503: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
};
|
||||
};
|
||||
retryNodeTask: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path: {
|
||||
id: string;
|
||||
};
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description New task created; original task is retained. */
|
||||
202: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["NodeControlTask"];
|
||||
};
|
||||
};
|
||||
400: components["responses"]["BadRequest"];
|
||||
401: components["responses"]["Unauthorized"];
|
||||
403: components["responses"]["Forbidden"];
|
||||
404: components["responses"]["NotFound"];
|
||||
/** @description Another node task is running or this task cannot be retried. */
|
||||
409: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
/** @description Host node execution service is unavailable. */
|
||||
503: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content?: never;
|
||||
};
|
||||
};
|
||||
};
|
||||
getWakePolicy: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
||||
@@ -776,3 +776,23 @@ export interface WakePolicySettings {
|
||||
revision: string;
|
||||
managed: boolean;
|
||||
}
|
||||
|
||||
export interface NodeControlRequest {
|
||||
action: "enable" | "join" | "approve";
|
||||
name?: string;
|
||||
sshTarget?: string;
|
||||
externalIP?: string;
|
||||
peers?: string[];
|
||||
confirmMaintenance: boolean;
|
||||
}
|
||||
export interface NodeControlTask {
|
||||
id: string;
|
||||
request: NodeControlRequest;
|
||||
actor: string;
|
||||
state: "running" | "succeeded" | "failed";
|
||||
stage: string;
|
||||
startedAt: string;
|
||||
finishedAt?: string;
|
||||
error?: string;
|
||||
log?: string;
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import userEvent from "@testing-library/user-event";
|
||||
import { MemoryRouter } from "react-router-dom";
|
||||
import { PlatformSettingsPage } from "./PlatformSettingsPage";
|
||||
import type { WakePolicySettings } from "@/lib/types";
|
||||
const calls = vi.hoisted(() => ({ getWakePolicy: vi.fn(), setWakePolicy: vi.fn(), nodes: vi.fn() }));
|
||||
const calls = vi.hoisted(() => ({ getWakePolicy: vi.fn(), setWakePolicy: vi.fn(), nodes: vi.fn(), nodeTasks: vi.fn(), nodeTask: vi.fn(), startNodeTask: vi.fn(), retryNodeTask: vi.fn() }));
|
||||
vi.mock("@/lib/api", async (original) => ({ ...await original<typeof import("@/lib/api")>(), api: calls }));
|
||||
const runtime = vi.hoisted(() => ({ distributed: false, fallback: false, apiBase: "/api/v1", rootDomain: "example.test" }));
|
||||
vi.mock("@/lib/hooks", async (original) => ({ ...await original<typeof import("@/lib/hooks")>(), useConfig: () => runtime }));
|
||||
@@ -18,6 +18,7 @@ beforeEach(() => {
|
||||
runtime.distributed = false;
|
||||
runtime.fallback = false;
|
||||
calls.nodes.mockResolvedValue([]);
|
||||
calls.nodeTasks.mockResolvedValue({ available: false, tasks: [] });
|
||||
calls.getWakePolicy.mockResolvedValue(policy);
|
||||
calls.setWakePolicy.mockImplementation(async (body) => ({ ...body, revision: "saved", managed: true }));
|
||||
});
|
||||
@@ -28,7 +29,7 @@ describe("platform policy", () => {
|
||||
page();
|
||||
expect(limit().disabled).toBe(true);
|
||||
expect(limit().value).toBe("");
|
||||
expect(screen.getByRole("status").textContent).toContain("Loading the saved policy");
|
||||
expect(screen.getAllByRole("status").some((node) => node.textContent?.includes("Loading the saved policy"))).toBe(true);
|
||||
await act(async () => resolve(policy));
|
||||
expect(limit().disabled).toBe(false);
|
||||
expect(limit().value).toBe("0");
|
||||
@@ -57,9 +58,8 @@ describe("distributed deployment entry", () => {
|
||||
page();
|
||||
await waitFor(() => expect(limit().disabled).toBe(false));
|
||||
expect(screen.getByText("Single-node mode")).toBeTruthy();
|
||||
expect(screen.getByText(/cannot be switched live from the panel/)).toBeTruthy();
|
||||
expect(screen.getByText("Connect and approve worker nodes")).toBeTruthy();
|
||||
expect(screen.getByRole("link", { name: "Deployment and acceptance runbook" }).getAttribute("href")).toContain("docs/distributed.md");
|
||||
expect(screen.getByText("Node operations")).toBeTruthy();
|
||||
expect(await screen.findByText(/Host node service is not installed/)).toBeTruthy();
|
||||
expect(calls.nodes).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Link } from "react-router-dom";
|
||||
import { useEffect, useState } from "react";
|
||||
import { Loader2, RefreshCw, Save, Settings } from "lucide-react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { NodeControlPanel } from "@/components/NodeControlPanel";
|
||||
import { DistributedNodes } from "@/components/DistributedNodes";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { PageHeader } from "@/components/PageHeader";
|
||||
@@ -75,21 +76,11 @@ export function PlatformSettingsPage() {
|
||||
{t(!config || config.fallback ? "platform_distribution_unknown" : config.distributed ? "platform_distribution_enabled" : "platform_distribution_disabled")}
|
||||
</Badge>
|
||||
<p className="text-sm leading-relaxed text-muted-foreground">{t("platform_distribution_architecture")}</p>
|
||||
{config && !config.fallback && !config.distributed && <p className="text-sm leading-relaxed">{t("platform_distribution_enable_hint")}</p>}
|
||||
<details className="rounded-lg border border-border p-4">
|
||||
<summary className="cursor-pointer text-sm font-medium">{t("platform_distribution_join")}</summary>
|
||||
<ol className="mt-4 list-decimal space-y-3 pl-5 text-sm leading-relaxed text-muted-foreground">
|
||||
<li>{t("platform_distribution_prepare")}</li>
|
||||
<li>{t("platform_distribution_token")}<pre className="mt-2 overflow-x-auto rounded-md bg-muted p-3 text-xs">{"felis node token --name <worker-name> --ttl 10m --out /root/worker.bootstrap"}</pre></li>
|
||||
<li>{t("platform_distribution_worker")}</li>
|
||||
<li>{t("platform_distribution_approve")}<pre className="mt-2 overflow-x-auto rounded-md bg-muted p-3 text-xs">{"felis node approve --name <worker-name> --ssh-target <worker-ssh-alias> --image <felis-image>\nfelis node list"}</pre></li>
|
||||
</ol>
|
||||
<Button asChild variant="outline" size="sm" className="mt-4"><a href="https://github.com/FelisMC/Felis/blob/main/docs/distributed.md" target="_blank" rel="noreferrer">{t("platform_distribution_runbook")}</a></Button>
|
||||
</details>
|
||||
<p className="text-sm leading-relaxed text-muted-foreground">{t("platform_distribution_management")}</p>
|
||||
<Button asChild variant="outline" size="sm"><Link to="/servers">{t("platform_distribution_servers")}</Link></Button>
|
||||
</CardContent>
|
||||
</Card>
|
||||
<NodeControlPanel distributed={!config || config.fallback ? null : !!config.distributed} />
|
||||
{config?.distributed && !config.fallback && <DistributedNodes />}
|
||||
{reauth.dialog}
|
||||
</div>;
|
||||
|
||||
Reference in new issue
Block a user