Unverified Commit 34b81ee8 authored by Lemon-miaow's avatar Lemon-miaow
Browse files

feat(schedules): 每台服务器可设计划任务,按星期和时区定时发命令、重启、停服、开服或备份,重启和备份前可在游戏内提醒

parent e8ff8f2f
Loading
Loading
Loading
Loading
+20 −0
Changes for cmd/felis/api.go: 20 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -370,6 +370,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
		InternalBaseURL: internalAPIBaseURL(),
		Submissions:     submissions,
		Mailer:          mailer,
		Schedules:       repo,
		// The external face authenticates the local session cookie the sign-in doors
		// mint, live once `felis breakGlass` flips local_auth_enabled on. Cloudflare
		// Access, when the install sits behind it, is enforced at the edge only.
@@ -464,6 +465,7 @@ func cmdAPI(args []string, stdout, stderr io.Writer) int {
	// reconciles it, but this loop converges builds nobody is polling.
	go reconcileBuilds(ctx, builder, stderr)
	go settleRestoreChains(ctx, a, stderr)
	go runSchedules(ctx, a, stderr)
	// A daily restore point of every world played since its last one, taken
	// once the server stops ([archive] scheduled_every; 0s turns it off).
	if backuper != nil && rcfg.ScheduledEvery > 0 {
@@ -734,6 +736,24 @@ func settleRestoreChains(ctx context.Context, a *api.API, stderr io.Writer) {
	}
}

// runSchedules runs the servers' scheduled tasks (api.API.RunSchedules). The
// interval is how late a task may start, and how often a restart or backup in
// progress checks whether it can take its next step.
func runSchedules(ctx context.Context, a *api.API, stderr io.Writer) {
	t := time.NewTicker(15 * time.Second)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
			if err := a.RunSchedules(ctx); err != nil {
				fmt.Fprintf(stderr, "felis api: scheduled tasks: %v\n", err)
			}
		}
	}
}

// scheduleBackups starts the scheduled backups (api.BackupScheduler). Each tick
// starts at most one, so the interval also spaces the worlds that stopped at
// the same time: a world that stops waits at most this long for its point to
+259 −0
Changes for docs/openapi.yaml: 259 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -123,6 +123,8 @@ tags:
    description: Read (SSE) and write (RCON) server console (external face, app tier).
  - name: backups
    description: World backup listing and restore (external face, app tier).
  - name: schedules
    description: Scheduled tasks of your own servers (external face, app tier).
  - name: account
    description: Web side of account linking (external face, app tier).
  - name: admin-servers
@@ -639,6 +641,68 @@ components:
          allOf: [{ $ref: '#/components/schemas/RetireState' }]
          description: Owned rows only. Present while the server is given up or being deleted.

    Schedule:
      type: object
      description: >-
        One scheduled task of a server (internal/api/schedules.go Schedule). It runs
        at minute_of_day on the weekdays, or with every_minutes set at every multiple
        of it since midnight on those days, in timezone.
      required: [id, server, label, action, command, every_minutes, minute_of_day, weekdays, timezone, warn_minutes, enabled, next_run_at, run_state, last_run_at, last_result, last_detail, created_by, created_at]
      properties:
        id: { type: integer, format: int64 }
        server: { type: string }
        label: { type: string, description: Free text naming the task; may be empty. }
        action:
          type: string
          enum: [command, restart, stop, start, backup]
          description: >-
            backup of a running server stops it, takes a backup (pruned with the daily
            restore points, [archive] scheduled_keep) and starts it again; of a stopped
            server it leaves the server stopped.
        command: { type: string, description: The console command of a command task, without a slash; empty otherwise. }
        every_minutes:
          type: integer
          enum: [0, 15, 30, 60, 120, 180, 240, 360, 480, 720]
          description: 0 runs once a day at minute_of_day. A restart, stop, start or backup repeats at most every 60 minutes.
        minute_of_day: { type: integer, minimum: 0, maximum: 1439, description: Minutes after local midnight; 0 when every_minutes is set. }
        weekdays: { type: integer, minimum: 1, maximum: 127, description: 'Bitmask of the days it runs on: bit 0 Sunday to bit 6 Saturday.' }
        timezone: { type: string, description: IANA zone the times are in, e.g. Asia/Shanghai. }
        warn_minutes:
          type: integer
          enum: [0, 1, 5, 10, 15, 30]
          description: How long before a restart, stop or backup the players on the server are told (say); 0 for none, and always 0 for a command or start.
        enabled: { type: boolean }
        next_run_at: { type: string, format: date-time, nullable: true, description: Null while disabled. }
        run_state:
          type: string
          enum: ['', claimed, stopping, backing_up, starting]
          description: What a run in progress is doing; empty when none is.
        last_run_at: { type: string, format: date-time, nullable: true }
        last_result:
          type: string
          enum: ['', ok, skipped, failed, missed]
          description: >-
            How the last run ended; empty before the first and during a run. missed is a
            run felis-api was down for, dropped once it was 10 minutes late.
        last_detail: { type: string, description: What happened, in English (a command's reply, or why the run was skipped or failed). }
        created_by: { type: string }
        created_at: { type: string, format: date-time }

    ScheduleInput:
      type: object
      required: [action, weekdays, timezone]
      additionalProperties: false
      properties:
        label: { type: string, maxLength: 64 }
        action: { type: string, enum: [command, restart, stop, start, backup] }
        command: { type: string, maxLength: 1024, description: Required for a command task and refused for the others. One line; a leading slash is dropped. }
        every_minutes: { type: integer, enum: [0, 15, 30, 60, 120, 180, 240, 360, 480, 720] }
        minute_of_day: { type: integer, minimum: 0, maximum: 1439 }
        weekdays: { type: integer, minimum: 1, maximum: 127 }
        timezone: { type: string }
        warn_minutes: { type: integer, enum: [0, 1, 5, 10, 15, 30] }
        enabled: { type: boolean, description: Default true. }

    BackupView:
      type: object
      description: One world backup (internal/api/repo.go BackupView). backup_ref is withheld (spec §286).
@@ -4595,6 +4659,201 @@ paths:
            application/json:
              schema: { $ref: '#/components/schemas/Error' }

  # --------------------------------------------------- scheduled tasks (app) ---
  /api/v1/servers/{name}/schedules:
    get:
      tags: [schedules]
      operationId: listServerSchedules
      summary: List a server's scheduled tasks (owner-or-admin).
      x-felis-face: [external]
      x-felis-tier: app
      security: [{ sessionCookie: [] }]
      parameters:
        - { name: name, in: path, required: true, schema: { type: string } }
      responses:
        '200':
          description: The server's tasks, oldest first, and how many it may have.
          content:
            application/json:
              schema:
                type: object
                required: [server, schedules, limit]
                properties:
                  server: { type: string }
                  schedules:
                    type: array
                    items: { $ref: '#/components/schemas/Schedule' }
                  limit: { type: integer }
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
        '503':
          $ref: '#/components/responses/ServiceUnavailable'
    post:
      tags: [schedules]
      operationId: createServerSchedule
      summary: Add a scheduled task to a server (owner-or-admin).
      description: >-
        The task belongs to the server's current owner: once the server has another
        owner felis-api disables it instead of running it, until somebody saves it
        again. felis-api checks the tasks every 15 seconds; a run it was down for is
        started late, up to 10 minutes, and dropped as missed after that. A command
        runs only on a running server, a restart only restarts a running one, and a
        start goes through the running-server cap and a pending retirement like a
        wake. Audited as schedule.create; each run as schedule.run by scheduler.
      x-felis-face: [external]
      x-felis-tier: app
      security: [{ sessionCookie: [] }]
      parameters:
        - { name: name, in: path, required: true, schema: { type: string } }
      requestBody:
        required: true
        content:
          application/json:
            schema: { $ref: '#/components/schemas/ScheduleInput' }
      responses:
        '201':
          description: Created.
          content:
            application/json:
              schema: { $ref: '#/components/schemas/Schedule' }
        '400':
          description: A malformed body or name, or settings out of range (bad_schedule, bad_request for the command).
          content:
            application/json:
              schema: { $ref: '#/components/schemas/Error' }
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
        '409':
          description: The server already has 20 tasks (schedule_limit).
          content:
            application/json:
              schema: { $ref: '#/components/schemas/Error' }
        '503':
          $ref: '#/components/responses/ServiceUnavailable'
  /api/v1/servers/{name}/schedules/{id}:
    put:
      tags: [schedules]
      operationId: updateServerSchedule
      summary: Change a scheduled task (owner-or-admin).
      description: >-
        Replaces the task's settings and recomputes its next run. The task passes to
        the server's current owner. Refused while a run is in progress. Audited as
        schedule.update.
      x-felis-face: [external]
      x-felis-tier: app
      security: [{ sessionCookie: [] }]
      parameters:
        - { name: name, in: path, required: true, schema: { type: string } }
        - { name: id, in: path, required: true, schema: { type: integer, format: int64 } }
      requestBody:
        required: true
        content:
          application/json:
            schema: { $ref: '#/components/schemas/ScheduleInput' }
      responses:
        '200':
          description: Saved.
          content:
            application/json:
              schema: { $ref: '#/components/schemas/Schedule' }
        '400':
          description: A malformed body, name or id, or settings out of range (bad_schedule).
          content:
            application/json:
              schema: { $ref: '#/components/schemas/Error' }
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
        '409':
          description: A run is in progress (schedule_running).
          content:
            application/json:
              schema: { $ref: '#/components/schemas/Error' }
        '503':
          $ref: '#/components/responses/ServiceUnavailable'
    delete:
      tags: [schedules]
      operationId: deleteServerSchedule
      summary: Remove a scheduled task (owner-or-admin).
      description: Refused while a run is in progress. Audited as schedule.delete.
      x-felis-face: [external]
      x-felis-tier: app
      security: [{ sessionCookie: [] }]
      parameters:
        - { name: name, in: path, required: true, schema: { type: string } }
        - { name: id, in: path, required: true, schema: { type: integer, format: int64 } }
      responses:
        '204':
          description: Removed.
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
        '409':
          description: A run is in progress (schedule_running).
          content:
            application/json:
              schema: { $ref: '#/components/schemas/Error' }
        '503':
          $ref: '#/components/responses/ServiceUnavailable'
  /api/v1/servers/{name}/schedules/{id}/run:
    post:
      tags: [schedules]
      operationId: runServerSchedule
      summary: Run a scheduled task now (owner-or-admin).
      description: >-
        Starts a run at once, without the players' warning, whether the task is
        enabled or not; its next scheduled run stays where it was. The answer is the
        task after the run's first step: a command, stop or start has finished, and a
        restart or backup goes on in the background (run_state). Audited as
        schedule.run_now, and the run itself as schedule.run.
      x-felis-face: [external]
      x-felis-tier: app
      security: [{ sessionCookie: [] }]
      parameters:
        - { name: name, in: path, required: true, schema: { type: string } }
        - { name: id, in: path, required: true, schema: { type: integer, format: int64 } }
      responses:
        '202':
          description: Started.
          content:
            application/json:
              schema: { $ref: '#/components/schemas/Schedule' }
        '400':
          $ref: '#/components/responses/BadRequest'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
        '409':
          description: >-
            A run is already in progress (schedule_running), or the server has another
            owner since the task was saved (schedule_stale).
          content:
            application/json:
              schema: { $ref: '#/components/schemas/Error' }
        '503':
          $ref: '#/components/responses/ServiceUnavailable'

  # ------------------------------------------------------ users (admin tier) ----
  /api/v1/users:
    get:
+60 −0
Changes for docs/troubleshooting.md: 60 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -3163,6 +3163,65 @@ end, has not been run on a cluster.]

---

## 19. Scheduled tasks: a run is skipped, missed or failed

A server's owner (or an admin) keeps up to 20 scheduled tasks on it, on the
panel's **Scheduled tasks** page (`/api/v1/servers/{name}/schedules`). Each one
sends a console command, restarts, stops, starts or backs up the server, once a
day at a set time or every 15 minutes to 12 hours, on the chosen weekdays, in an
IANA time zone (the browser's by default). A restart, stop or backup may warn the
players in game (`say`) 1 to 30 minutes ahead. felis-api runs the tasks itself: a
loop every 15 seconds, so a task starts within about 15 seconds of its time, and
the database hands each run to one felis-api only.

What each action does:

| Action | Server running | Server stopped |
| --- | --- | --- |
| command | runs it over RCON (a leading `/` is dropped) | skipped |
| restart | stops the pod, then starts it | skipped |
| stop | stops it | skipped |
| start | skipped (a start that `Failed` is started over) | starts it, as the panel's start does (running-server cap, retiring server, busy world) |
| backup | stops it, takes the same backup Job as "Back up now", then starts it again | takes the backup |

A step has a deadline: the pod must stop within 15 minutes, the backup Job must
end within 45, and the start is given 15. A restart or backup that cannot finish
still starts the server again when it was running before, so a failed backup never
leaves a world offline.

The row's last result says how the latest run went:

| `last_result` | `last_detail` | What to do |
| --- | --- | --- |
| `ok` | empty, or the reply of a console command | nothing |
| `skipped` | `the server has a new owner since this schedule was saved; save it again to use it` | the server changed owner (claim, account migration, admin). The task switched itself off; the new owner reviews it and saves it to turn it back on |
| `skipped` | `the server was not running`, `the server was already stopped`, `the server was already running`, `the server has no world yet`, `the server is being given up or deleted`, `the server no longer exists`, and for a start `the cluster is at its running-server cap` or `the world is busy with …` | the run had nothing to act on; the next run tries again |
| `missed` | `felis-api was not running at the scheduled time` | felis-api was down more than 10 minutes past the time. The run is dropped, so a restart never lands hours late; the next one runs as usual |
| `failed` | `felis-api stopped in the middle of this run` | felis-api restarted during the run's first step (the claim was over 2 minutes old); check that the server is in the state you want |
| `failed` | `the server did not stop within 15 minutes` | see §1 and §2 for a pod that hangs; the backup did not run |
| `failed` | `another operation kept the world busy for 15 minutes` | a restore, file change or another backup held the world (§3b) |
| `failed` | `the backup failed: …`, `the backup did not finish within 45 minutes`, `the backup store is full; ask an administrator to free space` | §10 for backup Jobs; the Backups page shows the Job |
| `failed` | `could not start the server again: …` (after a restart or a backup) | the server stopped and could not be started again: the running-server cap, a server being given up, or a world still busy after 15 minutes of retries. Start it from the panel once the cause is gone (§1, §3b) |
| `failed` | `the server console could not be reached`, `the command failed: …` | the server's RCON (§1c) |

"Run now" starts a run at once, including on a switched-off task, and leaves the
next planned run where it was. While a run is in progress (`run_state` is
`claimed`, `stopping`, `backing_up` or `starting`) the task cannot be edited,
deleted or run again (`409 schedule_running`).

A time the clock skips at a daylight-saving change runs an hour early (the offset
before the jump applies); a time it repeats runs once, the first time. A time zone
the host no longer knows runs in UTC. Deleting a server drops its tasks, and a new
server of the same name starts with none. The audit actions are `schedule.create`,
`schedule.update`, `schedule.delete`, `schedule.run_now` (by a person) and
`schedule.run` (every finished run, with its result and detail).

[GO-TESTED: `TestScheduleNextRun`, `TestScheduleInputValidation`,
`TestScheduleRunnerBackup`, `TestScheduleRunnerRestart`, `TestScheduleRunnerStaleClaim`;
PG-TESTED: `TestScheduleStoreRunCAS`, `TestDueSchedules`, `TestSchedulesFollowTheServer`]

---

## Quick reference: symptom → section

| Symptom | Section |
@@ -3208,3 +3267,4 @@ end, has not been run on a cluster.]
| `felis breakGlass` sends no code / shows `Root override`; `otp_skipped` in the audit | §17 |
| How long sessions, codes and audit rows are kept; export audit rows | §17 |
| Files page: a change or upload refused (`file_exists`, `bad_path`, `too_large`, `upload_staging_full`, `volume_full`, `files_timeout`) | §18 |
| A scheduled task shows `skipped`, `missed` or `failed`; a task switched itself off after an owner change | §19 |
+14 −0
Changes for internal/api/api.go: 14 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -98,6 +98,11 @@ type API struct {
	FileStage       *fileedit.Stage
	InternalBaseURL string

	// Schedules stores the servers' scheduled tasks (schedules.go), which
	// RunSchedules fires. Optional: when nil the schedule routes report 503 and
	// RunSchedules does nothing.
	Schedules ServerSchedules

	// Submissions is the user-modpack approval lane (a user-directed extension over
	// the §16 build subsystem; see internal/submit). It is optional: when
	// nil the /me/submissions and /submissions routes report 503 rather than 404, so
@@ -576,6 +581,15 @@ func (a *API) externalAPIRoutes() []apiRoute {
		{Method: "POST", Pattern: "/api/v1/servers/{name}/files/mkdir", h: a.handleMkdir},
		{Method: "POST", Pattern: "/api/v1/servers/{name}/files/rename", h: a.handleRenameFile},
		{Method: "PUT", Pattern: "/api/v1/servers/{name}/files/upload", h: a.handleUploadFile},
		// Scheduled tasks (handlers_schedules.go): a console command, restart, stop,
		// start or backup at set times, which felis-api's runner fires. App-tier and
		// owner-or-admin inside the handler, like the console and power routes they
		// automate; a schedule reaches nothing its owner could not do by hand.
		{Method: "GET", Pattern: "/api/v1/servers/{name}/schedules", h: a.handleListSchedules},
		{Method: "POST", Pattern: "/api/v1/servers/{name}/schedules", h: a.handleCreateSchedule},
		{Method: "PUT", Pattern: "/api/v1/servers/{name}/schedules/{id}", h: a.handleUpdateSchedule},
		{Method: "DELETE", Pattern: "/api/v1/servers/{name}/schedules/{id}", h: a.handleDeleteSchedule},
		{Method: "POST", Pattern: "/api/v1/servers/{name}/schedules/{id}/run", h: a.handleRunSchedule},
		// Account linking (spec §10), web side: /start reports link status (it is the
		// pointer handleClaim's 412 emits), /verify consumes the in-game code and binds
		// the account. App-tier, not admin — linking your own account is an ordinary
+4 −18
Changes for internal/api/handlers_console.go: 4 added lines, 18 removed lines.
Original line number Diff line number Diff line
@@ -3,7 +3,6 @@ package api
import (
	"errors"
	"net/http"
	"strings"

	"felis.lolicon.best/internal/naming"
)
@@ -51,23 +50,10 @@ func (a *API) handleCommand(w http.ResponseWriter, r *http.Request) {
		return
	}

	// A console command is exactly one line. Trim surrounding space, strip a
	// single leading '/' (players type "/say hi"; RCON wants "say hi"), then
	// reject control characters so one request can never smuggle a second command
	// past a newline.
	command := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(body.Command), "/"))
	if command == "" {
		writeError(w, r, newError(http.StatusBadRequest, "bad_request", "command is required"))
		return
	}
	if len(command) > maxConsoleCommandLen {
		writeError(w, r, newError(http.StatusBadRequest, "bad_request",
			"command too long (max %d bytes)", maxConsoleCommandLen))
		return
	}
	if strings.IndexFunc(command, func(c rune) bool { return c < 0x20 }) >= 0 {
		writeError(w, r, newError(http.StatusBadRequest, "bad_request",
			"command must be a single line (no control characters)"))
	// A console command is exactly one line (normalizeConsoleCommand).
	command, err := normalizeConsoleCommand(body.Command)
	if err != nil {
		writeError(w, r, err)
		return
	}

Loading