// Package maintenance is the per-server mutual exclusion between a game server // and the Jobs that write or snapshot its world volume (restore, backup, file // write). The world PVC is ReadWriteOnce, and RWO is exclusive per NODE: on a // single-node cluster the game pod and a restore pod mount it side by side, so // the access mode alone guards nothing. A server woken mid-restore boots on a // half-extracted world and the restore then prunes what it wrote; a server woken // mid-backup produces a torn archive that a later restore makes permanent. // // Two signals mark a volume as held: // // - an unfinished maintenance Job labelled for the server. Once the Job exists // it IS the lock, for as long as it runs, however long that is. // - the Annotation on the MinecraftServer, written by felis-api with an // optimistic-lock patch before it creates the Job and removed right after. // It bridges the gap between "admitted" and "the Job is visible", and it is // what serialises admission against a wake: both write the same object under // its resourceVersion, so one of two racing writers always loses with a // conflict and re-checks. // // A lock older than Grace with no Job behind it is stale (felis-api died between // the two writes) and holds nothing. // // File reads and listings are not holders. They mount the volume read-only for a // second or two, and a server starting beside one cannot hurt either side, so // nobody waits for them. package maintenance import ( "strings" "time" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" ) const ( // Annotation is the admission lock on a MinecraftServer. Its value is // "@" (LockValue). Annotation = "felis.lolicon.best/maintenance" // Grace is how long a lock counts as held when no Job backs it. felis-api // creates the Job within milliseconds of taking the lock and then drops it, so // a lock this old means the process died in between. Grace = 2 * time.Minute // LabelServer / LabelManagedBy are the labels every maintenance executor puts // on its Job (internal/restore, internal/backupjob, internal/fileedit keep // their own copies; maintenance_test pins them against these). LabelServer = "felis.lolicon.best/server" LabelManagedBy = "app.kubernetes.io/managed-by" // LabelFilesMode is the file-editor operation (list, read, write) a files Job // performs. Only write holds the volume. LabelFilesMode = "felis.lolicon.best/files-mode" ) // Kinds of holder. const ( KindRestore = "restore" KindBackup = "backup" KindFileWrite = "file-write" ) // FilesModeWrite is the LabelFilesMode value of a file write. const FilesModeWrite = "write" // JobKind names the holder a Job represents, or reports false for a Job that // holds nothing (a file read, a build, anything else in the namespace). A files // Job without LabelFilesMode predates the label and is counted as a write: it can // only be an old Job still inside its TTL, and over-counting it for that window // is the safe side. func JobKind(j *batchv1.Job) (string, bool) { switch j.Labels[LabelManagedBy] { case "felis-restore": return KindRestore, true case "felis-backup": return KindBackup, true case "felis-files": mode, ok := j.Labels[LabelFilesMode] if !ok || mode == FilesModeWrite { return KindFileWrite, true } } return "", false } // JobFinished reports whether a Job has reached a terminal condition. The // success/failure-target conditions count as terminal: the Job controller sets // them the moment the outcome is decided, before it finishes tearing the pods // down, and waiting for Complete would keep a wake refused for no reason. func JobFinished(j *batchv1.Job) bool { for _, c := range j.Status.Conditions { if c.Status != corev1.ConditionTrue { continue } switch c.Type { case batchv1.JobComplete, batchv1.JobFailed, batchv1.JobSuccessCriteriaMet, batchv1.JobFailureTarget: return true } } return false } // LockValue renders the Annotation value for a holder admitted at `at`. func LockValue(kind string, at time.Time) string { return kind + "@" + at.UTC().Format(time.RFC3339) } // parseLock splits a lock value. A value that does not parse is stale: a lock // nobody can date must not be able to hold a server down forever. func parseLock(v string) (string, time.Time, bool) { kind, stamp, ok := strings.Cut(v, "@") if !ok || kind == "" { return "", time.Time{}, false } at, err := time.Parse(time.RFC3339, stamp) if err != nil { return "", time.Time{}, false } return kind, at, true } // Holder reports what, if anything, holds the server's world volume at `now`: // the first unfinished maintenance Job among jobs, else a lock in annotations // younger than Grace. jobs may contain unrelated Jobs; only the server's own // holders count. func Holder(server string, annotations map[string]string, jobs []batchv1.Job, now time.Time) (string, bool) { for i := range jobs { j := &jobs[i] if j.Labels[LabelServer] != server || JobFinished(j) { continue } if kind, ok := JobKind(j); ok { return kind, true } } if v, ok := annotations[Annotation]; ok { if kind, at, ok := parseLock(v); ok && now.Sub(at) < Grace && at.Sub(now) < Grace { return kind, true } } return "", false }