Felis/cmd/felis/doctor.go

package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"slices"
	"strings"
	"time"

	"felis.lolicon.best/internal/config"
	"felis.lolicon.best/internal/watchdog"
)

// systemdUnitDir is where the installer writes its units.
const systemdUnitDir = "/etc/systemd/system"

// hostCommand runs a host tool (systemctl, journalctl, k3s) and returns its
// stdout. A tool that exits non-zero still returns what it printed:
// `systemctl is-active` prints "inactive" and exits 3.
func hostCommand(ctx context.Context, name string, args ...string) ([]byte, error) {
	return exec.CommandContext(ctx, name, args...).Output()
}

// hostServices are the long-running units a full install depends on, checked
// when their unit file is present: k3s runs the cluster, felis-velocity is the
// game proxy, felis-nano the single-binary host that runs without k3s.
var hostServices = []string{"k3s.service", "felis-velocity.service", "felis-nano.service"}

// cmdDoctor runs every check felis watchdog runs, with the settings
// felis-watchdog.service gives it, plus what only the host shows (systemd
// units that failed or stopped, timers that no longer fire, alerts that reach
// no one), and prints them grouped by area with where to look next. It mails
// nothing, pings no heartbeat and leaves the watchdog's state alone: it is
// safe to run at any time, as often as wanted.
func cmdDoctor(args []string, stdout, stderr io.Writer) int {
	fs := flag.NewFlagSet("doctor", flag.ContinueOnError)
	fs.SetOutput(stderr)
	unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
	if err := fs.Parse(args); err != nil {
		if errors.Is(err, flag.ErrHelp) {
			return 0
		}
		return 2
	}
	if os.Geteuid() != 0 {
		fmt.Fprintln(stderr, "felis doctor: run as root (sudo felis doctor): the checks read root-only state under /etc/felis and /var/lib/felis")
		return 1
	}
	host, _ := os.Hostname()
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
	defer cancel()
	return runDoctor(ctx, doctorEnv{unitDir: *unitDir, run: hostCommand, now: time.Now(), host: host}, stdout)
}

// doctorEnv is what one doctor run reads the host through.
type doctorEnv struct {
	unitDir string
	run     func(ctx context.Context, name string, args ...string) ([]byte, error)
	now     time.Time
	host    string
}

// doctorAreas are the report's headings in order, by the area findingArea
// puts a finding under.
var doctorAreas = []struct{ key, title string }{
	{key: "config", title: "configuration"},
	{key: "cluster", title: "Kubernetes cluster"},
	{key: "postgres", title: "PostgreSQL"},
	{key: "proxy", title: "game proxy"},
	{key: "db-backup", title: "database backups"},
	{key: "offsite", title: "off-site copy"},
	{key: "scan-db", title: "build scan database"},
	{key: "disk", title: "disk space"},
	{key: "memory", title: "memory"},
	{key: "k3s-certs", title: "k3s certificates"},
	{key: "host-address", title: "node address"},
	{key: "clock", title: "clock"},
	{key: "systemd", title: "systemd units and timers"},
	{key: "alerts", title: "alerting"},
}

func runDoctor(ctx context.Context, env doctorEnv, stdout io.Writer) int {
	unit := filepath.Join(env.unitDir, "felis-watchdog.service")
	w, found, unitErr := watchdogUnitFlags(unit)
	var report watchdog.Report
	var notes []string
	fmt.Fprintf(stdout, "felis doctor on %s at %s\n", env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
	switch {
	case unitErr != nil:
		report.Findings = append(report.Findings, watchdog.Finding{
			Key: "watchdog/unit", Severity: watchdog.Critical,
			SummaryEN: fmt.Sprintf("cannot read the watchdog's settings: %v", unitErr),
			Hint:      "rerun the installer (deploy/bootstrap.sh) to rewrite felis-watchdog.service",
		})
		fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
	case !found:
		report.Findings = append(report.Findings, watchdog.Finding{
			Key: "watchdog/unit", Severity: watchdog.Critical,
			SummaryEN: fmt.Sprintf("%s is not installed: nothing checks this host or mails anyone when it breaks", unit),
			Hint:      "rerun the installer (deploy/bootstrap.sh), which installs felis-watchdog.timer",
		})
		fmt.Fprintf(stdout, "checks run with the watchdog's defaults (config %s)\n", w.cfgPath)
	default:
		fmt.Fprintf(stdout, "checks run as %s runs them (config %s)\n", unit, w.cfgPath)
	}
	fmt.Fprintln(stdout)

	skip := map[string]string{}
	cfg, cfgErr := config.Load(w.cfgPath)
	if cfgErr != nil {
		report.Findings = append(report.Findings, watchdog.Finding{
			Key: "config", Severity: watchdog.Critical,
			SummaryEN: cfgErr.Error(),
			Hint:      "the installer writes it (deploy/bootstrap.sh); felis watchdog fails on every run until it loads",
		})
		// The host's units are read without it; whether alerts reach anyone
		// is not known without its relay.
		for _, a := range doctorAreas {
			if a.key != "config" && a.key != "systemd" {
				skip[a.key] = "the configuration did not load"
			}
		}
	} else {
		_, owners, ownersErr := watchdogProbes(ctx, w, cfg, env.now, &report)
		report.Findings = append(report.Findings, alertReachFindings(cfg, owners, ownersErr)...)
		if w.proxyAddr == "" {
			skip["proxy"] = "no -proxy-addr"
		}
		if w.backupDir == "" {
			skip["db-backup"] = "no -backup-dir"
		}
		if !cfg.Offsite.Enabled() {
			skip["offsite"] = "not configured"
		}
		if !usesMirroredScanDB(cfg) {
			skip["scan-db"] = "builds do not scan against the registry's copy"
		}
		if len(splitList(w.certDirs)) == 0 {
			skip["k3s-certs"] = "no -k3s-cert-dirs"
		}
		if w.nodeIP == "" {
			skip["host-address"] = "no -node-ip"
		}
	}
	report.Findings = append(report.Findings, unitFindings(ctx, env)...)

	switch url, err := readHeartbeatURL(w.heartbeatFile); {
	case err != nil:
		report.Findings = append(report.Findings, watchdog.Finding{
			Key: "watchdog/heartbeat", Severity: watchdog.Warning,
			SummaryEN: fmt.Sprintf("the heartbeat URL is unusable, so no run pings it: %v", err),
			Hint:      "rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL set (docs/troubleshooting.md §14)",
		})
	case url == "":
		notes = append(notes, "no heartbeat URL is set: a host that goes down entirely, or a watchdog that stops running, alerts no one. "+
			"Rerun the installer with FELIS_WATCHDOG_HEARTBEAT_URL (docs/troubleshooting.md §14)")
	}
	if until := watchdog.QuietUntil(w.quietPath); env.now.Before(until) {
		notes = append(notes, fmt.Sprintf("the watchdog mails nothing until %s (%s): the installer holds it while it restarts things on purpose, "+
			"and a marker an installer killed mid-run left behind holds it until then",
			until.UTC().Format("2006-01-02 15:04 UTC"), w.quietPath))
	}
	return printDoctorReport(stdout, report.Findings, skip, notes)
}

// printDoctorReport prints each area's findings under its heading, an area
// with none as fine or, when skip says why, as not checked, then the notes
// and the count. It returns the exit status: 1 when anything was found.
func printDoctorReport(stdout io.Writer, findings []watchdog.Finding, skip map[string]string, notes []string) int {
	byArea := map[string][]watchdog.Finding{}
	for _, f := range findings {
		a := findingArea(f.Key)
		byArea[a] = append(byArea[a], f)
	}
	var critical, warning int
	for _, a := range doctorAreas {
		fs := byArea[a.key]
		switch {
		case len(fs) > 0:
		case skip[a.key] != "":
			fmt.Fprintf(stdout, "-  %s: not checked, %s\n", a.title, skip[a.key])
			continue
		default:
			fmt.Fprintf(stdout, "✓  %s\n", a.title)
			continue
		}
		mark := "!"
		if slices.ContainsFunc(fs, func(f watchdog.Finding) bool { return f.Severity == watchdog.Critical }) {
			mark = "✗"
		}
		fmt.Fprintf(stdout, "%s  %s\n", mark, a.title)
		for _, f := range fs {
			if f.Severity == watchdog.Critical {
				critical++
			} else {
				warning++
			}
			fmt.Fprintf(stdout, "     %-8s %s: %s\n", f.Severity, f.Key, f.SummaryEN)
			if f.Hint != "" {
				fmt.Fprintf(stdout, "              → %s\n", f.Hint)
			}
		}
	}
	for _, n := range notes {
		fmt.Fprintf(stdout, "\nnote: %s\n", n)
	}
	fmt.Fprintln(stdout)
	if critical+warning == 0 {
		fmt.Fprintln(stdout, "no problems found")
		return 0
	}
	fmt.Fprintf(stdout, "%d problem(s): %d critical, %d warning(s)\n", critical+warning, critical, warning)
	return 1
}

// findingArea is the report heading a finding key goes under.
func findingArea(key string) string {
	head, _, _ := strings.Cut(key, "/")
	switch head {
	case "kube-api", "deployment", "system-server", "server-failed", "job-failed", "reaper-stale", "node":
		return "cluster"
	case "db-backup", "db-backup-servers":
		return "db-backup"
	case "unit", "timer":
		return "systemd"
	case "watchdog":
		return "alerts"
	}
	return head
}

// alertReachFindings is why the watchdog's alerts would reach no one, which
// its own runs only log: no relay, or no owner with a verified address.
func alertReachFindings(cfg *config.Config, owners []string, ownersErr error) []watchdog.Finding {
	var out []watchdog.Finding
	if cfg.SMTP.Host == "" {
		out = append(out, watchdog.Finding{
			Key: "alerts/relay", Severity: watchdog.Warning,
			SummaryEN: "no [smtp] relay is configured: the watchdog logs its alerts to the journal and mails no one",
			Hint:      "sudo felis setup, step SMTP",
		})
	}
	if ownersErr == nil && len(owners) == 0 {
		out = append(out, watchdog.Finding{
			Key: "alerts/recipients", Severity: watchdog.Warning,
			SummaryEN: "no owner account has a verified email: the watchdog's alerts reach no one",
			Hint:      "an owner verifies an address in the panel's account settings",
		})
	}
	return out
}

// unitFindings reports the installer's systemd units that failed, the
// long-running ones that are not running, and timers that no longer fire.
func unitFindings(ctx context.Context, env doctorEnv) []watchdog.Finding {
	var out []watchdog.Finding
	seen := map[string]bool{}
	failed, err := env.run(ctx, "systemctl", "list-units", "--all", "--plain", "--no-legend", "--no-pager", "--state=failed", "felis-*", "k3s.service")
	if err != nil && len(failed) == 0 {
		return []watchdog.Finding{{
			Key: "unit/systemctl", Severity: watchdog.Warning,
			SummaryEN: fmt.Sprintf("systemctl list-units failed, so no unit was checked: %v", err),
		}}
	}
	for _, line := range strings.Split(string(failed), "\n") {
		fields := strings.Fields(line)
		if len(fields) == 0 {
			continue
		}
		name := fields[0]
		seen[name] = true
		out = append(out, watchdog.Finding{
			Key: "unit/" + name, Severity: watchdog.Critical,
			SummaryEN: name + " failed",
			Hint:      fmt.Sprintf("journalctl -u %s -n 100 --no-pager; once fixed, sudo systemctl reset-failed %s (a timer's job clears on its next good run)", name, name),
		})
	}
	for _, name := range hostServices {
		if seen[name] {
			continue
		}
		if _, err := os.Stat(filepath.Join(env.unitDir, name)); err != nil {
			continue
		}
		if state := unitActiveState(ctx, env, name); state != "active" {
			out = append(out, watchdog.Finding{
				Key: "unit/" + name, Severity: watchdog.Critical,
				SummaryEN: fmt.Sprintf("%s is %s", name, state),
				Hint:      fmt.Sprintf("sudo systemctl start %s; journalctl -u %s -n 100 --no-pager", name, name),
			})
		}
	}
	timers, _ := filepath.Glob(filepath.Join(env.unitDir, "felis-*.timer"))
	for _, path := range timers {
		name := filepath.Base(path)
		if state := unitActiveState(ctx, env, name); state != "active" {
			out = append(out, watchdog.Finding{
				Key: "timer/" + name, Severity: watchdog.Warning,
				SummaryEN: fmt.Sprintf("%s is %s: the job it starts no longer runs", name, state),
				Hint:      fmt.Sprintf("sudo systemctl enable --now %s", name),
			})
		}
	}
	return out
}

// unitActiveState is what `systemctl is-active` says of unit.
func unitActiveState(ctx context.Context, env doctorEnv, unit string) string {
	out, err := env.run(ctx, "systemctl", "is-active", unit)
	if state := strings.TrimSpace(string(out)); state != "" {
		return state
	}
	return fmt.Sprintf("in an unknown state (systemctl is-active: %v)", err)
}

// watchdogUnitFlags reads the flags felis-watchdog.service runs felis
// watchdog with. found is false when there is no such unit; w is then the
// watchdog's defaults.
func watchdogUnitFlags(path string) (w watchdogFlags, found bool, err error) {
	fs := flag.NewFlagSet("watchdog", flag.ContinueOnError)
	fs.SetOutput(io.Discard)
	w.register(fs)
	raw, err := os.ReadFile(path)
	if errors.Is(err, os.ErrNotExist) {
		return w, false, nil
	}
	if err != nil {
		return w, false, err
	}
	for _, line := range strings.Split(string(raw), "\n") {
		cmd, ok := strings.CutPrefix(strings.TrimSpace(line), "ExecStart=")
		if !ok {
			continue
		}
		fields := execArgs(cmd)
		i := slices.Index(fields, "watchdog")
		if i < 0 {
			continue
		}
		if err := fs.Parse(fields[i+1:]); err != nil {
			return w, true, fmt.Errorf("%s: %w", path, err)
		}
		return w, true, nil
	}
	return w, true, fmt.Errorf("%s runs no `felis watchdog`", path)
}

// execArgs splits an ExecStart= command line into its words. A word may be
// quoted with " or ', as systemd allows, which is how an empty value is
// written.
func execArgs(s string) []string {
	var out []string
	var cur strings.Builder
	inWord := false
	var quote rune
	for _, r := range s {
		switch {
		case quote != 0:
			if r == quote {
				quote = 0
			} else {
				cur.WriteRune(r)
			}
		case r == '"' || r == '\'':
			quote, inWord = r, true
		case r == ' ' || r == '\t':
			if inWord {
				out = append(out, cur.String())
				cur.Reset()
				inWord = false
			}
		default:
			cur.WriteRune(r)
			inWord = true
		}
	}
	if inWord {
		out = append(out, cur.String())
	}
	return out
}