Felis/cmd/felis/status.go

package main

import (
	"bufio"
	"context"
	"errors"
	"flag"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"text/tabwriter"
	"time"

	"felis.lolicon.best/internal/apis/felis/v1alpha1"
	"felis.lolicon.best/internal/config"
	"felis.lolicon.best/internal/dbbackup"
	"felis.lolicon.best/internal/offsite"
	"felis.lolicon.best/internal/platform"
	"felis.lolicon.best/internal/store"
	"felis.lolicon.best/internal/watchdog"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"sigs.k8s.io/controller-runtime/pkg/client"
)

// cmdStatus prints the platform at a glance: the release and the node, the
// control plane's workloads, the game proxy, every server with its players
// and newest world backup, the database backups and the off-site copy, the
// host's disks and memory, and what the watchdog has open. It changes
// nothing, and a part that is down (the cluster, PostgreSQL) reads as such
// while the rest still prints. felis doctor says what is wrong and where to
// look.
func cmdStatus(args []string, stdout, stderr io.Writer) int {
	fs := flag.NewFlagSet("status", flag.ContinueOnError)
	fs.SetOutput(stderr)
	unitDir := fs.String("systemd-dir", systemdUnitDir, "where the installer's systemd units are")
	if err := fs.Parse(args); err != nil {
		if errors.Is(err, flag.ErrHelp) {
			return 0
		}
		return 2
	}
	if os.Geteuid() != 0 {
		fmt.Fprintln(stderr, "felis status: run as root (sudo felis status): it reads root-only state under /etc/felis and /var/lib/felis")
		return 1
	}
	env, err := hostStatusEnv(*unitDir)
	if err != nil {
		fmt.Fprintf(stderr, "felis status: %v\n", err)
		return 1
	}
	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
	defer cancel()
	printStatus(ctx, env, stdout)
	return 0
}

// statusEnv is what one status report reads the host through.
type statusEnv struct {
	cfg *config.Config
	// w is how felis-watchdog.service runs the watchdog: where the backups,
	// the off-site record and the proxy are.
	w       watchdogFlags
	cl      client.Client // nil while the API server is unreachable
	clErr   error
	backups func(ctx context.Context) (map[string]time.Time, error)
	run     func(ctx context.Context, name string, args ...string) ([]byte, error)
	unitDir string
	meminfo string
	host    string
	now     time.Time
}

// hostStatusEnv reads this host: the watchdog's settings, the configuration
// they name, and the cluster.
func hostStatusEnv(unitDir string) (statusEnv, error) {
	w, _, err := watchdogUnitFlags(filepath.Join(unitDir, "felis-watchdog.service"))
	if err != nil {
		return statusEnv{}, err
	}
	cfg, err := config.Load(w.cfgPath)
	if err != nil {
		return statusEnv{}, err
	}
	cl, clErr := buildSystemServerClient()
	if clErr != nil {
		cl = nil
	}
	host, _ := os.Hostname()
	return statusEnv{
		cfg: cfg, w: w, cl: cl, clErr: clErr,
		backups: func(ctx context.Context) (map[string]time.Time, error) {
			return newestWorldBackups(ctx, cfg.Database.URL)
		},
		run: hostCommand, unitDir: unitDir, meminfo: "/proc/meminfo", host: host, now: time.Now(),
	}, nil
}

func printStatus(ctx context.Context, env statusEnv, out io.Writer) {
	fmt.Fprintf(out, "felis %s on %s at %s\n", resolvedVersion(), env.host, env.now.UTC().Format("2006-01-02 15:04 UTC"))
	if env.cl == nil {
		fmt.Fprintf(out, "cluster:  unreachable (%v)\n", env.clErr)
	} else {
		statusCluster(ctx, env, out)
	}
	statusProxy(ctx, env, out)
	statusServers(ctx, env, out)
	statusBackups(env, out)
	statusHost(env, out)
	statusWatchdog(ctx, env, out)
}

func statusCluster(ctx context.Context, env statusEnv, out io.Writer) {
	var nodes corev1.NodeList
	if err := env.cl.List(ctx, &nodes); err != nil {
		fmt.Fprintf(out, "cluster:  unreachable (%v)\n", err)
		return
	}
	for _, n := range nodes.Items {
		ready := "NotReady"
		for _, c := range n.Status.Conditions {
			if c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue {
				ready = "Ready"
			}
		}
		info := n.Status.NodeInfo
		fmt.Fprintf(out, "node:     %s %s, k3s %s, %s, kernel %s\n", n.Name, ready, info.KubeletVersion, info.OSImage, info.KernelVersion)
	}

	ns := env.w.controlNS
	var deps appsv1.DeploymentList
	var pods corev1.PodList
	err := env.cl.List(ctx, &deps, client.InNamespace(ns))
	if err == nil {
		err = env.cl.List(ctx, &pods, client.InNamespace(ns))
	}
	fmt.Fprintf(out, "\ncontrol plane (namespace %s):\n", ns)
	if err != nil {
		fmt.Fprintf(out, "  cannot list it: %v\n", err)
		return
	}
	sort.Slice(deps.Items, func(i, j int) bool { return deps.Items[i].Name < deps.Items[j].Name })
	tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
	for _, d := range deps.Items {
		want := int32(1)
		if d.Spec.Replicas != nil {
			want = *d.Spec.Replicas
		}
		image := "-"
		if cs := d.Spec.Template.Spec.Containers; len(cs) > 0 {
			image = shortImage(cs[0].Image)
		}
		fmt.Fprintf(tw, "  %s\t%d/%d ready\t%s\trestarts %d\n", d.Name, d.Status.ReadyReplicas, want, image, podRestarts(d.Spec.Selector, pods.Items))
	}
	tw.Flush()
}

// podRestarts adds up the container restarts of the pods selector picks (the
// API server refuses a Deployment whose selector is empty).
func podRestarts(selector *metav1.LabelSelector, pods []corev1.Pod) int32 {
	sel, err := metav1.LabelSelectorAsSelector(selector)
	if err != nil {
		return 0
	}
	var n int32
	for _, p := range pods {
		if !sel.Matches(labels.Set(p.Labels)) {
			continue
		}
		for _, cs := range p.Status.ContainerStatuses {
			n += cs.RestartCount
		}
	}
	return n
}

// shortImage is an image reference without its registry and repository path,
// and with its digest cut to 12 characters.
func shortImage(ref string) string {
	if i := strings.LastIndex(ref, "/"); i >= 0 {
		ref = ref[i+1:]
	}
	if name, digest, ok := strings.Cut(ref, "@sha256:"); ok && len(digest) > 12 {
		ref = name + "@" + digest[:12]
	}
	return ref
}

func statusProxy(ctx context.Context, env statusEnv, out io.Writer) {
	var parts []string
	if _, err := os.Stat(filepath.Join(env.unitDir, "felis-velocity.service")); err == nil {
		parts = append(parts, "felis-velocity "+unitActiveState(ctx, doctorEnv{run: env.run}, "felis-velocity.service"))
	}
	if env.w.proxyAddr != "" {
		d := net.Dialer{Timeout: 3 * time.Second}
		if conn, err := d.DialContext(ctx, "tcp", env.w.proxyAddr); err != nil {
			parts = append(parts, fmt.Sprintf("%s refuses connections (%v)", env.w.proxyAddr, err))
		} else {
			conn.Close()
			parts = append(parts, env.w.proxyAddr+" accepts connections")
		}
	}
	if len(parts) == 0 {
		parts = append(parts, "not on this host")
	}
	fmt.Fprintf(out, "\nproxy:    %s\n", strings.Join(parts, ", "))
}

func statusServers(ctx context.Context, env statusEnv, out io.Writer) {
	ns := env.cfg.K8s.Namespace
	if ns == "" {
		ns = platform.DefaultMinecraftNamespace
	}
	if env.cl == nil {
		fmt.Fprintf(out, "\nservers (namespace %s): unknown while the cluster is unreachable\n", ns)
		return
	}
	var list v1alpha1.MinecraftServerList
	if err := env.cl.List(ctx, &list, client.InNamespace(ns)); err != nil {
		fmt.Fprintf(out, "\nservers (namespace %s): cannot list them: %v\n", ns, err)
		return
	}
	newest, backupErr := env.backups(ctx)
	running, online := 0, int32(0)
	for _, ms := range list.Items {
		if ms.Status.Phase == v1alpha1.PhaseRunning {
			running++
			online += ms.Status.Players.Online
		}
	}
	fmt.Fprintf(out, "\nservers (namespace %s): %d, %d running, %d players online\n", ns, len(list.Items), running, online)
	if len(list.Items) == 0 {
		return
	}
	sort.Slice(list.Items, func(i, j int) bool { return list.Items[i].Name < list.Items[j].Name })
	tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
	fmt.Fprintln(tw, "  NAME\tROLE\tDESIRED\tPHASE\tPLAYERS\tNEWEST WORLD BACKUP")
	for _, ms := range list.Items {
		role := ms.Labels[v1alpha1.LabelSystemRole]
		if role == "" {
			role = "-"
		}
		phase := string(ms.Status.Phase)
		if phase == "" {
			phase = "-"
		}
		players := "-"
		if ms.Status.Phase == v1alpha1.PhaseRunning {
			players = fmt.Sprintf("%d/%d", ms.Status.Players.Online, ms.Status.Players.Max)
		}
		backup := "none"
		switch at, ok := newest[ms.Name]; {
		case backupErr != nil:
			backup = "?"
		case ok:
			backup = dbbackup.Age(env.now.Sub(at)) + " ago"
		}
		fmt.Fprintf(tw, "  %s\t%s\t%s\t%s\t%s\t%s\n", ms.Name, role, orDash(string(ms.Spec.DesiredState)), phase, players, backup)
	}
	tw.Flush()
	if backupErr != nil {
		fmt.Fprintf(out, "  world backups unknown: %v\n", backupErr)
	}
}

func orDash(s string) string {
	if s == "" {
		return "-"
	}
	return s
}

// newestWorldBackups is when each server's newest world backup that a restore
// can use was taken.
func newestWorldBackups(ctx context.Context, url string) (map[string]time.Time, error) {
	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
	defer cancel()
	drv, err := store.Open(ctx, url)
	if err != nil {
		return nil, err
	}
	defer drv.Close()
	rows, err := drv.DB().QueryContext(ctx, `SELECT server_name, max(created_at) FROM world_backups
		WHERE status = 'present' AND corrupt_at IS NULL GROUP BY server_name`)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	out := map[string]time.Time{}
	for rows.Next() {
		var name string
		var at time.Time
		if err := rows.Scan(&name, &at); err != nil {
			return nil, err
		}
		out[name] = at
	}
	return out, rows.Err()
}

func statusBackups(env statusEnv, out io.Writer) {
	fmt.Fprintln(out, "\nbackups:")
	switch bundles, err := dbbackup.List(env.w.backupDir); {
	case env.w.backupDir == "":
		fmt.Fprintln(out, "  database: not checked (felis-watchdog.service names no -backup-dir)")
	case err != nil:
		fmt.Fprintf(out, "  database: cannot read %s: %v\n", env.w.backupDir, err)
	case len(bundles) == 0:
		fmt.Fprintf(out, "  database: none in %s\n", env.w.backupDir)
	default:
		fmt.Fprintf(out, "  database: newest %s, %s ago; %d bundles in %s\n",
			bundles[0].Name, dbbackup.Age(env.now.Sub(bundles[0].Created)), len(bundles), env.w.backupDir)
	}
	if !env.cfg.Offsite.Enabled() {
		fmt.Fprintln(out, "  off-site: not configured, every backup is on this machine only")
		return
	}
	switch st, err := offsite.ReadStatus(env.w.offsiteStatus); {
	case err != nil:
		fmt.Fprintf(out, "  off-site: %v\n", err)
	case st == nil:
		fmt.Fprintln(out, "  off-site: never synced")
	case st.LastSuccess.IsZero():
		fmt.Fprintf(out, "  off-site: never succeeded; last attempt %s ago: %s\n", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
	default:
		line := fmt.Sprintf("  off-site: last good sync %s ago to %s", dbbackup.Age(env.now.Sub(st.LastSuccess)), st.Bucket)
		if st.LastAttempt.After(st.LastSuccess) { // a failed run records no success
			line += fmt.Sprintf("; the last attempt, %s ago, failed: %s", dbbackup.Age(env.now.Sub(st.LastAttempt)), st.LastError)
		}
		fmt.Fprintln(out, line)
	}
}

func statusHost(env statusEnv, out io.Writer) {
	fmt.Fprintln(out, "\nhost:")
	seen := map[uint64]bool{}
	for _, p := range splitList(env.w.diskPaths) {
		var st syscall.Stat_t
		if err := syscall.Stat(p, &st); err != nil {
			continue
		}
		dev := uint64(st.Dev) // int32 on darwin
		if seen[dev] {
			continue
		}
		seen[dev] = true
		var fs syscall.Statfs_t
		if err := syscall.Statfs(p, &fs); err != nil || fs.Blocks == 0 {
			continue
		}
		bsize := uint64(fs.Bsize) // uint32 on darwin
		total, avail := uint64(fs.Blocks)*bsize, uint64(fs.Bavail)*bsize
		fmt.Fprintf(out, "  disk %s: %s free of %s (%.0f%% free)\n", p, offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)), float64(fs.Bavail)/float64(fs.Blocks)*100)
	}
	if total, avail, ok := readMeminfo(env.meminfo); ok {
		fmt.Fprintf(out, "  memory: %s available of %s\n", offsite.HumanBytes(int64(avail)), offsite.HumanBytes(int64(total)))
	}
}

// readMeminfo reads MemTotal and MemAvailable, in bytes, from a /proc/meminfo
// style file.
func readMeminfo(path string) (total, avail uint64, ok bool) {
	f, err := os.Open(path)
	if err != nil {
		return 0, 0, false
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		fields := strings.Fields(sc.Text())
		if len(fields) < 2 {
			continue
		}
		v, _ := strconv.ParseUint(fields[1], 10, 64) // the kernel writes numbers
		switch fields[0] {
		case "MemTotal:":
			total = v * 1024
		case "MemAvailable:":
			avail = v * 1024
		}
	}
	return total, avail, total > 0
}

func statusWatchdog(ctx context.Context, env statusEnv, out io.Writer) {
	fmt.Fprintln(out, "\nwatchdog:")
	if _, err := os.Stat(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
		fmt.Fprintln(out, "  not installed: nothing checks this host")
		return
	}
	line := "  timer " + unitActiveState(ctx, doctorEnv{run: env.run}, "felis-watchdog.timer")
	show, _ := env.run(ctx, "systemctl", "show", "--timestamp=unix", "-p", "Result", "-p", "ExecMainExitTimestamp", "felis-watchdog.service")
	props := map[string]string{}
	for _, ln := range strings.Split(string(show), "\n") {
		if k, v, ok := strings.Cut(strings.TrimSpace(ln), "="); ok {
			props[k] = v
		}
	}
	// ExecMainExitTimestamp is empty until the service has run.
	if sec, _ := strconv.ParseInt(strings.TrimPrefix(props["ExecMainExitTimestamp"], "@"), 10, 64); sec > 0 {
		line += fmt.Sprintf(", last run %s ago (%s)", dbbackup.Age(env.now.Sub(time.Unix(sec, 0))), orDash(props["Result"]))
	}
	fmt.Fprintln(out, line)

	state, err := watchdog.LoadState(watchdog.NewestState(env.w.statePath, env.w.fallbackState))
	if err != nil {
		fmt.Fprintf(out, "  alerts: unknown (%v)\n", err)
		return
	}
	var open []string
	for key, a := range state.Alerts {
		if !a.ClearedAt.IsZero() {
			continue
		}
		if a.Notified.IsZero() {
			open = append(open, fmt.Sprintf("%s (%s, seen %s ago, not mailed yet)", key, a.Severity, dbbackup.Age(env.now.Sub(a.FirstSeen))))
		} else {
			open = append(open, fmt.Sprintf("%s (%s, mailed %s ago)", key, a.Severity, dbbackup.Age(env.now.Sub(a.Notified))))
		}
	}
	if len(open) == 0 {
		fmt.Fprintln(out, "  alerts: none open")
		return
	}
	sort.Strings(open)
	fmt.Fprintf(out, "  alerts: %d open (sudo felis doctor says where to look)\n", len(open))
	for _, o := range open {
		fmt.Fprintf(out, "    %s\n", o)
	}
}