491 lines
21 KiB
Go
491 lines
21 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"felis.lolicon.best/internal/apis/felis/v1alpha1"
|
|
"felis.lolicon.best/internal/config"
|
|
"felis.lolicon.best/internal/dbbackup"
|
|
"felis.lolicon.best/internal/offsite"
|
|
"felis.lolicon.best/internal/watchdog"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/meta"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
|
)
|
|
|
|
// statusCluster is a one-node install: felis-api with a restarted pod, three
|
|
// servers (one of them the lobby), and a pod of another app whose restarts
|
|
// are not felis-api's.
|
|
func statusClusterObjects() []client.Object {
|
|
replicas := int32(1)
|
|
apiLabels := map[string]string{"app": "felis-api"}
|
|
return []client.Object{
|
|
&corev1.Node{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "felis-1"},
|
|
Status: corev1.NodeStatus{
|
|
Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}},
|
|
NodeInfo: corev1.NodeSystemInfo{KubeletVersion: "v1.36.4+k3s1", OSImage: "CentOS Stream 9", KernelVersion: "5.14.0-630.el9.aarch64"},
|
|
},
|
|
},
|
|
&appsv1.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "felis-api", Namespace: "felis"},
|
|
Spec: appsv1.DeploymentSpec{
|
|
Replicas: &replicas,
|
|
Selector: &metav1.LabelSelector{MatchLabels: apiLabels},
|
|
Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{
|
|
Name: "api", Image: "registry.felis.svc:5000/felis/felis-api:v1.4.0@sha256:0123456789abcdef0123456789abcdef",
|
|
}}}},
|
|
},
|
|
Status: appsv1.DeploymentStatus{ReadyReplicas: 1},
|
|
},
|
|
&corev1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "felis-api-7d9", Namespace: "felis", Labels: apiLabels},
|
|
Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "api", RestartCount: 2}}},
|
|
},
|
|
&corev1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "other-1", Namespace: "felis", Labels: map[string]string{"app": "other"}},
|
|
Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "x", RestartCount: 5}}},
|
|
},
|
|
&v1alpha1.MinecraftServer{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "survival", Namespace: "minecraft"},
|
|
Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
|
|
Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseRunning, Players: v1alpha1.PlayersStatus{Online: 2, Max: 20}},
|
|
},
|
|
&v1alpha1.MinecraftServer{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "creative", Namespace: "minecraft"},
|
|
Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredStopped},
|
|
Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStopped},
|
|
},
|
|
&v1alpha1.MinecraftServer{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "lobby", Namespace: "minecraft", Labels: map[string]string{v1alpha1.LabelSystemRole: "lobby"}},
|
|
Spec: v1alpha1.MinecraftServerSpec{DesiredState: v1alpha1.DesiredRunning},
|
|
Status: v1alpha1.MinecraftServerStatus{Phase: v1alpha1.PhaseStarting},
|
|
},
|
|
// Another namespace's server is not this install's.
|
|
&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "elsewhere", Namespace: "other"}},
|
|
}
|
|
}
|
|
|
|
// statusTestEnv is a host with the cluster above, a database backup nine hours
|
|
// old, an off-site copy last good two hours ago, and one alert open.
|
|
func statusTestEnv(t *testing.T) statusEnv {
|
|
t.Helper()
|
|
now := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
|
|
dir := t.TempDir()
|
|
var w watchdogFlags
|
|
w.controlNS = "felis"
|
|
w.backupDir = filepath.Join(dir, "db-backups")
|
|
w.offsiteStatus = filepath.Join(dir, "offsite-status.json")
|
|
w.statePath = filepath.Join(dir, "state.json")
|
|
w.fallbackState = filepath.Join(dir, "fallback.json")
|
|
w.diskPaths = "/nonexistent-felis-status-test"
|
|
unitDir := filepath.Join(dir, "systemd")
|
|
for _, d := range []string{w.backupDir, unitDir} {
|
|
if err := os.MkdirAll(d, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-9*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
|
|
writeTestFile(t, filepath.Join(w.backupDir, dbbackup.BundleName(now.Add(-33*time.Hour), dbbackup.LabelDaily)), "x", 0o600)
|
|
if err := offsite.WriteStatus(w.offsiteStatus, offsite.Status{
|
|
LastAttempt: now.Add(-2 * time.Hour), LastSuccess: now.Add(-2 * time.Hour), Bucket: "felis-dr", Endpoint: "https://s3.example.com",
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state := &watchdog.State{Alerts: map[string]*watchdog.Alert{
|
|
"db-backup-servers": {Finding: watchdog.Finding{Key: "db-backup-servers", Severity: watchdog.Warning}, FirstSeen: now.Add(-3 * time.Hour), Notified: now.Add(-90 * time.Minute)},
|
|
"proxy": {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: now.Add(-time.Hour), Notified: now.Add(-time.Hour), ClearedAt: now.Add(-10 * time.Minute)},
|
|
"disk//var": {Finding: watchdog.Finding{Key: "disk//var", Severity: watchdog.Critical}, FirstSeen: now.Add(-4 * time.Minute)},
|
|
}}
|
|
if err := watchdog.SaveState(w.statePath, state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
meminfo := filepath.Join(dir, "meminfo")
|
|
writeTestFile(t, meminfo, "MemTotal: 8000000 kB\nMemFree: 100000 kB\nMemAvailable: 2000000 kB\n", 0o644)
|
|
writeTestFile(t, filepath.Join(unitDir, "felis-watchdog.timer"), "[Unit]\n", 0o644)
|
|
cfg := &config.Config{Offsite: config.OffsiteConfig{Endpoint: "https://s3.example.com", Bucket: "felis-dr"}}
|
|
return statusEnv{
|
|
cfg: cfg, w: w, cl: fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).Build(),
|
|
backups: func(context.Context) (map[string]time.Time, error) {
|
|
return map[string]time.Time{"survival": now.Add(-3 * time.Hour)}, nil
|
|
},
|
|
run: func(_ context.Context, name string, args ...string) ([]byte, error) {
|
|
switch strings.Join(append([]string{name}, args...), " ") {
|
|
case "systemctl is-active felis-watchdog.timer":
|
|
return []byte("active\n"), nil
|
|
case "systemctl show --timestamp=unix -p Result -p ExecMainExitTimestamp felis-watchdog.service":
|
|
return []byte("Result=success\nExecMainExitTimestamp=@" + strconv.FormatInt(now.Add(-70*time.Second).Unix(), 10) + "\n"), nil
|
|
}
|
|
return nil, errors.New("unexpected")
|
|
},
|
|
unitDir: unitDir, meminfo: meminfo, host: "felis-test", now: now,
|
|
}
|
|
}
|
|
|
|
// lineFields is the words of the first line of out that starts with prefix
|
|
// once trimmed.
|
|
func lineFields(out, prefix string) []string {
|
|
for _, l := range strings.Split(out, "\n") {
|
|
if strings.HasPrefix(strings.TrimSpace(l), prefix) {
|
|
return strings.Fields(l)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func TestStatusReport(t *testing.T) {
|
|
env := statusTestEnv(t)
|
|
var out bytes.Buffer
|
|
printStatus(context.Background(), env, &out)
|
|
got := out.String()
|
|
for _, want := range []string{
|
|
"on felis-test at 2026-09-27 12:00 UTC\n",
|
|
"node: felis-1 Ready, k3s v1.36.4+k3s1, CentOS Stream 9, kernel 5.14.0-630.el9.aarch64\n",
|
|
"\ncontrol plane (namespace felis):\n",
|
|
"\nproxy: not on this host\n",
|
|
"\nservers (namespace minecraft): 3, 1 running, 2 players online\n",
|
|
" database: newest " + dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily) + ", 9h0m ago; 2 bundles in " + env.w.backupDir + "\n",
|
|
" off-site: last good sync 2h0m ago to felis-dr\n",
|
|
" memory: 1.9 GiB available of 7.6 GiB\n",
|
|
" timer active, last run 1m ago (success)\n",
|
|
" alerts: 2 open (sudo felis doctor says where to look)\n" +
|
|
" db-backup-servers (warning, mailed 1h30m ago)\n" +
|
|
" disk//var (critical, seen 4m ago, not mailed yet)\n",
|
|
} {
|
|
if !strings.Contains(got, want) {
|
|
t.Errorf("status lacks %q:\n%s", want, got)
|
|
}
|
|
}
|
|
for prefix, want := range map[string]string{
|
|
"felis-api": "felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
|
|
"NAME": "NAME ROLE DESIRED PHASE PLAYERS NEWEST WORLD BACKUP",
|
|
"creative": "creative - Stopped Stopped - none",
|
|
"lobby": "lobby lobby Running Starting - none",
|
|
"survival": "survival - Running Running 2/20 3h0m ago",
|
|
} {
|
|
if f := strings.Join(lineFields(got, prefix), " "); f != want {
|
|
t.Errorf("row %q = %q, want %q\n%s", prefix, f, want, got)
|
|
}
|
|
}
|
|
if strings.Contains(got, "elsewhere") || strings.Contains(got, "other-1") {
|
|
t.Errorf("status shows what is not this install's:\n%s", got)
|
|
}
|
|
}
|
|
|
|
// Whatever is down reads as down, and the rest of the report still prints.
|
|
func TestStatusWithPartsDown(t *testing.T) {
|
|
env := statusTestEnv(t)
|
|
env.cl, env.clErr = nil, errors.New("connection refused")
|
|
env.cfg = &config.Config{}
|
|
var out bytes.Buffer
|
|
printStatus(context.Background(), env, &out)
|
|
for _, want := range []string{
|
|
"cluster: unreachable (connection refused)\n",
|
|
"\nservers (namespace minecraft): unknown while the cluster is unreachable\n",
|
|
" off-site: not configured, every backup is on this machine only\n",
|
|
" timer active, last run",
|
|
} {
|
|
if !strings.Contains(out.String(), want) {
|
|
t.Errorf("status lacks %q:\n%s", want, out.String())
|
|
}
|
|
}
|
|
|
|
env = statusTestEnv(t)
|
|
env.backups = func(context.Context) (map[string]time.Time, error) { return nil, errors.New("postgres is down") }
|
|
out.Reset()
|
|
printStatus(context.Background(), env, &out)
|
|
if f := strings.Join(lineFields(out.String(), "survival"), " "); f != "survival - Running Running 2/20 ?" {
|
|
t.Errorf("survival with PostgreSQL down = %q", f)
|
|
}
|
|
if !strings.Contains(out.String(), " world backups unknown: postgres is down\n") {
|
|
t.Errorf("status does not say why the backups are unknown:\n%s", out.String())
|
|
}
|
|
}
|
|
|
|
func TestShortImage(t *testing.T) {
|
|
for in, want := range map[string]string{
|
|
"registry.felis.svc:5000/felis/felis-api:v1.4.0": "felis-api:v1.4.0",
|
|
"docker.io/library/postgres:17@sha256:0123456789abcdef0123": "postgres:17@0123456789ab",
|
|
"felis-operator@sha256:fedcba9876543210fedcba9876543210fedcba9876543210fedcba98765432": "felis-operator@fedcba987654",
|
|
"busybox": "busybox",
|
|
} {
|
|
if got := shortImage(in); got != want {
|
|
t.Errorf("shortImage(%q) = %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A node that is not ready, a Deployment without replicas or containers, and
|
|
// lists the API server refuses each read as such.
|
|
func TestStatusClusterEdges(t *testing.T) {
|
|
env := statusTestEnv(t)
|
|
env.cfg = &config.Config{K8s: config.K8sConfig{Namespace: "games"}}
|
|
env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(
|
|
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "felis-1"}, Status: corev1.NodeStatus{
|
|
Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionFalse}, {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionTrue}}}},
|
|
&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-bare", Namespace: "felis"}},
|
|
&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "felis"}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 4}}}},
|
|
).Build()
|
|
var out bytes.Buffer
|
|
printStatus(context.Background(), env, &out)
|
|
got := out.String()
|
|
for _, want := range []string{
|
|
"node: felis-1 NotReady, k3s , , kernel \n",
|
|
"\nservers (namespace games): 0, 0 running, 0 players online\n\nbackups:\n",
|
|
} {
|
|
if !strings.Contains(got, want) {
|
|
t.Errorf("status lacks %q:\n%s", want, got)
|
|
}
|
|
}
|
|
if f := strings.Join(lineFields(got, "felis-bare"), " "); f != "felis-bare 0/1 ready - restarts 0" {
|
|
t.Errorf("row felis-bare = %q, want its one replica wanted, no image, and no pod of its own:\n%s", f, got)
|
|
}
|
|
|
|
failing := func(what client.ObjectList) {
|
|
env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(statusClusterObjects()...).
|
|
WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
|
|
if fmt.Sprintf("%T", list) == fmt.Sprintf("%T", what) {
|
|
return errors.New("forbidden")
|
|
}
|
|
return c.List(ctx, list, opts...)
|
|
}}).Build()
|
|
out.Reset()
|
|
printStatus(context.Background(), env, &out)
|
|
}
|
|
for _, tc := range []struct {
|
|
list client.ObjectList
|
|
want string
|
|
}{
|
|
{&corev1.NodeList{}, "cluster: unreachable (forbidden)\n\nproxy:"},
|
|
{&appsv1.DeploymentList{}, "\ncontrol plane (namespace felis):\n cannot list it: forbidden\n\nproxy:"},
|
|
{&corev1.PodList{}, "\ncontrol plane (namespace felis):\n cannot list it: forbidden\n\nproxy:"},
|
|
{&v1alpha1.MinecraftServerList{}, "\nservers (namespace games): cannot list them: forbidden\n\nbackups:"},
|
|
} {
|
|
failing(tc.list)
|
|
if !strings.Contains(out.String(), tc.want) {
|
|
t.Errorf("listing %T refused: status lacks %q:\n%s", tc.list, tc.want, out.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestStatusProxy(t *testing.T) {
|
|
env := statusTestEnv(t)
|
|
writeTestFile(t, filepath.Join(env.unitDir, "felis-velocity.service"), "[Unit]\n", 0o644)
|
|
env.run = fakeSystemctl("", map[string]string{"felis-velocity.service": "active"})
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
env.w.proxyAddr = ln.Addr().String()
|
|
var out bytes.Buffer
|
|
statusProxy(context.Background(), env, &out)
|
|
if want := "\nproxy: felis-velocity active, " + env.w.proxyAddr + " accepts connections\n"; out.String() != want {
|
|
t.Errorf("proxy listening: %q, want %q", out.String(), want)
|
|
}
|
|
ln.Close()
|
|
out.Reset()
|
|
statusProxy(context.Background(), env, &out)
|
|
if want := "\nproxy: felis-velocity active, " + env.w.proxyAddr + " refuses connections (dial tcp " + env.w.proxyAddr + ": "; !strings.HasPrefix(out.String(), want) {
|
|
t.Errorf("proxy gone: %q, want it to start %q", out.String(), want)
|
|
}
|
|
}
|
|
|
|
func TestStatusBackups(t *testing.T) {
|
|
env := statusTestEnv(t)
|
|
status := func() string {
|
|
var out bytes.Buffer
|
|
statusBackups(env, &out)
|
|
return out.String()
|
|
}
|
|
for _, tc := range []struct {
|
|
what, dir, want string
|
|
}{
|
|
{"no -backup-dir", "", " database: not checked (felis-watchdog.service names no -backup-dir)\n"},
|
|
{"an empty directory", t.TempDir(), " database: none in %s\n"},
|
|
{"a file where the directory should be", filepath.Join(env.w.backupDir, dbbackup.BundleName(env.now.Add(-9*time.Hour), dbbackup.LabelDaily)), " database: cannot read %s: "},
|
|
} {
|
|
env.w.backupDir = tc.dir
|
|
want := tc.want
|
|
if strings.Contains(want, "%s") {
|
|
want = fmt.Sprintf(want, tc.dir)
|
|
}
|
|
if got := status(); !strings.Contains(got, want) {
|
|
t.Errorf("%s: %q, want %q", tc.what, got, want)
|
|
}
|
|
}
|
|
|
|
for _, tc := range []struct {
|
|
what string
|
|
status *offsite.Status
|
|
raw string
|
|
want string
|
|
}{
|
|
{"never synced", nil, "", " off-site: never synced\n"},
|
|
{"never succeeded", &offsite.Status{LastAttempt: env.now.Add(-time.Hour), LastError: "403 Forbidden"}, "", " off-site: never succeeded; last attempt 1h0m ago: 403 Forbidden\n"},
|
|
{"the last attempt failed", &offsite.Status{LastAttempt: env.now.Add(-30 * time.Minute), LastSuccess: env.now.Add(-26 * time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
|
|
" off-site: last good sync 26h0m ago to felis-dr; the last attempt, 30m ago, failed: timeout\n"},
|
|
{"an error an earlier attempt left", &offsite.Status{LastAttempt: env.now.Add(-2 * time.Hour), LastSuccess: env.now.Add(-time.Hour), LastError: "timeout", Bucket: "felis-dr"}, "",
|
|
" off-site: last good sync 1h0m ago to felis-dr\n"},
|
|
{"an unreadable record", nil, "{", " off-site: unexpected end of JSON input\n"},
|
|
} {
|
|
os.Remove(env.w.offsiteStatus)
|
|
switch {
|
|
case tc.status != nil:
|
|
if err := offsite.WriteStatus(env.w.offsiteStatus, *tc.status); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case tc.raw != "":
|
|
writeTestFile(t, env.w.offsiteStatus, tc.raw, 0o600)
|
|
}
|
|
if got := status(); !strings.HasSuffix(got, tc.want) {
|
|
t.Errorf("%s: %q, want it to end %q", tc.what, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestStatusHost(t *testing.T) {
|
|
env := statusTestEnv(t)
|
|
dir := t.TempDir()
|
|
env.w.diskPaths = "/nonexistent-felis-status-test," + dir + "," + dir
|
|
env.meminfo = filepath.Join(dir, "meminfo")
|
|
var out bytes.Buffer
|
|
statusHost(env, &out)
|
|
lines := strings.Split(strings.TrimSuffix(out.String(), "\n"), "\n")
|
|
if len(lines) != 3 || lines[0] != "" || lines[1] != "host:" || !strings.HasPrefix(lines[2], " disk "+dir+": ") || !strings.HasSuffix(lines[2], "% free)") {
|
|
t.Errorf("host: %q, want one line for %s (a path on a disk already shown, and one that is not there, print none) and no memory line", lines, dir)
|
|
}
|
|
|
|
writeTestFile(t, env.meminfo, "MemTotal: 4096 kB\ngarbage\nMemAvailable: 1024 kB\n", 0o644)
|
|
if total, avail, ok := readMeminfo(env.meminfo); !ok || total != 4096*1024 || avail != 1024*1024 {
|
|
t.Errorf("readMeminfo = %d, %d, %v", total, avail, ok)
|
|
}
|
|
writeTestFile(t, env.meminfo, "MemAvailable: 1024 kB\n", 0o644)
|
|
if _, _, ok := readMeminfo(env.meminfo); ok {
|
|
t.Error("readMeminfo without MemTotal: ok")
|
|
}
|
|
}
|
|
|
|
func TestStatusWatchdog(t *testing.T) {
|
|
env := statusTestEnv(t)
|
|
status := func() string {
|
|
var out bytes.Buffer
|
|
statusWatchdog(context.Background(), env, &out)
|
|
return out.String()
|
|
}
|
|
run := env.run
|
|
env.run = func(ctx context.Context, name string, args ...string) ([]byte, error) {
|
|
if len(args) > 0 && args[0] == "show" {
|
|
return []byte("Result=success\nExecMainExitTimestamp=\n"), nil
|
|
}
|
|
return run(ctx, name, args...)
|
|
}
|
|
if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: map[string]*watchdog.Alert{
|
|
"proxy": {Finding: watchdog.Finding{Key: "proxy", Severity: watchdog.Critical}, FirstSeen: env.now.Add(-time.Hour), ClearedAt: env.now.Add(-time.Minute)},
|
|
}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, want := status(), "\nwatchdog:\n timer active\n alerts: none open\n"; got != want {
|
|
t.Errorf("a watchdog that has not run and has nothing open: %q, want %q", got, want)
|
|
}
|
|
|
|
writeTestFile(t, env.w.statePath, "{", 0o600)
|
|
if got := status(); !strings.Contains(got, "\n alerts: unknown (") {
|
|
t.Errorf("an unreadable state: %q", got)
|
|
}
|
|
|
|
if err := os.Remove(filepath.Join(env.unitDir, "felis-watchdog.timer")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, want := status(), "\nwatchdog:\n not installed: nothing checks this host\n"; got != want {
|
|
t.Errorf("no watchdog timer: %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
// Rows print by name in whatever order the API server lists them, a server
|
|
// the operator has not reconciled yet reads as dashes, and open alerts print
|
|
// by key in whatever order the state's map yields them.
|
|
func TestStatusOrder(t *testing.T) {
|
|
env := statusTestEnv(t)
|
|
objs := append(statusClusterObjects(),
|
|
&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "felis-operator", Namespace: "felis"}},
|
|
&v1alpha1.MinecraftServer{ObjectMeta: metav1.ObjectMeta{Name: "fresh", Namespace: "minecraft"}})
|
|
env.cl = fake.NewClientBuilder().WithScheme(newSystemServerScheme(t)).WithObjects(objs...).
|
|
WithInterceptorFuncs(interceptor.Funcs{List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
|
|
// The fake lists by name; the other way round, then.
|
|
if err := c.List(ctx, list, opts...); err != nil {
|
|
return err
|
|
}
|
|
items, err := meta.ExtractList(list)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
slices.Reverse(items)
|
|
return meta.SetList(list, items)
|
|
}}).Build()
|
|
var out bytes.Buffer
|
|
printStatus(context.Background(), env, &out)
|
|
var rows []string
|
|
for _, l := range strings.Split(out.String(), "\n") {
|
|
if f := strings.Fields(l); len(f) > 0 && (strings.HasPrefix(f[0], "felis-") || slices.Contains([]string{"creative", "fresh", "lobby", "survival"}, f[0])) {
|
|
rows = append(rows, strings.Join(f, " "))
|
|
}
|
|
}
|
|
want := []string{
|
|
"felis-api 1/1 ready felis-api:v1.4.0@0123456789ab restarts 2",
|
|
"felis-operator 0/1 ready - restarts 0",
|
|
"creative - Stopped Stopped - none",
|
|
"fresh - - - - none",
|
|
"lobby lobby Running Starting - none",
|
|
"survival - Running Running 2/20 3h0m ago",
|
|
}
|
|
if !slices.Equal(rows, want) {
|
|
t.Errorf("rows %q, want %q:\n%s", rows, want, out.String())
|
|
}
|
|
|
|
alerts := map[string]*watchdog.Alert{}
|
|
for i, key := range []string{"a", "b", "c", "d"} {
|
|
alerts[key] = &watchdog.Alert{Finding: watchdog.Finding{Key: key, Severity: watchdog.Warning}, FirstSeen: env.now.Add(-time.Duration(i+1) * time.Minute)}
|
|
}
|
|
if err := watchdog.SaveState(env.w.statePath, &watchdog.State{Alerts: alerts}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wantAlerts := " alerts: 4 open (sudo felis doctor says where to look)\n" +
|
|
" a (warning, seen 1m ago, not mailed yet)\n b (warning, seen 2m ago, not mailed yet)\n" +
|
|
" c (warning, seen 3m ago, not mailed yet)\n d (warning, seen 4m ago, not mailed yet)\n"
|
|
// A map starts its walk at random: fifty walks all in order by chance
|
|
// is out of the question.
|
|
for range 50 {
|
|
out.Reset()
|
|
statusWatchdog(context.Background(), env, &out)
|
|
if !strings.HasSuffix(out.String(), wantAlerts) {
|
|
t.Fatalf("alerts %q, want them to end %q", out.String(), wantAlerts)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A selector the API server would have refused picks no pods.
|
|
func TestPodRestartsBadSelector(t *testing.T) {
|
|
pods := []corev1.Pod{{Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{RestartCount: 3}}}}}
|
|
if n := podRestarts(&metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "app", Operator: "Near"}}}, pods); n != 0 {
|
|
t.Errorf("podRestarts with a bad selector = %d, want 0", n)
|
|
}
|
|
if n := podRestarts(&metav1.LabelSelector{}, pods); n != 3 {
|
|
t.Errorf("podRestarts with a selector that picks all = %d, want 3", n)
|
|
}
|
|
}
|