Felis/cmd/felis/domain_test.go

package main

import (
	"bytes"
	"context"
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"math/big"
	"net"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"testing"
	"time"

	"felis.lolicon.best/internal/apis/felis/v1alpha1"
	"felis.lolicon.best/internal/naming"
	"felis.lolicon.best/internal/platform"

	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"
	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)

// installerTOML is felis.toml as deploy/bootstrap.sh write_felis_toml renders it,
// comments included: the domain move has to leave all of it but three values alone.
func installerTOML(root, dbHost string) string {
	return `# Generated by deploy/bootstrap.sh; rerun the installer to regenerate. Hand edits are
# overwritten, except [smtp], [[auth_source]], [offsite], and the operator-owned
# [registry] / [archive] overrides, which carry forward.
[server]
listen = "0.0.0.0:8080"
root_domain = "` + root + `"

[database]
url = "postgres://felis:pw@` + dbHost + `:5432/felis?sslmode=disable"

[k8s]
namespace = "minecraft"
egress_mode = "nodeport"

[velocity]
# The two always-on system servers that felis setup provisions.
login_image = "felis/limbo:1"
lobby_image = "felis/lobby:1"
game_port = 25565

[registry]
url = "registry.felis.svc:5000"
build_namespace = "felis-build"

[archive]
store = "tarLocal"
local_path = "/var/lib/felis/archives"

[auth]
admin_hostname = "op.console.` + root + `"
panel_hostname = "console.` + root + `"
access_jwt_aud = "aud123"

# Third-party Yggdrasil sources federated by the hasJoined multiplexer.
[[auth_source]]
tag = "littleskin"
prefix = "LS"
url = "https://littleskin.cn/api/yggdrasil/sessionserver/session/minecraft/hasJoined"
`
}

const linkPropsBody = `# Generated by deploy/bootstrap.sh — do not edit by hand; rerun the installer.
api-base-url=http://10.43.0.9:8081
service-token=TOKEN-NOT-TO-TOUCH
root-domain=old.example
panel-hostname=console.old.example
admin-hostname=op.console.old.example
login-server=login
lobby-server=lobby
`

var oldNames = domainNames{root: "old.example", panel: "console.old.example", admin: "op.console.old.example"}
var newNames = domainNames{root: "new.example", panel: "console.new.example", admin: "op.console.new.example"}

// domainRig models the host: the files, the cluster, and a felis-api, proxy and
// operator that pick up config the way the real ones do — the api serves what it
// read at its last restart, the proxy runs since its last restart, and the login
// pod carries the env its MinecraftServer had when it was last rolled.
type domainRig struct {
	h      domainHost
	cl     client.Client
	out    *bytes.Buffer
	dir    string
	events []string

	served      domainNames
	servedCert  *x509.Certificate
	proxySince  time.Time
	proxyLoaded bool
	unresolved  map[string]bool
	// The fake operator: the CR env the login pod was last rolled to, and how
	// many looks at the pod since the CR moved on.
	rolledTo string
	pending  int
}

func (rig *domainRig) path(name string) string { return filepath.Join(rig.dir, name) }

func newDomainRig(t *testing.T) *domainRig {
	t.Helper()
	rig := &domainRig{out: &bytes.Buffer{}, dir: t.TempDir(), proxyLoaded: true, unresolved: map[string]bool{}}
	writeTestFile(t, rig.path("felis.host.toml"), installerTOML("old.example", "127.0.0.1"), 0o600)
	writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
	if err := os.Symlink(rig.path("felis.host.toml"), rig.path("felis.toml")); err != nil {
		t.Fatal(err)
	}
	certPEM, keyPEM, err := issuePanelCert(oldNames, []net.IP{net.ParseIP("10.211.55.6")}, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
	writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
	// The proxy started before its config was last written, which is how it
	// stands after an install.
	rig.proxySince = time.Now().Add(-time.Hour)

	pod := []byte(installerTOML("old.example", "10.211.55.6"))
	login, err := loginSystemServer("felis/limbo:1", "minecraft", platform.InternalAPIBaseURL("felis"), oldNames.root, oldNames.panel)
	if err != nil {
		t.Fatal(err)
	}
	lobby, err := lobbySystemServer("felis/lobby:1", "minecraft")
	if err != nil {
		t.Fatal(err)
	}
	loginPod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"},
		Spec:       corev1.PodSpec{Containers: []corev1.Container{{Name: "minecraft", Env: podEnv(login.Spec.Env)}}},
		Status:     corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}},
	}
	rig.rolledTo = envKey(login.Spec.Env)
	rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
		Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
			if key.Name == naming.SystemLoginServer+"-0" {
				rig.operatorTick(t, c)
			}
			return c.Get(ctx, key, obj, opts...)
		},
	}).WithObjects(
		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.ConfigSecretName},
			Data: map[string][]byte{platform.ConfigSecretKey: pod}},
		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: platform.ConfigSecretName},
			Data: map[string][]byte{platform.ConfigSecretKey: pod}},
		&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.APITLSSecretName},
			Type: corev1.SecretTypeTLS, Data: map[string][]byte{corev1.TLSCertKey: certPEM, corev1.TLSPrivateKeyKey: keyPEM}},
		login, lobby, loginPod,
	).Build()
	rig.served = oldNames
	rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))

	rig.h = domainHost{
		paths: domainPaths{
			hostTOML: rig.path("felis.host.toml"), podTOML: rig.path("felis.pod.toml"), defaultTOML: rig.path("felis.toml"),
			cert: rig.path("panel-tls.crt"), key: rig.path("panel-tls.key"),
			linkProps: rig.path("felis-link.properties"), tunnelConfig: rig.path("cloudflared.yml"),
		},
		cl:        rig.cl,
		controlNS: "felis",
		rollAPI: func(ctx context.Context) error {
			rig.events = append(rig.events, "roll-api")
			rig.restartAPI(t)
			return nil
		},
		restartUnit: func(_ context.Context, unit string) error {
			rig.events = append(rig.events, "restart "+unit)
			rig.proxySince = time.Now().Add(time.Second)
			return nil
		},
		unitState: func(context.Context, string) (unitStatus, error) {
			return unitStatus{loaded: rig.proxyLoaded, active: rig.proxyLoaded, since: rig.proxySince}, nil
		},
		liveAPI: func(context.Context, string) (liveAPIView, error) {
			return liveAPIView{names: rig.served, cert: rig.servedCert}, nil
		},
		lookupHost: func(_ context.Context, host string) ([]string, error) {
			if rig.unresolved[host] {
				return nil, errors.New("no such host")
			}
			return []string{"10.211.55.6"}, nil
		},
		passkeys:  func(context.Context) (int, int, error) { return 3, 2, nil },
		now:       time.Now,
		out:       rig.out,
		loginWait: 50 * time.Millisecond,
		pollEvery: time.Millisecond,
	}
	return rig
}

func podEnv(env []v1alpha1.EnvVar) []corev1.EnvVar {
	out := make([]corev1.EnvVar, len(env))
	for i, e := range env {
		out[i] = corev1.EnvVar{Name: e.Name, Value: e.Value}
	}
	return out
}

// restartAPI makes the fake felis-api load the config and certificate its
// Secrets hold now.
func (rig *domainRig) restartAPI(t *testing.T) {
	t.Helper()
	var cfg, tlsSec corev1.Secret
	ctx := context.Background()
	if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.ConfigSecretName}, &cfg); err != nil {
		t.Fatal(err)
	}
	if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.APITLSSecretName}, &tlsSec); err != nil {
		t.Fatal(err)
	}
	names, err := tomlDomainNames(cfg.Data[platform.ConfigSecretKey])
	if err != nil {
		t.Fatal(err)
	}
	rig.served = names
	rig.servedCert, err = x509.ParseCertificate(mustCertDER(tlsSec.Data[corev1.TLSCertKey]))
	if err != nil {
		t.Fatal(err)
	}
}

// rollLoginPod is the operator restarting the login pod onto its CR's env.
// operatorTick is the operator as the login pod is watched: once the CR's env
// changes it takes operatorLag looks at the pod before the restarted pod
// carries the new env, the way a real rollout lags the CR.
func (rig *domainRig) operatorTick(t *testing.T, c client.Client) {
	t.Helper()
	ctx := context.Background()
	var ms v1alpha1.MinecraftServer
	if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
		t.Fatal(err)
	}
	if envKey(ms.Spec.Env) == rig.rolledTo {
		return
	}
	if rig.pending++; rig.pending < operatorLag {
		return
	}
	var pod corev1.Pod
	if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"}, &pod); err != nil {
		t.Fatal(err)
	}
	pod.Spec.Containers[0].Env = podEnv(ms.Spec.Env)
	if err := c.Update(ctx, &pod); err != nil {
		t.Fatal(err)
	}
	rig.rolledTo, rig.pending = envKey(ms.Spec.Env), 0
}

const operatorLag = 3

func envKey(env []v1alpha1.EnvVar) string {
	var b strings.Builder
	for _, e := range env {
		b.WriteString(e.Name + "=" + e.Value + "\n")
	}
	return b.String()
}

func (rig *domainRig) read(t *testing.T, name string) string {
	t.Helper()
	b, err := os.ReadFile(rig.path(name))
	if err != nil {
		t.Fatal(err)
	}
	return string(b)
}

func (rig *domainRig) secret(t *testing.T, ns, name string) map[string][]byte {
	t.Helper()
	var s corev1.Secret
	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
		t.Fatal(err)
	}
	return s.Data
}

func (rig *domainRig) crEnv(t *testing.T, name string) map[string]string {
	t.Helper()
	var ms v1alpha1.MinecraftServer
	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
		t.Fatal(err)
	}
	env := map[string]string{}
	for _, e := range ms.Spec.Env {
		env[e.Name] = e.Value
	}
	return env
}

// snapshot is every byte `set` may touch, for proving a refused or dry run
// touched none of it.
func (rig *domainRig) snapshot(t *testing.T) string {
	t.Helper()
	var b strings.Builder
	entries, _ := os.ReadDir(rig.dir)
	for _, e := range entries {
		b.WriteString(e.Name() + "\n" + rig.read(t, e.Name()) + "\n")
	}
	var lines []string
	for _, s := range []struct{ ns, name string }{{"felis", platform.ConfigSecretName}, {"minecraft", platform.ConfigSecretName}, {"felis", platform.APITLSSecretName}} {
		for k, v := range rig.secret(t, s.ns, s.name) {
			lines = append(lines, s.ns+"/"+s.name+"/"+k+"\n"+string(v))
		}
	}
	for k, v := range rig.crEnv(t, naming.SystemLoginServer) {
		lines = append(lines, "env "+k+"="+v)
	}
	sort.Strings(lines)
	b.WriteString(strings.Join(lines, "\n"))
	return b.String()
}

func TestNormalizeRootDomain(t *testing.T) {
	for in, want := range map[string]string{
		"Example.COM.":          "example.com",
		"  mc.example.org ":     "mc.example.org",
		"10.211.55.6.nip.io":    "10.211.55.6.nip.io",
		"xn--bcher-kva.example": "xn--bcher-kva.example",
	} {
		got, err := normalizeRootDomain(in)
		if err != nil || got != want {
			t.Errorf("normalizeRootDomain(%q) = %q, %v; want %q", in, got, err, want)
		}
	}
	for _, in := range []string{"", "https://example.com", "example.com:443", "example.com/x", "10.0.0.1", "::1",
		"localhost", "a_b.example", "-a.example", "a-.example", strings.Repeat("a", 64) + ".example",
		strings.Repeat("abcdefghi.", 25) + "example"} {
		if got, err := normalizeRootDomain(in); err == nil {
			t.Errorf("normalizeRootDomain(%q) = %q, want an error", in, got)
		}
	}
}

func TestPlanDomainChangeMovesDefaultsAndKeepsHandSetNames(t *testing.T) {
	p := planDomainChange(oldNames, "new.example")
	if p.to != newNames || p.customPanel || p.customAdmin {
		t.Fatalf("defaults: %+v", p)
	}
	p = planDomainChange(domainNames{root: "old.example", panel: "play.corp.net", admin: "op.console.old.example"}, "new.example")
	if p.to.panel != "play.corp.net" || !p.customPanel || p.to.admin != "op.console.new.example" || p.customAdmin {
		t.Fatalf("hand-set panel: %+v", p)
	}
	p = planDomainChange(domainNames{root: "old.example", panel: "console.old.example", admin: "admin.corp.net"}, "new.example")
	if p.to.admin != "admin.corp.net" || !p.customAdmin || p.to.panel != "console.new.example" {
		t.Fatalf("hand-set admin: %+v", p)
	}
}

func TestEditTOMLStringsChangesOnlyTheDomainLines(t *testing.T) {
	orig := installerTOML("old.example", "127.0.0.1")
	out, err := editTOMLStrings([]byte(orig), domainTOMLEdits(newNames))
	if err != nil {
		t.Fatal(err)
	}
	a, b := strings.Split(orig, "\n"), strings.Split(string(out), "\n")
	if len(a) != len(b) {
		t.Fatalf("line count %d → %d:\n%s", len(a), len(b), out)
	}
	changed := map[string]string{}
	for i := range a {
		if a[i] != b[i] {
			changed[a[i]] = b[i]
		}
	}
	want := map[string]string{
		`root_domain = "old.example"`:               `root_domain = "new.example"`,
		`admin_hostname = "op.console.old.example"`: `admin_hostname = "op.console.new.example"`,
		`panel_hostname = "console.old.example"`:    `panel_hostname = "console.new.example"`,
	}
	if len(changed) != len(want) {
		t.Fatalf("changed lines %v, want %v", changed, want)
	}
	for k, v := range want {
		if changed[k] != v {
			t.Errorf("%q → %q, want %q", k, changed[k], v)
		}
	}
}

func TestEditTOMLStringsAddsMissingKeysInTheirTable(t *testing.T) {
	in := "[server]\nroot_domain = \"old.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\n\n[smtp]\nhost = \"relay\"\n"
	out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames))
	if err != nil {
		t.Fatal(err)
	}
	want := "[server]\nroot_domain = \"new.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\npanel_hostname = \"console.new.example\"\nadmin_hostname = \"op.console.new.example\"\n\n[smtp]\nhost = \"relay\"\n"
	if string(out) != want {
		t.Fatalf("got:\n%s\nwant:\n%s", out, want)
	}

	out, err = editTOMLStrings([]byte("[server]\nroot_domain = \"old.example\"\n\n[[auth_source]]\ntag = \"ls\"\n"), domainTOMLEdits(newNames))
	if err != nil {
		t.Fatal(err)
	}
	got, err := tomlDomainNames(out)
	if err != nil || got != newNames || !strings.Contains(string(out), "[[auth_source]]\ntag = \"ls\"\n") {
		t.Fatalf("no [auth] table: %v %+v\n%s", err, got, out)
	}
}

func TestEditTOMLStringsRefusesWhatItCannotEditExactly(t *testing.T) {
	for name, in := range map[string]string{
		"multi-line value": "[server]\nroot_domain = \"\"\"\nold.example\"\"\"\n[auth]\n",
		// The key's line sits inside another value; the real key is absent.
		"key inside a string": "[server]\nmotd = \"\"\"\nroot_domain = \"old.example\"\n\"\"\"\n[auth]\n",
		"quoted header":       "[server]\nroot_domain = \"old.example\"\n[\"auth\"]\npanel_hostname = \"console.old.example\"\n",
		"dotted key":          "server.root_domain = \"old.example\"\n",
		"inline table":        "server = { root_domain = \"old.example\" }\n",
	} {
		if out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames)); err == nil {
			t.Errorf("%s: edited instead of refusing:\n%s", name, out)
		}
	}
}

// caSignedCert is an operator's certificate from their own CA; it names
// localhost too, so only the issuer tells it apart from the installer's.
func caSignedCert(t *testing.T, hosts ...string) (certPEM, keyPEM []byte) {
	t.Helper()
	caKey, _ := rsa.GenerateKey(rand.Reader, 2048)
	ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Corp CA"}, IsCA: true,
		BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
	caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey)
	if err != nil {
		t.Fatal(err)
	}
	caCert, _ := x509.ParseCertificate(caDER)
	key, _ := rsa.GenerateKey(rand.Reader, 2048)
	leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: hosts[0]},
		DNSNames: append(hosts, "localhost"), IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
		NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
	der, err := x509.CreateCertificate(rand.Reader, leaf, caCert, &key.PublicKey, caKey)
	if err != nil {
		t.Fatal(err)
	}
	pk, _ := x509.MarshalPKCS8PrivateKey(key)
	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pk})
}

func TestFelisIssuedCert(t *testing.T) {
	mine, _, err := issuePanelCert(oldNames, nil, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	c, _ := x509.ParseCertificate(mustCertDER(mine))
	if !felisIssuedCert(c) {
		t.Error("the installer's kind of certificate is not recognised as Felis-issued")
	}
	theirs, _ := caSignedCert(t, "console.old.example")
	c, _ = x509.ParseCertificate(mustCertDER(theirs))
	if felisIssuedCert(c) {
		t.Error("a CA-signed certificate is taken for Felis-issued")
	}

	// Self-signed but without one of the installer's localhost names: someone
	// else's.
	key, _ := rsa.GenerateKey(rand.Reader, 2048)
	for name, self := range map[string]*x509.Certificate{
		"no localhost": {DNSNames: []string{"console.old.example"}, IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}},
		"no 127.0.0.1": {DNSNames: []string{"console.old.example", "localhost"}},
	} {
		self.SerialNumber, self.Subject = big.NewInt(3), pkix.Name{CommonName: "x"}
		self.NotBefore, self.NotAfter = time.Now().Add(-time.Hour), time.Now().Add(time.Hour)
		der, _ := x509.CreateCertificate(rand.Reader, self, self, &key.PublicKey, key)
		c, _ = x509.ParseCertificate(der)
		if felisIssuedCert(c) {
			t.Errorf("%s: a self-signed certificate is taken for Felis-issued", name)
		}
	}
}

func TestIssuePanelCertIsTheInstallersShape(t *testing.T) {
	now := time.Now()
	certPEM, keyPEM, err := issuePanelCert(newNames, []net.IP{net.ParseIP("10.211.55.6"), net.IPv4(127, 0, 0, 1)}, now)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil {
		t.Fatalf("key does not match the certificate: %v", err)
	}
	if b, _ := pem.Decode(keyPEM); b == nil || b.Type != "PRIVATE KEY" {
		t.Fatalf("key is not PKCS#8 PEM like openssl writes")
	}
	c, _ := x509.ParseCertificate(mustCertDER(certPEM))
	if !certCovers(c, newNames.panel, newNames.admin, "localhost") || !felisIssuedCert(c) {
		t.Fatalf("names %v", c.DNSNames)
	}
	if len(c.IPAddresses) != 2 || !c.IPAddresses[1].Equal(net.ParseIP("10.211.55.6")) {
		t.Fatalf("addresses %v, want 127.0.0.1 and the node address once each", c.IPAddresses)
	}
	if c.Subject.CommonName != newNames.admin || c.NotAfter.Sub(now) < 824*24*time.Hour || c.NotAfter.Sub(now) > 826*24*time.Hour {
		t.Fatalf("CN %q, valid until %s", c.Subject.CommonName, c.NotAfter)
	}
	if len(c.ExtKeyUsage) != 1 || c.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth || c.IsCA {
		t.Fatalf("usage %v, CA %v", c.ExtKeyUsage, c.IsCA)
	}
}

func TestDomainSetMovesEverySurface(t *testing.T) {
	rig := newDomainRig(t)
	hostBefore := rig.read(t, "felis.host.toml")
	code, err := rig.h.set(context.Background(), "New.Example", true)
	if err != nil || code != 0 {
		t.Fatalf("set = %d, %v\n%s", code, err, rig.out)
	}

	// The configs: the three values moved, everything else — comments, the
	// database host of each copy, the Access audience — is as it was.
	host := rig.read(t, "felis.host.toml")
	if want := strings.NewReplacer("old.example", "new.example").Replace(hostBefore); host != want {
		t.Fatalf("host toml:\n%s", host)
	}
	pod := rig.read(t, "felis.pod.toml")
	if got, _ := tomlDomainNames([]byte(pod)); got != newNames || !strings.Contains(pod, "@10.211.55.6:5432") {
		t.Fatalf("pod toml:\n%s", pod)
	}
	if link, err := os.Readlink(rig.path("felis.toml")); err != nil || link != rig.path("felis.host.toml") {
		t.Fatalf("felis.toml is no longer the link to the host copy: %q %v", link, err)
	}
	if st, _ := os.Stat(rig.path("felis.host.toml")); st.Mode().Perm() != 0o600 {
		t.Fatalf("host toml mode %v", st.Mode().Perm())
	}

	// The certificate: reissued for the new names, the node address kept, the old
	// pair beside it.
	c, err := readCertFile(rig.path("panel-tls.crt"))
	if err != nil || !certCovers(c, newNames.panel, newNames.admin) || !felisIssuedCert(c) {
		t.Fatalf("certificate: %v %v", err, c.DNSNames)
	}
	if !c.IPAddresses[len(c.IPAddresses)-1].Equal(net.ParseIP("10.211.55.6")) {
		t.Fatalf("addresses %v", c.IPAddresses)
	}
	if _, err := tls.LoadX509KeyPair(rig.path("panel-tls.crt"), rig.path("panel-tls.key")); err != nil {
		t.Fatalf("new pair: %v", err)
	}
	if st, _ := os.Stat(rig.path("panel-tls.key")); st.Mode().Perm() != 0o600 {
		t.Fatalf("key mode %v", st.Mode().Perm())
	}
	backups, _ := filepath.Glob(rig.path("panel-tls.*.pre-domain-*"))
	if len(backups) != 2 {
		t.Fatalf("old pair kept as %v", backups)
	}
	for _, b := range backups {
		if st, _ := os.Stat(b); strings.Contains(b, ".key.") && st.Mode().Perm() != 0o600 {
			t.Fatalf("kept key %s has mode %v", b, st.Mode().Perm())
		}
		old, _ := os.ReadFile(b)
		if strings.Contains(b, ".crt.") {
			oc, _ := x509.ParseCertificate(mustCertDER(old))
			if oc == nil || !certCovers(oc, oldNames.panel) {
				t.Fatalf("kept certificate is not the old one")
			}
		}
	}

	// The Secrets carry the files.
	for _, ns := range []string{"felis", "minecraft"} {
		if got := rig.secret(t, ns, platform.ConfigSecretName)[platform.ConfigSecretKey]; string(got) != pod {
			t.Fatalf("%s/felis-config is not felis.pod.toml", ns)
		}
	}
	tlsData := rig.secret(t, "felis", platform.APITLSSecretName)
	if string(tlsData[corev1.TLSCertKey]) != rig.read(t, "panel-tls.crt") || string(tlsData[corev1.TLSPrivateKeyKey]) != rig.read(t, "panel-tls.key") {
		t.Fatal("felis-api-tls does not hold the new pair")
	}

	// The login gate's env moved and nothing else did.
	env := rig.crEnv(t, naming.SystemLoginServer)
	if env[envRootDomain] != newNames.root || env[envPanelHostname] != newNames.panel || env[envAPIBaseURL] != platform.InternalAPIBaseURL("felis") {
		t.Fatalf("login env %v", env)
	}

	// The proxy's file: the three keys moved, its token and mode did not.
	props := rig.read(t, "felis-link.properties")
	if want := strings.NewReplacer("old.example", "new.example").Replace(linkPropsBody); props != want {
		t.Fatalf("felis-link.properties:\n%s", props)
	}
	if st, _ := os.Stat(rig.path("felis-link.properties")); st.Mode().Perm() != 0o640 {
		t.Fatalf("felis-link.properties mode %v", st.Mode().Perm())
	}

	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
		t.Fatalf("events %v", rig.events)
	}
	if !strings.Contains(rig.out.String(), "Every surface is on new.example.") {
		t.Fatalf("the closing check did not pass:\n%s", rig.out)
	}
}

func TestDomainSetWithoutYesChangesNothing(t *testing.T) {
	rig := newDomainRig(t)
	before := rig.snapshot(t)
	code, err := rig.h.set(context.Background(), "new.example", false)
	if err != nil || code != 0 {
		t.Fatalf("set = %d, %v", code, err)
	}
	if rig.snapshot(t) != before || len(rig.events) != 0 {
		t.Fatalf("a dry run changed something (events %v)", rig.events)
	}
	out := rig.out.String()
	for _, want := range []string{
		"console.old.example → console.new.example",
		"3 passkey(s) of 2 user(s) are bound to console.old.example",
		"does not cover op.console.new.example",
		"No [smtp] relay is configured",
		"sudo felis domain set -yes new.example",
	} {
		if !strings.Contains(out, want) {
			t.Errorf("plan lacks %q:\n%s", want, out)
		}
	}
}

func TestDomainSetKeepsAHandSetPanelHostname(t *testing.T) {
	rig := newDomainRig(t)
	for _, f := range []string{"felis.host.toml", "felis.pod.toml"} {
		writeTestFile(t, rig.path(f), strings.Replace(rig.read(t, f), `panel_hostname = "console.old.example"`, `panel_hostname = "play.corp.net"`, 1), 0o600)
	}
	code, err := rig.h.set(context.Background(), "new.example", true)
	if err != nil {
		t.Fatalf("set: %v\n%s", err, rig.out)
	}
	got, _ := tomlDomainNames([]byte(rig.read(t, "felis.host.toml")))
	if got != (domainNames{root: "new.example", panel: "play.corp.net", admin: "op.console.new.example"}) {
		t.Fatalf("names %+v", got)
	}
	c, _ := readCertFile(rig.path("panel-tls.crt"))
	if !certCovers(c, "play.corp.net", "op.console.new.example") {
		t.Fatalf("certificate names %v", c.DNSNames)
	}
	if env := rig.crEnv(t, naming.SystemLoginServer); env[envPanelHostname] != "play.corp.net" {
		t.Fatalf("login env %v", env)
	}
	if code != 0 || !strings.Contains(rig.out.String(), "play.corp.net (set by hand, kept") {
		t.Fatalf("code %d:\n%s", code, rig.out)
	}
}

func TestDomainSetRefusesAnOperatorCertificateForOtherNames(t *testing.T) {
	rig := newDomainRig(t)
	certPEM, keyPEM := caSignedCert(t, "console.old.example", "op.console.old.example")
	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
	before := rig.snapshot(t)
	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "not issued by Felis") {
		t.Fatalf("err = %v", err)
	}
	if rig.snapshot(t) != before || len(rig.events) != 0 {
		t.Fatal("a refused move changed something")
	}

	// The operator's certificate for the new names is kept as it is.
	certPEM, keyPEM = caSignedCert(t, "console.new.example", "op.console.new.example")
	writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
	writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
		t.Fatalf("set: %v", err)
	}
	if rig.read(t, "panel-tls.crt") != string(certPEM) {
		t.Fatal("the operator's certificate was replaced")
	}
}

func TestDomainSetRefusesAConfigItCannotEdit(t *testing.T) {
	rig := newDomainRig(t)
	writeTestFile(t, rig.path("felis.pod.toml"), strings.Replace(rig.read(t, "felis.pod.toml"), "[auth]", "[\"auth\"]", 1), 0o600)
	before := rig.snapshot(t)
	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "by hand") {
		t.Fatalf("err = %v", err)
	}
	if rig.snapshot(t) != before || len(rig.events) != 0 {
		t.Fatal("a refused move changed something")
	}
}

func TestDomainSetAgainOnlyConvergesWhatIsBehind(t *testing.T) {
	rig := newDomainRig(t)
	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
		t.Fatal(err)
	}
	rig.events, rig.out = nil, &bytes.Buffer{}
	rig.h.out = rig.out
	before := rig.snapshot(t)
	code, err := rig.h.set(context.Background(), "new.example", true)
	if err != nil || code != 0 {
		t.Fatalf("second set = %d, %v\n%s", code, err, rig.out)
	}
	if len(rig.events) != 0 || rig.snapshot(t) != before {
		t.Fatalf("a converged install was touched again: %v\n%s", rig.events, rig.out)
	}

	// A proxy that was not restarted after the move is restarted by a re-run, and
	// an api still on the old config is rolled.
	rig.proxySince = time.Now().Add(-time.Hour)
	rig.served = oldNames
	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
		t.Fatal(err)
	}
	if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
		t.Fatalf("events %v", rig.events)
	}

	// An api on the new names that still presents the old certificate is rolled.
	rig.events = nil
	oldCert, _, _ := issuePanelCert(oldNames, nil, time.Now())
	rig.servedCert, _ = x509.ParseCertificate(mustCertDER(oldCert))
	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
		t.Fatal(err)
	}
	if strings.Join(rig.events, ",") != "roll-api" {
		t.Fatalf("events %v", rig.events)
	}
}

func TestDomainSetRefusesALoginServerItDoesNotOwn(t *testing.T) {
	rig := newDomainRig(t)
	var ms v1alpha1.MinecraftServer
	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
		t.Fatal(err)
	}
	delete(ms.Labels, v1alpha1.LabelSystemRole)
	if err := rig.cl.Update(context.Background(), &ms); err != nil {
		t.Fatal(err)
	}
	if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "system role") {
		t.Fatalf("err = %v", err)
	}
	if env := rig.crEnv(t, naming.SystemLoginServer); env[envRootDomain] != oldNames.root {
		t.Fatalf("a server not marked as the login gate was changed: %v", env)
	}
}

func TestDomainCheckNamesTheSurfaceThatIsBehind(t *testing.T) {
	cases := []struct {
		surface string
		breakIt func(t *testing.T, rig *domainRig)
	}{
		{"felis.pod.toml", func(t *testing.T, rig *domainRig) {
			writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
		}},
		{"Secret minecraft/felis-config", func(t *testing.T, rig *domainRig) {
			rig.putSecret(t, "minecraft", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
		}},
		{"Secret felis/felis-config", func(t *testing.T, rig *domainRig) {
			rig.putSecret(t, "felis", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
		}},
		{"panel certificate", func(t *testing.T, rig *domainRig) {
			// The Secret follows the file, so only the certificate's names are wrong.
			certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
			writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
			rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
		}},
		{"Secret felis/felis-api-tls", func(t *testing.T, rig *domainRig) {
			certPEM, _, _ := issuePanelCert(newNames, nil, time.Now())
			rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
		}},
		{"felis-api", func(t *testing.T, rig *domainRig) { rig.served = oldNames }},
		{"felis-api", func(t *testing.T, rig *domainRig) {
			certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
			rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
		}},
		{"proxy", func(t *testing.T, rig *domainRig) {
			writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
			rig.proxySince = time.Now().Add(time.Hour)
		}},
		{"proxy", func(t *testing.T, rig *domainRig) { rig.proxySince = time.Now().Add(-time.Hour) }},
		{"login gate", func(t *testing.T, rig *domainRig) {
			var ms v1alpha1.MinecraftServer
			_ = rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms)
			for i := range ms.Spec.Env {
				if ms.Spec.Env[i].Name == envRootDomain {
					ms.Spec.Env[i].Value = "old.example"
				}
			}
			if err := rig.cl.Update(context.Background(), &ms); err != nil {
				t.Fatal(err)
			}
		}},
		{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envRootDomain, "old.example") }},
		{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envPanelHostname, "console.old.example") }},
		{"Cloudflare tunnel", func(t *testing.T, rig *domainRig) {
			writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n  service: https://127.0.0.1:30443\n- hostname: op.console.old.example\n  service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
		}},
	}
	for _, tc := range cases {
		rig := newDomainRig(t)
		if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
			t.Fatal(err)
		}
		rig.out.Reset()
		tc.breakIt(t, rig)
		if code := rig.h.check(context.Background()); code != 1 {
			t.Errorf("%s behind: check = %d\n%s", tc.surface, code, rig.out)
			continue
		}
		var failed []string
		for _, ln := range strings.Split(rig.out.String(), "\n") {
			if strings.HasPrefix(ln, "  FAIL ") {
				failed = append(failed, ln)
			}
		}
		if len(failed) != 1 || !strings.Contains(failed[0], tc.surface) {
			t.Errorf("%s behind: FAIL lines %q", tc.surface, failed)
		}
	}
}

func TestDomainCheckPassesAConvergedInstallAndWarnsOnDNS(t *testing.T) {
	rig := newDomainRig(t)
	if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
		t.Fatal(err)
	}
	writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n  service: https://127.0.0.1:30443\n- hostname: op.console.new.example\n  service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
	rig.unresolved["op.console.new.example"] = true
	rig.unresolved[dnsProbeLabel+".new.example"] = true
	rig.out.Reset()
	if code := rig.h.check(context.Background()); code != 0 {
		t.Fatalf("check = %d\n%s", code, rig.out)
	}
	out := rig.out.String()
	for _, want := range []string{"  ok   Cloudflare tunnel: routes both names", "  warn DNS: op.console.new.example, *.new.example do not resolve from this host\n"} {
		if !strings.Contains(out, want) {
			t.Errorf("check lacks %q:\n%s", want, out)
		}
	}
	if strings.Contains(out, "TOKEN-NOT-TO-TOUCH") {
		t.Fatal("check printed the proxy's service token")
	}

	// A zone with only the wildcard: the admin name alone is missing, and why is said.
	delete(rig.unresolved, dnsProbeLabel+".new.example")
	rig.out.Reset()
	rig.h.check(context.Background())
	if want := "  warn DNS: op.console.new.example does not resolve from this host: the *.new.example wildcard does not cover op.console.new.example, which needs its own record\n"; !strings.Contains(rig.out.String(), want) {
		t.Errorf("check lacks %q:\n%s", want, rig.out)
	}
}

func (rig *domainRig) setPodEnv(t *testing.T, name, value string) {
	t.Helper()
	var pod corev1.Pod
	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
		t.Fatal(err)
	}
	for i, e := range pod.Spec.Containers[0].Env {
		if e.Name == name {
			pod.Spec.Containers[0].Env[i].Value = value
		}
	}
	if err := rig.cl.Update(context.Background(), &pod); err != nil {
		t.Fatal(err)
	}
}

func (rig *domainRig) putSecret(t *testing.T, ns, name, key, val string) {
	t.Helper()
	var s corev1.Secret
	if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
		t.Fatal(err)
	}
	s.Data[key] = []byte(val)
	if err := rig.cl.Update(context.Background(), &s); err != nil {
		t.Fatal(err)
	}
}

func TestParseUnitShow(t *testing.T) {
	st := parseUnitShow("LoadState=loaded\nActiveState=active\nActiveEnterTimestamp=@1790000000\n")
	if !st.loaded || !st.active || !st.since.Equal(time.Unix(1790000000, 0)) {
		t.Fatalf("%+v", st)
	}
	st = parseUnitShow("LoadState=not-found\nActiveState=inactive\nActiveEnterTimestamp=\n")
	if st.loaded || st.active || !st.since.IsZero() {
		t.Fatalf("%+v", st)
	}
}