feat(domain): felis domain set/check 把根域名换到所有面并逐面核对 (#12)

This commit is contained in:
Lemon-miaow committed 2026-09-26 10:23:55 +08:00
1 parent d601abb09b
commit 6a061859dc
6 files changed
+2333 -1

No files matched your search

+1353
View File
File diff suppressed because it is too large. Load diff
+892
View File
@@ -0,0 +1,892 @@
package main
import (
"bytes"
"context"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"errors"
"math/big"
"net"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"felis.lolicon.best/internal/apis/felis/v1alpha1"
"felis.lolicon.best/internal/naming"
"felis.lolicon.best/internal/platform"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)
// installerTOML is felis.toml as deploy/bootstrap.sh write_felis_toml renders it,
// comments included: the domain move has to leave all of it but three values alone.
func installerTOML(root, dbHost string) string {
return `# Generated by deploy/bootstrap.sh; rerun the installer to regenerate. Hand edits are
# overwritten, except [smtp], [[auth_source]], [offsite], and the operator-owned
# [registry] / [archive] overrides, which carry forward.
[server]
listen = "0.0.0.0:8080"
root_domain = "` + root + `"
[database]
url = "postgres://felis:pw@` + dbHost + `:5432/felis?sslmode=disable"
[k8s]
namespace = "minecraft"
egress_mode = "nodeport"
[velocity]
# The two always-on system servers that felis setup provisions.
login_image = "felis/limbo:1"
lobby_image = "felis/lobby:1"
game_port = 25565
[registry]
url = "registry.felis.svc:5000"
build_namespace = "felis-build"
[archive]
store = "tarLocal"
local_path = "/var/lib/felis/archives"
[auth]
admin_hostname = "op.console.` + root + `"
panel_hostname = "console.` + root + `"
access_jwt_aud = "aud123"
# Third-party Yggdrasil sources federated by the hasJoined multiplexer.
[[auth_source]]
tag = "littleskin"
prefix = "LS"
url = "https://littleskin.cn/api/yggdrasil/sessionserver/session/minecraft/hasJoined"
`
}
const linkPropsBody = `# Generated by deploy/bootstrap.sh — do not edit by hand; rerun the installer.
api-base-url=http://10.43.0.9:8081
service-token=TOKEN-NOT-TO-TOUCH
root-domain=old.example
panel-hostname=console.old.example
admin-hostname=op.console.old.example
login-server=login
lobby-server=lobby
`
var oldNames = domainNames{root: "old.example", panel: "console.old.example", admin: "op.console.old.example"}
var newNames = domainNames{root: "new.example", panel: "console.new.example", admin: "op.console.new.example"}
// domainRig models the host: the files, the cluster, and a felis-api, proxy and
// operator that pick up config the way the real ones do — the api serves what it
// read at its last restart, the proxy runs since its last restart, and the login
// pod carries the env its MinecraftServer had when it was last rolled.
type domainRig struct {
h domainHost
cl client.Client
out *bytes.Buffer
dir string
events []string
served domainNames
servedCert *x509.Certificate
proxySince time.Time
proxyLoaded bool
unresolved map[string]bool
// The fake operator: the CR env the login pod was last rolled to, and how
// many looks at the pod since the CR moved on.
rolledTo string
pending int
}
func (rig *domainRig) path(name string) string { return filepath.Join(rig.dir, name) }
func newDomainRig(t *testing.T) *domainRig {
t.Helper()
rig := &domainRig{out: &bytes.Buffer{}, dir: t.TempDir(), proxyLoaded: true, unresolved: map[string]bool{}}
writeTestFile(t, rig.path("felis.host.toml"), installerTOML("old.example", "127.0.0.1"), 0o600)
writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
if err := os.Symlink(rig.path("felis.host.toml"), rig.path("felis.toml")); err != nil {
t.Fatal(err)
}
certPEM, keyPEM, err := issuePanelCert(oldNames, []net.IP{net.ParseIP("10.211.55.6")}, time.Now())
if err != nil {
t.Fatal(err)
}
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
// The proxy started before its config was last written, which is how it
// stands after an install.
rig.proxySince = time.Now().Add(-time.Hour)
pod := []byte(installerTOML("old.example", "10.211.55.6"))
login, err := loginSystemServer("felis/limbo:1", "minecraft", platform.InternalAPIBaseURL("felis"), oldNames.root, oldNames.panel)
if err != nil {
t.Fatal(err)
}
lobby, err := lobbySystemServer("felis/lobby:1", "minecraft")
if err != nil {
t.Fatal(err)
}
loginPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"},
Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "minecraft", Env: podEnv(login.Spec.Env)}}},
Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}},
}
rig.rolledTo = envKey(login.Spec.Env)
rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithInterceptorFuncs(interceptor.Funcs{
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
if key.Name == naming.SystemLoginServer+"-0" {
rig.operatorTick(t, c)
}
return c.Get(ctx, key, obj, opts...)
},
}).WithObjects(
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.ConfigSecretName},
Data: map[string][]byte{platform.ConfigSecretKey: pod}},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: platform.ConfigSecretName},
Data: map[string][]byte{platform.ConfigSecretKey: pod}},
&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.APITLSSecretName},
Type: corev1.SecretTypeTLS, Data: map[string][]byte{corev1.TLSCertKey: certPEM, corev1.TLSPrivateKeyKey: keyPEM}},
login, lobby, loginPod,
).Build()
rig.served = oldNames
rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
rig.h = domainHost{
paths: domainPaths{
hostTOML: rig.path("felis.host.toml"), podTOML: rig.path("felis.pod.toml"), defaultTOML: rig.path("felis.toml"),
cert: rig.path("panel-tls.crt"), key: rig.path("panel-tls.key"),
linkProps: rig.path("felis-link.properties"), tunnelConfig: rig.path("cloudflared.yml"),
},
cl: rig.cl,
controlNS: "felis",
rollAPI: func(ctx context.Context) error {
rig.events = append(rig.events, "roll-api")
rig.restartAPI(t)
return nil
},
restartUnit: func(_ context.Context, unit string) error {
rig.events = append(rig.events, "restart "+unit)
rig.proxySince = time.Now().Add(time.Second)
return nil
},
unitState: func(context.Context, string) (unitStatus, error) {
return unitStatus{loaded: rig.proxyLoaded, active: rig.proxyLoaded, since: rig.proxySince}, nil
},
liveAPI: func(context.Context, string) (liveAPIView, error) {
return liveAPIView{names: rig.served, cert: rig.servedCert}, nil
},
lookupHost: func(_ context.Context, host string) ([]string, error) {
if rig.unresolved[host] {
return nil, errors.New("no such host")
}
return []string{"10.211.55.6"}, nil
},
passkeys: func(context.Context) (int, int, error) { return 3, 2, nil },
now: time.Now,
out: rig.out,
loginWait: 50 * time.Millisecond,
pollEvery: time.Millisecond,
}
return rig
}
func podEnv(env []v1alpha1.EnvVar) []corev1.EnvVar {
out := make([]corev1.EnvVar, len(env))
for i, e := range env {
out[i] = corev1.EnvVar{Name: e.Name, Value: e.Value}
}
return out
}
// restartAPI makes the fake felis-api load the config and certificate its
// Secrets hold now.
func (rig *domainRig) restartAPI(t *testing.T) {
t.Helper()
var cfg, tlsSec corev1.Secret
ctx := context.Background()
if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.ConfigSecretName}, &cfg); err != nil {
t.Fatal(err)
}
if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.APITLSSecretName}, &tlsSec); err != nil {
t.Fatal(err)
}
names, err := tomlDomainNames(cfg.Data[platform.ConfigSecretKey])
if err != nil {
t.Fatal(err)
}
rig.served = names
rig.servedCert, err = x509.ParseCertificate(mustCertDER(tlsSec.Data[corev1.TLSCertKey]))
if err != nil {
t.Fatal(err)
}
}
// rollLoginPod is the operator restarting the login pod onto its CR's env.
// operatorTick is the operator as the login pod is watched: once the CR's env
// changes it takes operatorLag looks at the pod before the restarted pod
// carries the new env, the way a real rollout lags the CR.
func (rig *domainRig) operatorTick(t *testing.T, c client.Client) {
t.Helper()
ctx := context.Background()
var ms v1alpha1.MinecraftServer
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
t.Fatal(err)
}
if envKey(ms.Spec.Env) == rig.rolledTo {
return
}
if rig.pending++; rig.pending < operatorLag {
return
}
var pod corev1.Pod
if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"}, &pod); err != nil {
t.Fatal(err)
}
pod.Spec.Containers[0].Env = podEnv(ms.Spec.Env)
if err := c.Update(ctx, &pod); err != nil {
t.Fatal(err)
}
rig.rolledTo, rig.pending = envKey(ms.Spec.Env), 0
}
const operatorLag = 3
func envKey(env []v1alpha1.EnvVar) string {
var b strings.Builder
for _, e := range env {
b.WriteString(e.Name + "=" + e.Value + "\n")
}
return b.String()
}
func (rig *domainRig) read(t *testing.T, name string) string {
t.Helper()
b, err := os.ReadFile(rig.path(name))
if err != nil {
t.Fatal(err)
}
return string(b)
}
func (rig *domainRig) secret(t *testing.T, ns, name string) map[string][]byte {
t.Helper()
var s corev1.Secret
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
t.Fatal(err)
}
return s.Data
}
func (rig *domainRig) crEnv(t *testing.T, name string) map[string]string {
t.Helper()
var ms v1alpha1.MinecraftServer
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil {
t.Fatal(err)
}
env := map[string]string{}
for _, e := range ms.Spec.Env {
env[e.Name] = e.Value
}
return env
}
// snapshot is every byte `set` may touch, for proving a refused or dry run
// touched none of it.
func (rig *domainRig) snapshot(t *testing.T) string {
t.Helper()
var b strings.Builder
entries, _ := os.ReadDir(rig.dir)
for _, e := range entries {
b.WriteString(e.Name() + "\n" + rig.read(t, e.Name()) + "\n")
}
var lines []string
for _, s := range []struct{ ns, name string }{{"felis", platform.ConfigSecretName}, {"minecraft", platform.ConfigSecretName}, {"felis", platform.APITLSSecretName}} {
for k, v := range rig.secret(t, s.ns, s.name) {
lines = append(lines, s.ns+"/"+s.name+"/"+k+"\n"+string(v))
}
}
for k, v := range rig.crEnv(t, naming.SystemLoginServer) {
lines = append(lines, "env "+k+"="+v)
}
sort.Strings(lines)
b.WriteString(strings.Join(lines, "\n"))
return b.String()
}
func TestNormalizeRootDomain(t *testing.T) {
for in, want := range map[string]string{
"Example.COM.": "example.com",
" mc.example.org ": "mc.example.org",
"10.211.55.6.nip.io": "10.211.55.6.nip.io",
"xn--bcher-kva.example": "xn--bcher-kva.example",
} {
got, err := normalizeRootDomain(in)
if err != nil || got != want {
t.Errorf("normalizeRootDomain(%q) = %q, %v; want %q", in, got, err, want)
}
}
for _, in := range []string{"", "https://example.com", "example.com:443", "example.com/x", "10.0.0.1", "::1",
"localhost", "a_b.example", "-a.example", "a-.example", strings.Repeat("a", 64) + ".example",
strings.Repeat("abcdefghi.", 25) + "example"} {
if got, err := normalizeRootDomain(in); err == nil {
t.Errorf("normalizeRootDomain(%q) = %q, want an error", in, got)
}
}
}
func TestPlanDomainChangeMovesDefaultsAndKeepsHandSetNames(t *testing.T) {
p := planDomainChange(oldNames, "new.example")
if p.to != newNames || p.customPanel || p.customAdmin {
t.Fatalf("defaults: %+v", p)
}
p = planDomainChange(domainNames{root: "old.example", panel: "play.corp.net", admin: "op.console.old.example"}, "new.example")
if p.to.panel != "play.corp.net" || !p.customPanel || p.to.admin != "op.console.new.example" || p.customAdmin {
t.Fatalf("hand-set panel: %+v", p)
}
p = planDomainChange(domainNames{root: "old.example", panel: "console.old.example", admin: "admin.corp.net"}, "new.example")
if p.to.admin != "admin.corp.net" || !p.customAdmin || p.to.panel != "console.new.example" {
t.Fatalf("hand-set admin: %+v", p)
}
}
func TestEditTOMLStringsChangesOnlyTheDomainLines(t *testing.T) {
orig := installerTOML("old.example", "127.0.0.1")
out, err := editTOMLStrings([]byte(orig), domainTOMLEdits(newNames))
if err != nil {
t.Fatal(err)
}
a, b := strings.Split(orig, "\n"), strings.Split(string(out), "\n")
if len(a) != len(b) {
t.Fatalf("line count %d → %d:\n%s", len(a), len(b), out)
}
changed := map[string]string{}
for i := range a {
if a[i] != b[i] {
changed[a[i]] = b[i]
}
}
want := map[string]string{
`root_domain = "old.example"`: `root_domain = "new.example"`,
`admin_hostname = "op.console.old.example"`: `admin_hostname = "op.console.new.example"`,
`panel_hostname = "console.old.example"`: `panel_hostname = "console.new.example"`,
}
if len(changed) != len(want) {
t.Fatalf("changed lines %v, want %v", changed, want)
}
for k, v := range want {
if changed[k] != v {
t.Errorf("%q → %q, want %q", k, changed[k], v)
}
}
}
func TestEditTOMLStringsAddsMissingKeysInTheirTable(t *testing.T) {
in := "[server]\nroot_domain = \"old.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\n\n[smtp]\nhost = \"relay\"\n"
out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames))
if err != nil {
t.Fatal(err)
}
want := "[server]\nroot_domain = \"new.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\npanel_hostname = \"console.new.example\"\nadmin_hostname = \"op.console.new.example\"\n\n[smtp]\nhost = \"relay\"\n"
if string(out) != want {
t.Fatalf("got:\n%s\nwant:\n%s", out, want)
}
out, err = editTOMLStrings([]byte("[server]\nroot_domain = \"old.example\"\n\n[[auth_source]]\ntag = \"ls\"\n"), domainTOMLEdits(newNames))
if err != nil {
t.Fatal(err)
}
got, err := tomlDomainNames(out)
if err != nil || got != newNames || !strings.Contains(string(out), "[[auth_source]]\ntag = \"ls\"\n") {
t.Fatalf("no [auth] table: %v %+v\n%s", err, got, out)
}
}
func TestEditTOMLStringsRefusesWhatItCannotEditExactly(t *testing.T) {
for name, in := range map[string]string{
"multi-line value": "[server]\nroot_domain = \"\"\"\nold.example\"\"\"\n[auth]\n",
// The key's line sits inside another value; the real key is absent.
"key inside a string": "[server]\nmotd = \"\"\"\nroot_domain = \"old.example\"\n\"\"\"\n[auth]\n",
"quoted header": "[server]\nroot_domain = \"old.example\"\n[\"auth\"]\npanel_hostname = \"console.old.example\"\n",
"dotted key": "server.root_domain = \"old.example\"\n",
"inline table": "server = { root_domain = \"old.example\" }\n",
} {
if out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames)); err == nil {
t.Errorf("%s: edited instead of refusing:\n%s", name, out)
}
}
}
// caSignedCert is an operator's certificate from their own CA; it names
// localhost too, so only the issuer tells it apart from the installer's.
func caSignedCert(t *testing.T, hosts ...string) (certPEM, keyPEM []byte) {
t.Helper()
caKey, _ := rsa.GenerateKey(rand.Reader, 2048)
ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Corp CA"}, IsCA: true,
BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
caCert, _ := x509.ParseCertificate(caDER)
key, _ := rsa.GenerateKey(rand.Reader, 2048)
leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: hosts[0]},
DNSNames: append(hosts, "localhost"), IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)}
der, err := x509.CreateCertificate(rand.Reader, leaf, caCert, &key.PublicKey, caKey)
if err != nil {
t.Fatal(err)
}
pk, _ := x509.MarshalPKCS8PrivateKey(key)
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pk})
}
func TestFelisIssuedCert(t *testing.T) {
mine, _, err := issuePanelCert(oldNames, nil, time.Now())
if err != nil {
t.Fatal(err)
}
c, _ := x509.ParseCertificate(mustCertDER(mine))
if !felisIssuedCert(c) {
t.Error("the installer's kind of certificate is not recognised as Felis-issued")
}
theirs, _ := caSignedCert(t, "console.old.example")
c, _ = x509.ParseCertificate(mustCertDER(theirs))
if felisIssuedCert(c) {
t.Error("a CA-signed certificate is taken for Felis-issued")
}
// Self-signed but without one of the installer's localhost names: someone
// else's.
key, _ := rsa.GenerateKey(rand.Reader, 2048)
for name, self := range map[string]*x509.Certificate{
"no localhost": {DNSNames: []string{"console.old.example"}, IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}},
"no 127.0.0.1": {DNSNames: []string{"console.old.example", "localhost"}},
} {
self.SerialNumber, self.Subject = big.NewInt(3), pkix.Name{CommonName: "x"}
self.NotBefore, self.NotAfter = time.Now().Add(-time.Hour), time.Now().Add(time.Hour)
der, _ := x509.CreateCertificate(rand.Reader, self, self, &key.PublicKey, key)
c, _ = x509.ParseCertificate(der)
if felisIssuedCert(c) {
t.Errorf("%s: a self-signed certificate is taken for Felis-issued", name)
}
}
}
func TestIssuePanelCertIsTheInstallersShape(t *testing.T) {
now := time.Now()
certPEM, keyPEM, err := issuePanelCert(newNames, []net.IP{net.ParseIP("10.211.55.6"), net.IPv4(127, 0, 0, 1)}, now)
if err != nil {
t.Fatal(err)
}
if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil {
t.Fatalf("key does not match the certificate: %v", err)
}
if b, _ := pem.Decode(keyPEM); b == nil || b.Type != "PRIVATE KEY" {
t.Fatalf("key is not PKCS#8 PEM like openssl writes")
}
c, _ := x509.ParseCertificate(mustCertDER(certPEM))
if !certCovers(c, newNames.panel, newNames.admin, "localhost") || !felisIssuedCert(c) {
t.Fatalf("names %v", c.DNSNames)
}
if len(c.IPAddresses) != 2 || !c.IPAddresses[1].Equal(net.ParseIP("10.211.55.6")) {
t.Fatalf("addresses %v, want 127.0.0.1 and the node address once each", c.IPAddresses)
}
if c.Subject.CommonName != newNames.admin || c.NotAfter.Sub(now) < 824*24*time.Hour || c.NotAfter.Sub(now) > 826*24*time.Hour {
t.Fatalf("CN %q, valid until %s", c.Subject.CommonName, c.NotAfter)
}
if len(c.ExtKeyUsage) != 1 || c.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth || c.IsCA {
t.Fatalf("usage %v, CA %v", c.ExtKeyUsage, c.IsCA)
}
}
func TestDomainSetMovesEverySurface(t *testing.T) {
rig := newDomainRig(t)
hostBefore := rig.read(t, "felis.host.toml")
code, err := rig.h.set(context.Background(), "New.Example", true)
if err != nil || code != 0 {
t.Fatalf("set = %d, %v\n%s", code, err, rig.out)
}
// The configs: the three values moved, everything else — comments, the
// database host of each copy, the Access audience — is as it was.
host := rig.read(t, "felis.host.toml")
if want := strings.NewReplacer("old.example", "new.example").Replace(hostBefore); host != want {
t.Fatalf("host toml:\n%s", host)
}
pod := rig.read(t, "felis.pod.toml")
if got, _ := tomlDomainNames([]byte(pod)); got != newNames || !strings.Contains(pod, "@10.211.55.6:5432") {
t.Fatalf("pod toml:\n%s", pod)
}
if link, err := os.Readlink(rig.path("felis.toml")); err != nil || link != rig.path("felis.host.toml") {
t.Fatalf("felis.toml is no longer the link to the host copy: %q %v", link, err)
}
if st, _ := os.Stat(rig.path("felis.host.toml")); st.Mode().Perm() != 0o600 {
t.Fatalf("host toml mode %v", st.Mode().Perm())
}
// The certificate: reissued for the new names, the node address kept, the old
// pair beside it.
c, err := readCertFile(rig.path("panel-tls.crt"))
if err != nil || !certCovers(c, newNames.panel, newNames.admin) || !felisIssuedCert(c) {
t.Fatalf("certificate: %v %v", err, c.DNSNames)
}
if !c.IPAddresses[len(c.IPAddresses)-1].Equal(net.ParseIP("10.211.55.6")) {
t.Fatalf("addresses %v", c.IPAddresses)
}
if _, err := tls.LoadX509KeyPair(rig.path("panel-tls.crt"), rig.path("panel-tls.key")); err != nil {
t.Fatalf("new pair: %v", err)
}
if st, _ := os.Stat(rig.path("panel-tls.key")); st.Mode().Perm() != 0o600 {
t.Fatalf("key mode %v", st.Mode().Perm())
}
backups, _ := filepath.Glob(rig.path("panel-tls.*.pre-domain-*"))
if len(backups) != 2 {
t.Fatalf("old pair kept as %v", backups)
}
for _, b := range backups {
if st, _ := os.Stat(b); strings.Contains(b, ".key.") && st.Mode().Perm() != 0o600 {
t.Fatalf("kept key %s has mode %v", b, st.Mode().Perm())
}
old, _ := os.ReadFile(b)
if strings.Contains(b, ".crt.") {
oc, _ := x509.ParseCertificate(mustCertDER(old))
if oc == nil || !certCovers(oc, oldNames.panel) {
t.Fatalf("kept certificate is not the old one")
}
}
}
// The Secrets carry the files.
for _, ns := range []string{"felis", "minecraft"} {
if got := rig.secret(t, ns, platform.ConfigSecretName)[platform.ConfigSecretKey]; string(got) != pod {
t.Fatalf("%s/felis-config is not felis.pod.toml", ns)
}
}
tlsData := rig.secret(t, "felis", platform.APITLSSecretName)
if string(tlsData[corev1.TLSCertKey]) != rig.read(t, "panel-tls.crt") || string(tlsData[corev1.TLSPrivateKeyKey]) != rig.read(t, "panel-tls.key") {
t.Fatal("felis-api-tls does not hold the new pair")
}
// The login gate's env moved and nothing else did.
env := rig.crEnv(t, naming.SystemLoginServer)
if env[envRootDomain] != newNames.root || env[envPanelHostname] != newNames.panel || env[envAPIBaseURL] != platform.InternalAPIBaseURL("felis") {
t.Fatalf("login env %v", env)
}
// The proxy's file: the three keys moved, its token and mode did not.
props := rig.read(t, "felis-link.properties")
if want := strings.NewReplacer("old.example", "new.example").Replace(linkPropsBody); props != want {
t.Fatalf("felis-link.properties:\n%s", props)
}
if st, _ := os.Stat(rig.path("felis-link.properties")); st.Mode().Perm() != 0o640 {
t.Fatalf("felis-link.properties mode %v", st.Mode().Perm())
}
if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
t.Fatalf("events %v", rig.events)
}
if !strings.Contains(rig.out.String(), "Every surface is on new.example.") {
t.Fatalf("the closing check did not pass:\n%s", rig.out)
}
}
func TestDomainSetWithoutYesChangesNothing(t *testing.T) {
rig := newDomainRig(t)
before := rig.snapshot(t)
code, err := rig.h.set(context.Background(), "new.example", false)
if err != nil || code != 0 {
t.Fatalf("set = %d, %v", code, err)
}
if rig.snapshot(t) != before || len(rig.events) != 0 {
t.Fatalf("a dry run changed something (events %v)", rig.events)
}
out := rig.out.String()
for _, want := range []string{
"console.old.example → console.new.example",
"3 passkey(s) of 2 user(s) are bound to console.old.example",
"does not cover op.console.new.example",
"No [smtp] relay is configured",
"sudo felis domain set -yes new.example",
} {
if !strings.Contains(out, want) {
t.Errorf("plan lacks %q:\n%s", want, out)
}
}
}
func TestDomainSetKeepsAHandSetPanelHostname(t *testing.T) {
rig := newDomainRig(t)
for _, f := range []string{"felis.host.toml", "felis.pod.toml"} {
writeTestFile(t, rig.path(f), strings.Replace(rig.read(t, f), `panel_hostname = "console.old.example"`, `panel_hostname = "play.corp.net"`, 1), 0o600)
}
code, err := rig.h.set(context.Background(), "new.example", true)
if err != nil {
t.Fatalf("set: %v\n%s", err, rig.out)
}
got, _ := tomlDomainNames([]byte(rig.read(t, "felis.host.toml")))
if got != (domainNames{root: "new.example", panel: "play.corp.net", admin: "op.console.new.example"}) {
t.Fatalf("names %+v", got)
}
c, _ := readCertFile(rig.path("panel-tls.crt"))
if !certCovers(c, "play.corp.net", "op.console.new.example") {
t.Fatalf("certificate names %v", c.DNSNames)
}
if env := rig.crEnv(t, naming.SystemLoginServer); env[envPanelHostname] != "play.corp.net" {
t.Fatalf("login env %v", env)
}
if code != 0 || !strings.Contains(rig.out.String(), "play.corp.net (set by hand, kept") {
t.Fatalf("code %d:\n%s", code, rig.out)
}
}
func TestDomainSetRefusesAnOperatorCertificateForOtherNames(t *testing.T) {
rig := newDomainRig(t)
certPEM, keyPEM := caSignedCert(t, "console.old.example", "op.console.old.example")
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
before := rig.snapshot(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "not issued by Felis") {
t.Fatalf("err = %v", err)
}
if rig.snapshot(t) != before || len(rig.events) != 0 {
t.Fatal("a refused move changed something")
}
// The operator's certificate for the new names is kept as it is.
certPEM, keyPEM = caSignedCert(t, "console.new.example", "op.console.new.example")
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatalf("set: %v", err)
}
if rig.read(t, "panel-tls.crt") != string(certPEM) {
t.Fatal("the operator's certificate was replaced")
}
}
func TestDomainSetRefusesAConfigItCannotEdit(t *testing.T) {
rig := newDomainRig(t)
writeTestFile(t, rig.path("felis.pod.toml"), strings.Replace(rig.read(t, "felis.pod.toml"), "[auth]", "[\"auth\"]", 1), 0o600)
before := rig.snapshot(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "by hand") {
t.Fatalf("err = %v", err)
}
if rig.snapshot(t) != before || len(rig.events) != 0 {
t.Fatal("a refused move changed something")
}
}
func TestDomainSetAgainOnlyConvergesWhatIsBehind(t *testing.T) {
rig := newDomainRig(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
rig.events, rig.out = nil, &bytes.Buffer{}
rig.h.out = rig.out
before := rig.snapshot(t)
code, err := rig.h.set(context.Background(), "new.example", true)
if err != nil || code != 0 {
t.Fatalf("second set = %d, %v\n%s", code, err, rig.out)
}
if len(rig.events) != 0 || rig.snapshot(t) != before {
t.Fatalf("a converged install was touched again: %v\n%s", rig.events, rig.out)
}
// A proxy that was not restarted after the move is restarted by a re-run, and
// an api still on the old config is rolled.
rig.proxySince = time.Now().Add(-time.Hour)
rig.served = oldNames
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" {
t.Fatalf("events %v", rig.events)
}
// An api on the new names that still presents the old certificate is rolled.
rig.events = nil
oldCert, _, _ := issuePanelCert(oldNames, nil, time.Now())
rig.servedCert, _ = x509.ParseCertificate(mustCertDER(oldCert))
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
if strings.Join(rig.events, ",") != "roll-api" {
t.Fatalf("events %v", rig.events)
}
}
func TestDomainSetRefusesALoginServerItDoesNotOwn(t *testing.T) {
rig := newDomainRig(t)
var ms v1alpha1.MinecraftServer
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil {
t.Fatal(err)
}
delete(ms.Labels, v1alpha1.LabelSystemRole)
if err := rig.cl.Update(context.Background(), &ms); err != nil {
t.Fatal(err)
}
if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "system role") {
t.Fatalf("err = %v", err)
}
if env := rig.crEnv(t, naming.SystemLoginServer); env[envRootDomain] != oldNames.root {
t.Fatalf("a server not marked as the login gate was changed: %v", env)
}
}
func TestDomainCheckNamesTheSurfaceThatIsBehind(t *testing.T) {
cases := []struct {
surface string
breakIt func(t *testing.T, rig *domainRig)
}{
{"felis.pod.toml", func(t *testing.T, rig *domainRig) {
writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600)
}},
{"Secret minecraft/felis-config", func(t *testing.T, rig *domainRig) {
rig.putSecret(t, "minecraft", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
}},
{"Secret felis/felis-config", func(t *testing.T, rig *domainRig) {
rig.putSecret(t, "felis", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x"))
}},
{"panel certificate", func(t *testing.T, rig *domainRig) {
// The Secret follows the file, so only the certificate's names are wrong.
certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644)
rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
}},
{"Secret felis/felis-api-tls", func(t *testing.T, rig *domainRig) {
certPEM, _, _ := issuePanelCert(newNames, nil, time.Now())
rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM))
}},
{"felis-api", func(t *testing.T, rig *domainRig) { rig.served = oldNames }},
{"felis-api", func(t *testing.T, rig *domainRig) {
certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now())
rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM))
}},
{"proxy", func(t *testing.T, rig *domainRig) {
writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640)
rig.proxySince = time.Now().Add(time.Hour)
}},
{"proxy", func(t *testing.T, rig *domainRig) { rig.proxySince = time.Now().Add(-time.Hour) }},
{"login gate", func(t *testing.T, rig *domainRig) {
var ms v1alpha1.MinecraftServer
_ = rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms)
for i := range ms.Spec.Env {
if ms.Spec.Env[i].Name == envRootDomain {
ms.Spec.Env[i].Value = "old.example"
}
}
if err := rig.cl.Update(context.Background(), &ms); err != nil {
t.Fatal(err)
}
}},
{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envRootDomain, "old.example") }},
{"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envPanelHostname, "console.old.example") }},
{"Cloudflare tunnel", func(t *testing.T, rig *domainRig) {
writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n service: https://127.0.0.1:30443\n- hostname: op.console.old.example\n service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
}},
}
for _, tc := range cases {
rig := newDomainRig(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
rig.out.Reset()
tc.breakIt(t, rig)
if code := rig.h.check(context.Background()); code != 1 {
t.Errorf("%s behind: check = %d\n%s", tc.surface, code, rig.out)
continue
}
var failed []string
for _, ln := range strings.Split(rig.out.String(), "\n") {
if strings.HasPrefix(ln, " FAIL ") {
failed = append(failed, ln)
}
}
if len(failed) != 1 || !strings.Contains(failed[0], tc.surface) {
t.Errorf("%s behind: FAIL lines %q", tc.surface, failed)
}
}
}
func TestDomainCheckPassesAConvergedInstallAndWarnsOnDNS(t *testing.T) {
rig := newDomainRig(t)
if _, err := rig.h.set(context.Background(), "new.example", true); err != nil {
t.Fatal(err)
}
writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n service: https://127.0.0.1:30443\n- hostname: op.console.new.example\n service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644)
rig.unresolved["op.console.new.example"] = true
rig.unresolved[dnsProbeLabel+".new.example"] = true
rig.out.Reset()
if code := rig.h.check(context.Background()); code != 0 {
t.Fatalf("check = %d\n%s", code, rig.out)
}
out := rig.out.String()
for _, want := range []string{" ok Cloudflare tunnel: routes both names", " warn DNS: op.console.new.example, *.new.example do not resolve from this host\n"} {
if !strings.Contains(out, want) {
t.Errorf("check lacks %q:\n%s", want, out)
}
}
if strings.Contains(out, "TOKEN-NOT-TO-TOUCH") {
t.Fatal("check printed the proxy's service token")
}
// A zone with only the wildcard: the admin name alone is missing, and why is said.
delete(rig.unresolved, dnsProbeLabel+".new.example")
rig.out.Reset()
rig.h.check(context.Background())
if want := " warn DNS: op.console.new.example does not resolve from this host: the *.new.example wildcard does not cover op.console.new.example, which needs its own record\n"; !strings.Contains(rig.out.String(), want) {
t.Errorf("check lacks %q:\n%s", want, rig.out)
}
}
func (rig *domainRig) setPodEnv(t *testing.T, name, value string) {
t.Helper()
var pod corev1.Pod
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil {
t.Fatal(err)
}
for i, e := range pod.Spec.Containers[0].Env {
if e.Name == name {
pod.Spec.Containers[0].Env[i].Value = value
}
}
if err := rig.cl.Update(context.Background(), &pod); err != nil {
t.Fatal(err)
}
}
func (rig *domainRig) putSecret(t *testing.T, ns, name, key, val string) {
t.Helper()
var s corev1.Secret
if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil {
t.Fatal(err)
}
s.Data[key] = []byte(val)
if err := rig.cl.Update(context.Background(), &s); err != nil {
t.Fatal(err)
}
}
func TestParseUnitShow(t *testing.T) {
st := parseUnitShow("LoadState=loaded\nActiveState=active\nActiveEnterTimestamp=@1790000000\n")
if !st.loaded || !st.active || !st.since.Equal(time.Unix(1790000000, 0)) {
t.Fatalf("%+v", st)
}
st = parseUnitShow("LoadState=not-found\nActiveState=inactive\nActiveEnterTimestamp=\n")
if st.loaded || st.active || !st.since.IsZero() {
t.Fatalf("%+v", st)
}
}
+16 -1
View File
@@ -259,6 +259,11 @@ func writeTokenSecret(ctx context.Context, cl client.Client, namespace, name, to
// file is replaced atomically and keeps its mode and owner: felis-link.properties // file is replaced atomically and keeps its mode and owner: felis-link.properties
// is root:felis-velocity 0640, and the proxy must still be able to read it. // is root:felis-velocity 0640, and the proxy must still be able to read it.
func setKeyValueLine(path, key, sep, value string) error { func setKeyValueLine(path, key, sep, value string) error {
return setKeyValueLines(path, sep, [][2]string{{key, value}})
}
// setKeyValueLines is setKeyValueLine for several keys in one rewrite.
func setKeyValueLines(path, sep string, kv [][2]string) error {
info, err := os.Stat(path) info, err := os.Stat(path)
if err != nil { if err != nil {
return err return err
@@ -268,6 +273,8 @@ func setKeyValueLine(path, key, sep, value string) error {
return err return err
} }
lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n") lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n")
for _, p := range kv {
key, value := p[0], p[1]
found := false found := false
for i, ln := range lines { for i, ln := range lines {
k, _, ok := strings.Cut(ln, sep) k, _, ok := strings.Cut(ln, sep)
@@ -279,6 +286,14 @@ func setKeyValueLine(path, key, sep, value string) error {
if !found { if !found {
lines = append(lines, key+sep+value) lines = append(lines, key+sep+value)
} }
}
return replaceFileKeepingMode(path, info, []byte(strings.Join(lines, "\n")+"\n"))
}
// replaceFileKeepingMode atomically replaces path with data, keeping the mode and
// owner info describes: these files are read by other users (the proxy's) and
// some hold credentials, so a rewrite must not widen or narrow who can read them.
func replaceFileKeepingMode(path string, info os.FileInfo, data []byte) error {
tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*") tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")
if err != nil { if err != nil {
return err return err
@@ -294,7 +309,7 @@ func setKeyValueLine(path, key, sep, value string) error {
return err return err
} }
} }
if _, err := tmp.WriteString(strings.Join(lines, "\n") + "\n"); err != nil { if _, err := tmp.Write(data); err != nil {
tmp.Close() tmp.Close()
return err return err
} }
+2
View File
@@ -32,6 +32,7 @@ Commands:
setup Run host bootstrap + first-run setup console (TUI; requires root/sudo) setup Run host bootstrap + first-run setup console (TUI; requires root/sudo)
converge Fill in fields a newer desired spec added to already-installed system servers converge Fill in fields a newer desired spec added to already-installed system servers
rotate-token Replace one internal caller's token and restart what holds it (velocity|limbo|build|ops; requires root/sudo) rotate-token Replace one internal caller's token and restart what holds it (velocity|limbo|build|ops; requires root/sudo)
domain Move the install to a new root domain on every surface that carries it, or check each one (set|check; requires root/sudo)
watchdog Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer) watchdog Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer)
version Print the build stamp of this binary version Print the build stamp of this binary
update Report which platform components have updates available update Report which platform components have updates available
@@ -71,6 +72,7 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{
"setup": cmdSetup, "setup": cmdSetup,
"converge": cmdConverge, "converge": cmdConverge,
"rotate-token": cmdRotateToken, "rotate-token": cmdRotateToken,
"domain": cmdDomain,
"breakGlass": cmdBreakGlass, "breakGlass": cmdBreakGlass,
"bootstrap-assets": cmdBootstrapAssets, "bootstrap-assets": cmdBootstrapAssets,
"init-forwarding": cmdInitForwarding, "init-forwarding": cmdInitForwarding,
+62
View File
@@ -444,3 +444,65 @@ production install:
then log in and restore one world. `felis offsite status` and `felis db check` exit then log in and restore one world. `felis offsite status` and `felis db check` exit
non-zero when the copy or the newest bundle is stale; wire them into your monitoring, non-zero when the copy or the newest bundle is stale; wire them into your monitoring,
or rely on the watchdog's mail. or rely on the watchdog's mail.
## 6. Changing the root domain [VM-VERIFIED] [GO-TESTED] [SH-TESTED]
The root domain is written into more places than the installer's config: the panel
certificate (`/etc/felis/panel-tls.crt`), the `felis-config` Secret in both namespaces,
the `felis-api-tls` Secret, the proxy's `felis-link.properties`, the login gate's
`MinecraftServer` env (`FELIS_ROOT_DOMAIN`, `FELIS_PANEL_HOSTNAME`), the Cloudflare tunnel
and DNS. `felis domain set` moves every one of them that lives on the host, in that
order, then restarts what reads them; `felis domain check` reports each surface on its
own line. The installer keeps the installed domain: a rerun with a different
`FELIS_ROOT_DOMAIN` stops and names this command.
```sh
sudo felis domain set new.example.net # the plan: every surface, what it moves to, what it costs
sudo felis domain set -yes new.example.net # do it
sudo felis domain check # one line per surface; exits 1 while any is behind
```
What it keeps:
- A panel or admin-console hostname set by hand in `[auth]` (anything other than
`console.<root>` / `op.console.<root>`) stays as it is; change it in
`/etc/felis/felis.host.toml` yourself if it should move, then run `set` again.
- The other `[auth]` keys (`access_jwt_aud`, `client_ip_header`) and every other line of
both config files. The edit refuses a file it cannot change line for line (a multi-line
value, a quoted or dotted key) and names what to fix.
- An operator's own certificate. The installer's self-signed certificate is reissued for
the new names (same shape, the old pair saved beside it as `*.pre-domain-<time>`); a
certificate from another issuer that does not cover the new names stops the command
before anything changes. Replace it with one that does, then run `set` again.
What it costs, which the plan prints before `-yes`:
- **DNS.** `<root>`, `console.<root>`, `op.console.<root>` and `*.<root>` must reach the
host. The wildcard does not cover `op.console.<root>`, a third-level name: give it its
own record. `check` resolves each name and warns on the ones that do not resolve yet.
- **Players.** Servers are reached as `<name>.<new root>`; the old addresses stop routing,
and the proxy restart disconnects everyone online. The first installer re-run after a
move restarts the proxy once more: the fingerprint it keeps of the proxy's files
predates the move.
- **Sign-in.** Session cookies belong to the old hostnames, so everyone signs in again.
Passkeys are bound to the panel hostname: when it changes, the plan counts the passkeys
that stop working, and their users sign in with an email code and register a new one.
Without an `[smtp]` relay no code is delivered; an Owner locked out that way recovers
with `sudo felis breakGlass`.
- **Cloudflare.** The tunnel's ingress and the Access application still carry the old
names. Re-run the Cloudflare step of `sudo felis setup` after the move; `check` lists
the tunnel's hostnames against the new ones.
- **A proxy on another host** (a remote `felis-link.properties`) is outside this host's
reach: `set` prints the three keys to put there.
`set` is safe to repeat: a second run changes only what is still behind, and on an
install that is already on the domain it converges whatever `check` reports. The same
holds after an interruption.
On the reference VM the move from `10.211.55.6.nip.io` to `10-211-55-6.nip.io` took 34
seconds. The certificate served on 30443, `/config.json` on both hostnames, the proxy's
`Felis routing ready: rootDomain=` log line and the login pod's env all carried the new
names afterwards. A second `set -yes` changed and restarted nothing; the installer run
with the old `FELIS_ROOT_DOMAIN` stopped at its first check; a full installer re-run kept
the moved domain and left `check` clean; moving back restored every surface
**[VM-VERIFIED]**.
+8
View File
@@ -191,6 +191,14 @@ const (
// namespace; the host-side off-site copy reads and restores it. // namespace; the host-side off-site copy reads and restores it.
const UploadsPVCName = uploadsPVCName const UploadsPVCName = uploadsPVCName
// The Secrets felis-api mounts its config and panel certificate from, which the
// host rewrites when the install moves to a new root domain (felis domain set).
const (
ConfigSecretName = configSecretName
ConfigSecretKey = configSecretKey
APITLSSecretName = apiTLSSecretName
)
// ControlPlaneUID is the uid and gid the control-plane pods run as, which own // ControlPlaneUID is the uid and gid the control-plane pods run as, which own
// what they write to their volumes; a host-side restore into one of them // what they write to their volumes; a host-side restore into one of them
// writes as it. // writes as it.