From 6a061859dc8b17a2b94fdc9032b2684f32b18772 Mon Sep 17 00:00:00 2001 From: Lemon-miaow Date: Sat, 26 Sep 2026 10:23:55 +0800 Subject: [PATCH] =?UTF-8?q?feat(domain):=20felis=20domain=20set/check=20?= =?UTF-8?q?=E6=8A=8A=E6=A0=B9=E5=9F=9F=E5=90=8D=E6=8D=A2=E5=88=B0=E6=89=80?= =?UTF-8?q?=E6=9C=89=E9=9D=A2=E5=B9=B6=E9=80=90=E9=9D=A2=E6=A0=B8=E5=AF=B9?= =?UTF-8?q?=20(#12)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/felis/domain.go | 1353 ++++++++++++++++++++++++++++++++ cmd/felis/domain_test.go | 892 +++++++++++++++++++++ cmd/felis/rotatetoken.go | 35 +- cmd/felis/run.go | 2 + docs/operations.md | 62 ++ internal/platform/workloads.go | 8 + 6 files changed, 2342 insertions(+), 10 deletions(-) create mode 100644 cmd/felis/domain.go create mode 100644 cmd/felis/domain_test.go diff --git a/cmd/felis/domain.go b/cmd/felis/domain.go new file mode 100644 index 0000000..88b2a62 --- /dev/null +++ b/cmd/felis/domain.go @@ -0,0 +1,1353 @@ +package main + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "math/big" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/config" + "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/platform" + "felis.lolicon.best/internal/store" + + "github.com/BurntSushi/toml" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" +) + +// felis domain moves an installed platform to a new root domain (#12). The domain +// is rendered into places nothing re-renders afterwards, and a surface left behind +// breaks one feature rather than the whole install: +// +// - the toml configs (felis.host.toml, felis.pod.toml, and felis.toml when it is +// a file of its own rather than the link to the host copy); +// - the panel certificate, which the installer writes once; +// - the felis-config Secret (and its workload-namespace mirror) and the +// felis-api-tls Secret, which felis-api mounts; +// - the proxy's felis-link.properties; +// - the login gate's MinecraftServer env. +// +// `set` rewrites each of them in place and keeps every other value; `check` reads +// each back, including what the running felis-api, proxy and login pod serve, so a +// half-moved install says which surface is behind. Re-running the installer is not +// the way: it regenerates files an operator has tuned, and it keeps the old panel +// certificate. + +const ( + domainUsage = "Usage: felis domain set [-yes] | felis domain check" + // dnsProbeLabel is looked up under the root domain to see whether the wildcard + // record the game subdomains need exists: no real server is named this. + dnsProbeLabel = "felis-dns-probe" + // panelCertDays matches the installer's `openssl req -days 825`. + panelCertDays = 825 +) + +var domainLabelRE = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`) + +// domainNames are the three names a root domain puts on the install. +type domainNames struct { + root, panel, admin string +} + +func effectiveDomainNames(root, panel, admin string) domainNames { + return domainNames{root: root, panel: defaultPanelHostname(root, panel), admin: defaultAdminHostname(root, admin)} +} + +// domainPlan is what `felis domain set` changes. A hostname that is not the +// default under the old root (console., op.console.) was set by hand, +// and it stays as it is. +type domainPlan struct { + from, to domainNames + customPanel, customAdmin bool +} + +func planDomainChange(cur domainNames, newRoot string) domainPlan { + p := domainPlan{from: cur, to: domainNames{root: newRoot, panel: "console." + newRoot, admin: "op.console." + newRoot}} + if cur.panel != "console."+cur.root { + p.customPanel, p.to.panel = true, cur.panel + } + if cur.admin != "op.console."+cur.root { + p.customAdmin, p.to.admin = true, cur.admin + } + return p +} + +// normalizeRootDomain lowercases a root domain and refuses anything that is not +// a DNS name the panel and the game subdomains can live under. +func normalizeRootDomain(s string) (string, error) { + d := strings.ToLower(strings.Trim(strings.TrimSpace(s), ".")) + switch { + case d == "": + return "", errors.New("the new root domain is empty") + case strings.Contains(d, "://") || strings.ContainsAny(d, "/:@ \t"): + return "", fmt.Errorf("%q is not a bare domain: give the name alone, without a scheme, port or path", s) + case net.ParseIP(d) != nil: + return "", fmt.Errorf("%q is an IP address: the panel and the game subdomains need a DNS name (for a test install, .nip.io)", s) + case len("op.console.")+len(d) > 253: + return "", fmt.Errorf("%q is too long: op.console. must fit in 253 characters", s) + } + labels := strings.Split(d, ".") + if len(labels) < 2 { + return "", fmt.Errorf("%q needs at least two labels, e.g. example.com", s) + } + for _, l := range labels { + if !domainLabelRE.MatchString(l) { + return "", fmt.Errorf("%q is not a valid domain: label %q may hold only a-z, 0-9 and inner hyphens, 63 characters at most", s, l) + } + } + return d, nil +} + +// tomlStringEdit sets one string key of one table. +type tomlStringEdit struct{ table, key, value string } + +func domainTOMLEdits(n domainNames) []tomlStringEdit { + return []tomlStringEdit{ + {"server", "root_domain", n.root}, + {"auth", "panel_hostname", n.panel}, + {"auth", "admin_hostname", n.admin}, + } +} + +var ( + tomlTableHeaderRE = regexp.MustCompile(`^\s*\[\s*([A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*)\s*\]\s*(#.*)?$`) + tomlKeyLineRE = regexp.MustCompile(`^\s*([A-Za-z0-9_-]+)\s*=`) +) + +// editTOMLStrings sets string keys in named tables by editing lines, so the +// comments the installer writes to explain the file, every other key and the +// layout stay as they were (a decode/encode round trip drops the comments). A key +// that is missing goes after the last key of its table, and a missing table goes +// at the end. The result is decoded and compared with the original plus the +// intended edits: a file this editor reads differently from the TOML decoder (a +// multi-line value, a quoted or dotted key) is refused rather than half-edited. +func editTOMLStrings(raw []byte, edits []tomlStringEdit) ([]byte, error) { + var lines []string + if len(raw) > 0 { + lines = strings.Split(strings.TrimSuffix(string(raw), "\n"), "\n") + } + done := make([]bool, len(edits)) + // last[table] is the line of that table's header or of its last key. + last := map[string]int{} + table := "" + for i, ln := range lines { + if trimmed := strings.TrimSpace(ln); strings.HasPrefix(trimmed, "[") { + table = "\x00" // an array table, or a header this editor does not read: never a target + if m := tomlTableHeaderRE.FindStringSubmatch(ln); m != nil { + table = m[1] + last[table] = i + } + continue + } + m := tomlKeyLineRE.FindStringSubmatch(ln) + if m == nil { + continue + } + last[table] = i + for j, e := range edits { + if e.table == table && e.key == m[1] { + indent := ln[:len(ln)-len(strings.TrimLeft(ln, " \t"))] + lines[i] = indent + tomlStringLine(e.key, e.value) + done[j] = true + } + } + } + + pending := map[string][]string{} + var newTables []string + for j, e := range edits { + if done[j] { + continue + } + if _, ok := last[e.table]; !ok && pending[e.table] == nil { + newTables = append(newTables, e.table) + } + pending[e.table] = append(pending[e.table], tomlStringLine(e.key, e.value)) + } + var existing []string + for t := range pending { + if _, ok := last[t]; ok { + existing = append(existing, t) + } + } + // Bottom-up, so the positions of the tables above stay valid. + sort.Slice(existing, func(a, b int) bool { return last[existing[a]] > last[existing[b]] }) + for _, t := range existing { + at := last[t] + 1 + lines = append(lines[:at], append(append([]string{}, pending[t]...), lines[at:]...)...) + } + for _, t := range newTables { + lines = append(lines, "", "["+t+"]") + lines = append(lines, pending[t]...) + } + out := []byte(strings.Join(lines, "\n") + "\n") + if err := verifyTOMLEdit(raw, out, edits); err != nil { + return nil, err + } + return out, nil +} + +// verifyTOMLEdit proves edited decodes to exactly orig plus the edits. +func verifyTOMLEdit(orig, edited []byte, edits []tomlStringEdit) error { + want, got := map[string]any{}, map[string]any{} + if _, err := toml.Decode(string(orig), &want); err != nil { + return fmt.Errorf("parse: %w", err) + } + if _, err := toml.Decode(string(edited), &got); err != nil { + return fmt.Errorf("the edited file would not parse: %w", err) + } + for _, e := range edits { + t, ok := want[e.table].(map[string]any) + if !ok { + if _, taken := want[e.table]; taken { + return fmt.Errorf("%s is not a table", e.table) + } + t = map[string]any{} + want[e.table] = t + } + t[e.key] = e.value + } + if !reflect.DeepEqual(want, got) { + return errors.New("a line edit would change more than the domain keys (a multi-line value, or a quoted or dotted key?)") + } + return nil +} + +func tomlStringLine(key, value string) string { + var b strings.Builder + for _, r := range value { + switch { + case r == '"' || r == '\\': + b.WriteByte('\\') + b.WriteRune(r) + case r < 0x20 || r == 0x7f: + fmt.Fprintf(&b, `\u%04X`, r) + default: + b.WriteRune(r) + } + } + return key + ` = "` + b.String() + `"` +} + +// tomlDomainNames reads the effective names out of a felis.toml. +func tomlDomainNames(raw []byte) (domainNames, error) { + var doc struct { + Server struct { + RootDomain string `toml:"root_domain"` + } `toml:"server"` + Auth struct { + PanelHostname string `toml:"panel_hostname"` + AdminHostname string `toml:"admin_hostname"` + } `toml:"auth"` + } + if _, err := toml.Decode(string(raw), &doc); err != nil { + return domainNames{}, err + } + return effectiveDomainNames(doc.Server.RootDomain, doc.Auth.PanelHostname, doc.Auth.AdminHostname), nil +} + +// felisIssuedCert reports whether c is a panel certificate Felis made for itself: +// self-signed, and carrying the localhost names the installer always adds. One an +// operator installed (from a CA, or their own) is theirs to replace. +func felisIssuedCert(c *x509.Certificate) bool { + if !bytes.Equal(c.RawIssuer, c.RawSubject) || c.CheckSignature(c.SignatureAlgorithm, c.RawTBSCertificate, c.Signature) != nil { + return false + } + hasLocalhost := false + for _, n := range c.DNSNames { + hasLocalhost = hasLocalhost || n == "localhost" + } + hasLoopback := false + for _, ip := range c.IPAddresses { + hasLoopback = hasLoopback || ip.Equal(net.IPv4(127, 0, 0, 1)) + } + return hasLocalhost && hasLoopback +} + +func certCovers(c *x509.Certificate, hosts ...string) bool { + for _, h := range hosts { + if c.VerifyHostname(h) != nil { + return false + } + } + return true +} + +func readCertFile(path string) (*x509.Certificate, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + for { + var block *pem.Block + block, raw = pem.Decode(raw) + if block == nil { + return nil, fmt.Errorf("%s holds no certificate", path) + } + if block.Type == "CERTIFICATE" { + return x509.ParseCertificate(block.Bytes) + } + } +} + +// issuePanelCert makes the certificate the installer would have made for these +// names (ensure_panel_tls_cert): self-signed RSA 2048 for 825 days, naming the +// admin console, the panel and localhost, and the addresses the old one named. +func issuePanelCert(n domainNames, ips []net.IP, now time.Time) (certPEM, keyPEM []byte, err error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 127)) + if err != nil { + return nil, nil, err + } + var dns []string + for _, h := range []string{n.admin, n.panel, "localhost"} { + if !containsString(dns, h) { + dns = append(dns, h) + } + } + addrs := []net.IP{net.IPv4(127, 0, 0, 1)} + for _, ip := range ips { + dup := false + for _, a := range addrs { + dup = dup || a.Equal(ip) + } + if !dup { + addrs = append(addrs, ip) + } + } + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: n.admin}, + NotBefore: now.Add(-time.Minute), + NotAfter: now.AddDate(0, 0, panelCertDays), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: dns, + IPAddresses: addrs, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return nil, nil, err + } + pkcs8, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, nil, err + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), nil +} + +func containsString(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +// certAction is what `set` does with the panel certificate. +type certAction int + +const ( + certKeep certAction = iota // it already covers the new names + certReissue // Felis made it (or there is none): make a new one + certForeign // the operator's, and it does not cover the new names +) + +type domainPaths struct { + hostTOML, podTOML, defaultTOML string + cert, key string + linkProps string + tunnelConfig string +} + +// unitStatus is what systemd reports about the proxy unit. +type unitStatus struct { + loaded, active bool + since time.Time // when it last became active; zero when unknown +} + +// liveAPIView is what the running felis-api serves: its /config.json names and +// the certificate it presents. +type liveAPIView struct { + names domainNames + cert *x509.Certificate +} + +type domainHost struct { + paths domainPaths + cl client.Client + controlNS string + rollAPI func(ctx context.Context) error + restartUnit func(ctx context.Context, unit string) error + unitState func(ctx context.Context, unit string) (unitStatus, error) + liveAPI func(ctx context.Context, serverName string) (liveAPIView, error) + lookupHost func(ctx context.Context, host string) ([]string, error) + // passkeys counts the registered passkeys and the users holding them. + passkeys func(ctx context.Context) (creds, users int, err error) + now func() time.Time + out io.Writer + loginWait time.Duration + pollEvery time.Duration +} + +func cmdDomain(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 || (args[0] != "set" && args[0] != "check") { + fmt.Fprintln(stderr, domainUsage) + fmt.Fprintln(stderr, "Moves the install to a new root domain on every surface that carries it, or checks each of them.") + return 2 + } + sub := args[0] + fs := flag.NewFlagSet("domain "+sub, flag.ContinueOnError) + fs.SetOutput(stderr) + yes := false + if sub == "set" { + fs.BoolVar(&yes, "yes", false, "apply the change; without it the plan is printed and nothing changes") + } + fs.Usage = func() { + fmt.Fprintln(stderr, domainUsage) + fs.PrintDefaults() + } + if err := fs.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if (sub == "set" && fs.NArg() != 1) || (sub == "check" && fs.NArg() != 0) { + fs.Usage() + return 2 + } + if os.Geteuid() != 0 { + fmt.Fprintf(stderr, "felis domain: refused — it reads the cluster, the panel key and the proxy's config, so it must run as root (try: sudo felis domain %s)\n", strings.Join(args, " ")) + return 1 + } + cl, err := buildSystemServerClient() + if err != nil { + fmt.Fprintf(stderr, "felis domain: %v\n", err) + return 1 + } + h := newDomainHost(cl, stdout) + ctx := context.Background() + if sub == "check" { + return h.check(ctx) + } + code, err := h.set(ctx, fs.Arg(0), yes) + if err != nil { + fmt.Fprintf(stderr, "felis domain set: %v\n", err) + return 1 + } + return code +} + +func newDomainHost(cl client.Client, out io.Writer) domainHost { + controlNS := platform.DefaultControlNamespace + return domainHost{ + paths: domainPaths{ + hostTOML: hostSetupConfigPath, podTOML: podSetupConfigPath, defaultTOML: defaultSetupConfigPath, + cert: "/etc/felis/panel-tls.crt", key: "/etc/felis/panel-tls.key", + linkProps: defaultLinkPropsPath, tunnelConfig: defaultTunnelConfigPath, + }, + cl: cl, + controlNS: controlNS, + rollAPI: func(ctx context.Context) error { + if err := kubectl(ctx, "-n", controlNS, "rollout", "restart", "deployment/felis-api"); err != nil { + return err + } + return kubectl(ctx, "-n", controlNS, "rollout", "status", "deployment/felis-api", "--timeout=180s") + }, + restartUnit: func(ctx context.Context, unit string) error { return systemctl(ctx, "restart", unit) }, + unitState: systemdUnitState, + liveAPI: func(ctx context.Context, serverName string) (liveAPIView, error) { + return fetchLiveAPI(ctx, fmt.Sprintf("https://127.0.0.1:%d/config.json", setupPanelNodePort()), serverName) + }, + lookupHost: net.DefaultResolver.LookupHost, + passkeys: func(ctx context.Context) (int, int, error) { + cfg, err := config.Load(hostSetupConfigPath) + if err != nil { + return 0, 0, err + } + return countPasskeys(ctx, cfg.Database.URL) + }, + now: time.Now, + out: out, + loginWait: 3 * time.Minute, + pollEvery: 3 * time.Second, + } +} + +func countPasskeys(ctx context.Context, url string) (int, int, error) { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + drv, err := store.Open(ctx, url) + if err != nil { + return 0, 0, err + } + defer drv.Close() + var creds, users int + err = drv.DB().QueryRowContext(ctx, `SELECT count(*), count(DISTINCT user_id) FROM webauthn_credentials`).Scan(&creds, &users) + return creds, users, err +} + +// systemdUnitState reads LoadState, ActiveState and when the unit last became +// active. +func systemdUnitState(ctx context.Context, unit string) (unitStatus, error) { + out, err := exec.CommandContext(ctx, "systemctl", "show", "--timestamp=unix", + "-p", "LoadState", "-p", "ActiveState", "-p", "ActiveEnterTimestamp", unit).Output() + if err != nil { + return unitStatus{}, fmt.Errorf("systemctl show %s: %w", unit, err) + } + return parseUnitShow(string(out)), nil +} + +func parseUnitShow(out string) unitStatus { + var st unitStatus + for _, ln := range strings.Split(out, "\n") { + k, v, _ := strings.Cut(strings.TrimSpace(ln), "=") + switch k { + case "LoadState": + st.loaded = v == "loaded" + case "ActiveState": + st.active = v == "active" + case "ActiveEnterTimestamp": + if sec, err := strconv.ParseInt(strings.TrimPrefix(v, "@"), 10, 64); err == nil && sec > 0 { + st.since = time.Unix(sec, 0) + } + } + } + return st +} + +// fetchLiveAPI reads what felis-api serves on the panel port. The panel +// certificate is self-signed, so verification is skipped: this reads the +// certificate to check its names, it does not trust it. +func fetchLiveAPI(ctx context.Context, url, serverName string) (liveAPIView, error) { + c := &http.Client{Timeout: 10 * time.Second, Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true, ServerName: serverName}, // #nosec G402 -- inspected, not trusted + }} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return liveAPIView{}, err + } + resp, err := c.Do(req) + if err != nil { + return liveAPIView{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return liveAPIView{}, fmt.Errorf("GET %s: %s", url, resp.Status) + } + var rc struct { + RootDomain string `json:"rootDomain"` + PanelHostname string `json:"panelHostname"` + AdminHostname string `json:"adminHostname"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&rc); err != nil { + return liveAPIView{}, fmt.Errorf("GET %s: %w", url, err) + } + v := liveAPIView{names: domainNames{root: rc.RootDomain, panel: rc.PanelHostname, admin: rc.AdminHostname}} + if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 { + v.cert = resp.TLS.PeerCertificates[0] + } + return v, nil +} + +// tomlTarget is a config file carrying the domain: path as configured, and real +// with links resolved, so a rewrite replaces the file and keeps the link. +type tomlTarget struct{ path, real string } + +// tomlTargets are the host and pod copies, and felis.toml when it is a file of +// its own rather than the link to the host copy. +func (h domainHost) tomlTargets() ([]tomlTarget, error) { + var out []tomlTarget + seen := map[string]bool{} + for i, p := range []string{h.paths.hostTOML, h.paths.podTOML, h.paths.defaultTOML} { + real, err := filepath.EvalSymlinks(p) + if errors.Is(err, fs.ErrNotExist) && i == 2 { + continue + } + if err != nil { + return nil, err + } + if !seen[real] { + seen[real] = true + out = append(out, tomlTarget{p, real}) + } + } + return out, nil +} + +// certPlan decides what happens to the panel certificate for these names. +func (h domainHost) certPlan(to domainNames) (certAction, *x509.Certificate, error) { + c, err := readCertFile(h.paths.cert) + if errors.Is(err, fs.ErrNotExist) { + return certReissue, nil, nil + } + if err != nil { + return 0, nil, err + } + switch { + case certCovers(c, to.panel, to.admin): + return certKeep, c, nil + case felisIssuedCert(c): + return certReissue, c, nil + default: + return certForeign, c, nil + } +} + +func (h domainHost) set(ctx context.Context, arg string, apply bool) (int, error) { + cfg, err := config.Load(h.paths.hostTOML) + if err != nil { + return 1, err + } + newRoot, err := normalizeRootDomain(arg) + if err != nil { + return 1, err + } + cur := effectiveDomainNames(cfg.Server.RootDomain, cfg.Auth.PanelHostname, cfg.Auth.AdminHostname) + plan := planDomainChange(cur, newRoot) + + // Everything that can refuse runs before anything is written. + targets, err := h.tomlTargets() + if err != nil { + return 1, err + } + edited := make(map[string][]byte, len(targets)) + for _, t := range targets { + raw, err := os.ReadFile(t.real) + if err != nil { + return 1, err + } + out, err := editTOMLStrings(raw, domainTOMLEdits(plan.to)) + if err != nil { + return 1, fmt.Errorf("%s: %w; set [server] root_domain and [auth] panel_hostname / admin_hostname there by hand, then run this again", t.path, err) + } + if !bytes.Equal(raw, out) { + edited[t.real] = out + } + } + action, oldCert, err := h.certPlan(plan.to) + if err != nil { + return 1, fmt.Errorf("read the panel certificate: %w", err) + } + + h.printPlan(ctx, plan, action, cfg.SMTP.Host != "") + if action == certForeign { + return 1, fmt.Errorf("the panel certificate at %s was not issued by Felis and does not cover %s and %s; install one that does at the same path (key at %s), then run this again", + h.paths.cert, plan.to.panel, plan.to.admin, h.paths.key) + } + if !apply { + fmt.Fprintf(h.out, "\nNothing was changed. To apply: sudo felis domain set -yes %s\n", plan.to.root) + return 0, nil + } + + fmt.Fprintln(h.out, "\nApplying:") + for _, t := range targets { + out, ok := edited[t.real] + if !ok { + fmt.Fprintf(h.out, " - %s: already on %s\n", t.path, plan.to.root) + continue + } + info, err := os.Stat(t.real) + if err != nil { + return 1, err + } + if err := replaceFileKeepingMode(t.real, info, out); err != nil { + return 1, fmt.Errorf("write %s: %w", t.path, err) + } + fmt.Fprintf(h.out, " - %s: updated\n", t.path) + } + + if action == certReissue { + if err := h.reissueCert(plan.to, oldCert); err != nil { + return 1, err + } + } else { + fmt.Fprintf(h.out, " - panel certificate: already covers %s and %s\n", plan.to.panel, plan.to.admin) + } + + secretsChanged, err := h.syncSecrets(ctx, cfg.K8s.Namespace) + if err != nil { + return 1, err + } + login, err := h.convergeLogin(ctx, cfg, plan.to) + if err != nil { + return 1, err + } + propsChanged, hostProxy, err := h.syncLinkProps(plan.to) + if err != nil { + return 1, err + } + + roll := secretsChanged + if !roll { + live, err := h.liveAPI(ctx, plan.to.panel) + roll = err != nil || live.names != plan.to || live.cert == nil || !certCovers(live.cert, plan.to.panel, plan.to.admin) + } + if roll { + if err := h.rollAPI(ctx); err != nil { + return 1, fmt.Errorf("roll felis-api: %w", err) + } + fmt.Fprintln(h.out, " - felis-api: rolled out on the new config and certificate (everyone signs in again)") + } else { + fmt.Fprintln(h.out, " - felis-api: already serving the new names") + } + + if hostProxy { + if err := h.restartProxyIfStale(ctx, propsChanged); err != nil { + return 1, err + } + } + if login { + h.awaitLogin(ctx, cfg.K8s.Namespace, plan.to) + } + + fmt.Fprintln(h.out) + return h.check(ctx), nil +} + +func (h domainHost) printPlan(ctx context.Context, p domainPlan, action certAction, smtp bool) { + if p.from == p.to { + fmt.Fprintf(h.out, "felis domain set: the install is already on %s; bringing every surface in line with it.\n", p.to.root) + } else { + fmt.Fprintf(h.out, "felis domain set: moving the install from %s to %s\n", p.from.root, p.to.root) + } + row := func(label, from, to string, custom bool) { + switch { + case custom: + fmt.Fprintf(h.out, " %-14s %s (set by hand, kept; change [auth] in %s yourself if it should move)\n", label, to, h.paths.hostTOML) + case from == to: + fmt.Fprintf(h.out, " %-14s %s\n", label, to) + default: + fmt.Fprintf(h.out, " %-14s %s → %s\n", label, from, to) + } + } + row("root domain", p.from.root, p.to.root, false) + row("panel", p.from.panel, p.to.panel, p.customPanel) + row("admin console", p.from.admin, p.to.admin, p.customAdmin) + switch action { + case certKeep: + fmt.Fprintf(h.out, " %-14s already covers the names, kept\n", "certificate") + case certReissue: + fmt.Fprintf(h.out, " %-14s reissued for the names (self-signed, as the installer makes it); the old pair is kept beside it\n", "certificate") + case certForeign: + fmt.Fprintf(h.out, " %-14s NOT issued by Felis and does not cover the names\n", "certificate") + } + fmt.Fprintln(h.out, " also: the felis-config and felis-api-tls Secrets, the proxy's felis-link.properties, the login gate's env") + if p.from == p.to { + return + } + + fmt.Fprintln(h.out, "\nWhat the move does:") + fmt.Fprintf(h.out, " - DNS: %s, %s, %s and *.%s must reach this host. The *.%s wildcard does not cover %s (a third-level name): it needs its own record.\n", + p.to.root, p.to.panel, p.to.admin, p.to.root, p.to.root, p.to.admin) + fmt.Fprintf(h.out, " - Players reach servers as .%s from now on; the %s addresses stop routing. Restarting the proxy disconnects everyone online.\n", p.to.root, p.from.root) + fmt.Fprintln(h.out, " - Everyone signs in again on the new address: sign-in cookies belong to the old hostname.") + if p.from.panel != p.to.panel { + switch creds, users, err := h.passkeys(ctx); { + case err != nil: + fmt.Fprintf(h.out, " - Passkeys are bound to %s and stop working on %s (could not count them: %v).\n", p.from.panel, p.to.panel, err) + case creds > 0: + fmt.Fprintf(h.out, " - %d passkey(s) of %d user(s) are bound to %s and stop working on %s: those users sign in with their email code and register a new passkey.\n", + creds, users, p.from.panel, p.to.panel) + } + if !smtp { + fmt.Fprintln(h.out, " No [smtp] relay is configured, so email codes are not delivered: an Owner locked out this way recovers with sudo felis breakGlass.") + } + } + if _, err := os.Stat(h.paths.tunnelConfig); err == nil { + fmt.Fprintln(h.out, " - The Cloudflare tunnel and Access application still route the old names: re-run the Cloudflare step of sudo felis setup afterwards.") + } +} + +// reissueCert writes a new panel certificate and key for n, keeping the old pair +// beside them. +func (h domainHost) reissueCert(n domainNames, old *x509.Certificate) error { + var ips []net.IP + if old != nil { + ips = old.IPAddresses + } + certPEM, keyPEM, err := issuePanelCert(n, ips, h.now()) + if err != nil { + return fmt.Errorf("issue the panel certificate: %w", err) + } + stamp := h.now().UTC().Format("20060102T150405Z") + kept := "" + for _, f := range []struct { + path string + data []byte + mode os.FileMode + }{{h.paths.key, keyPEM, 0o600}, {h.paths.cert, certPEM, 0o644}} { + info, err := os.Stat(f.path) + switch { + case errors.Is(err, fs.ErrNotExist): + if err := os.WriteFile(f.path, f.data, f.mode); err != nil { + return err + } + continue + case err != nil: + return err + } + prev, err := os.ReadFile(f.path) + if err != nil { + return err + } + backup := f.path + ".pre-domain-" + stamp + if err := replaceFileKeepingMode(backup, info, prev); err != nil { + return fmt.Errorf("keep %s: %w", f.path, err) + } + kept = ".pre-domain-" + stamp + if err := replaceFileKeepingMode(f.path, info, f.data); err != nil { + return fmt.Errorf("write %s: %w", f.path, err) + } + } + if kept != "" { + fmt.Fprintf(h.out, " - panel certificate: reissued for %s and %s (the old pair is kept as *%s)\n", n.panel, n.admin, kept) + } else { + fmt.Fprintf(h.out, " - panel certificate: issued for %s and %s\n", n.panel, n.admin) + } + return nil +} + +// putSecretKeys sets keys of a Secret, creating it with typ when absent, and +// reports whether anything changed. Keys not named are left alone. +func putSecretKeys(ctx context.Context, cl client.Client, ns, name string, typ corev1.SecretType, data map[string][]byte) (bool, error) { + var sec corev1.Secret + err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &sec) + if apierrors.IsNotFound(err) { + return true, cl.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name}, Type: typ, Data: data}) + } + if err != nil { + return false, err + } + changed := false + for k, v := range data { + if !bytes.Equal(sec.Data[k], v) { + changed = true + } + } + if !changed { + return false, nil + } + if sec.Data == nil { + sec.Data = map[string][]byte{} + } + for k, v := range data { + sec.Data[k] = v + } + return true, cl.Update(ctx, &sec) +} + +// syncSecrets puts the pod config and the panel certificate into the Secrets +// felis-api (and the workload-namespace Jobs) mount. +func (h domainHost) syncSecrets(ctx context.Context, minecraftNS string) (bool, error) { + pod, err := os.ReadFile(h.paths.podTOML) + if err != nil { + return false, err + } + certPEM, err := os.ReadFile(h.paths.cert) + if err != nil { + return false, err + } + keyPEM, err := os.ReadFile(h.paths.key) + if err != nil { + return false, err + } + changedAny := false + put := func(ns, name string, typ corev1.SecretType, data map[string][]byte) error { + changed, err := putSecretKeys(ctx, h.cl, ns, name, typ, data) + if err != nil { + return fmt.Errorf("write Secret %s/%s: %w", ns, name, err) + } + changedAny = changedAny || changed + state := "unchanged" + if changed { + state = "updated" + } + fmt.Fprintf(h.out, " - Secret %s/%s: %s\n", ns, name, state) + return nil + } + for _, ns := range h.configNamespaces(minecraftNS) { + if err := put(ns, platform.ConfigSecretName, corev1.SecretTypeOpaque, map[string][]byte{platform.ConfigSecretKey: pod}); err != nil { + return false, err + } + } + if err := put(h.controlNS, platform.APITLSSecretName, corev1.SecretTypeTLS, + map[string][]byte{corev1.TLSCertKey: certPEM, corev1.TLSPrivateKeyKey: keyPEM}); err != nil { + return false, err + } + return changedAny, nil +} + +func (h domainHost) configNamespaces(minecraftNS string) []string { + ns := []string{h.controlNS} + if minecraftNS != "" && minecraftNS != h.controlNS { + ns = append(ns, minecraftNS) + } + return ns +} + +// convergeLogin sets the config-derived env of the system servers (the login +// gate carries the domain) and nothing else, and reports whether the login gate +// is installed. +func (h domainHost) convergeLogin(ctx context.Context, cfg *config.Config, n domainNames) (bool, error) { + ns := cfg.K8s.Namespace + login := false + for _, p := range systemServerPlans(cfg.Velocity.LoginImage, cfg.Velocity.LobbyImage, platform.InternalAPIBaseURL(h.controlNS), n.root, n.panel) { + if p.image == "" { + continue + } + desired, err := p.build(p.image, ns) + if err != nil { + return false, err + } + if len(derivedEnvWanted(desired)) == 0 { + continue + } + var existing v1alpha1.MinecraftServer + switch err := h.cl.Get(ctx, client.ObjectKeyFromObject(desired), &existing); { + case apierrors.IsNotFound(err): + fmt.Fprintf(h.out, " - %s: not installed, skipped\n", p.name) + continue + case err != nil: + return false, err + } + if existing.Labels[v1alpha1.LabelSystemRole] != p.name { + return false, fmt.Errorf("MinecraftServer %s/%s is not marked as the Felis %q system role; refusing to change it", ns, p.name, p.name) + } + var changes []string + changed, err := patchOnConflictRetry(ctx, h.cl, &existing, func() bool { + changes = convergeDerivedEnv(&existing, desired) + return len(changes) > 0 + }) + if err != nil { + return false, fmt.Errorf("update %s: %w", p.name, err) + } + login = login || p.name == naming.SystemLoginServer + if changed { + fmt.Fprintf(h.out, " - %s: updated (%s)\n", p.name, strings.Join(changes, ", ")) + } else { + fmt.Fprintf(h.out, " - %s: env already on the new names\n", p.name) + } + } + return login, nil +} + +func linkPropsDomain(n domainNames) [][2]string { + return [][2]string{{"root-domain", n.root}, {"panel-hostname", n.panel}, {"admin-hostname", n.admin}} +} + +// readKeyValues reads the named keys of a key=value file, and only those: the +// proxy's file also holds its service token. +func readKeyValues(path string, keys ...string) (map[string]string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, ln := range strings.Split(string(raw), "\n") { + k, v, ok := strings.Cut(ln, "=") + if k = strings.TrimSpace(k); ok && containsString(keys, k) { + out[k] = strings.TrimSpace(v) + } + } + return out, nil +} + +// syncLinkProps writes the names into the host proxy's felis-link.properties. +// hostProxy is false when the proxy runs elsewhere. +func (h domainHost) syncLinkProps(n domainNames) (changed, hostProxy bool, err error) { + want := linkPropsDomain(n) + keys := make([]string, len(want)) + for i, kv := range want { + keys[i] = kv[0] + } + have, err := readKeyValues(h.paths.linkProps, keys...) + if errors.Is(err, fs.ErrNotExist) { + fmt.Fprintf(h.out, " - %s: not on this host; in your proxy's felis-link.properties set root-domain=%s, panel-hostname=%s, admin-hostname=%s and restart it\n", + h.paths.linkProps, n.root, n.panel, n.admin) + return false, false, nil + } + if err != nil { + return false, false, err + } + var stale [][2]string + for _, kv := range want { + if have[kv[0]] != kv[1] { + stale = append(stale, kv) + } + } + if len(stale) == 0 { + fmt.Fprintf(h.out, " - %s: already on the new names\n", h.paths.linkProps) + return false, true, nil + } + if err := setKeyValueLines(h.paths.linkProps, "=", stale); err != nil { + return false, true, fmt.Errorf("write %s: %w", h.paths.linkProps, err) + } + fmt.Fprintf(h.out, " - %s: updated\n", h.paths.linkProps) + return true, true, nil +} + +// restartProxyIfStale restarts the host proxy when its config changed or it has +// been running since before the last change. +func (h domainHost) restartProxyIfStale(ctx context.Context, changed bool) error { + st, err := h.unitState(ctx, velocityUnit) + if err != nil { + return err + } + if !st.loaded { + fmt.Fprintf(h.out, " - %s: no such unit on this host; restart your proxy so it reads felis-link.properties\n", velocityUnit) + return nil + } + if !st.active { + fmt.Fprintf(h.out, " - %s: not running; it reads the new names when it starts\n", velocityUnit) + return nil + } + if !changed { + info, err := os.Stat(h.paths.linkProps) + if err != nil { + return err + } + if !st.since.IsZero() && !st.since.Before(info.ModTime()) { + fmt.Fprintf(h.out, " - %s: already running on the new names\n", velocityUnit) + return nil + } + } + if err := h.restartUnit(ctx, velocityUnit); err != nil { + return fmt.Errorf("restart %s: %w", velocityUnit, err) + } + fmt.Fprintf(h.out, " - %s: restarted (players online were disconnected and reconnect on the new addresses)\n", velocityUnit) + return nil +} + +// loginPodState reports whether the login gate's pod runs with n and is Ready. +func (h domainHost) loginPodState(ctx context.Context, ns string, n domainNames) (envOK, ready bool, err error) { + var pod corev1.Pod + if err := h.cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: naming.SystemLoginServer + "-0"}, &pod); err != nil { + return false, false, err + } + env := map[string]string{} + for _, c := range pod.Spec.Containers { + if c.Name == "minecraft" { + for _, e := range c.Env { + env[e.Name] = e.Value + } + } + } + envOK = env[envRootDomain] == n.root && env[envPanelHostname] == n.panel + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady { + ready = c.Status == corev1.ConditionTrue + } + } + return envOK, ready, nil +} + +// awaitLogin waits for the operator to restart the login gate onto the new env. +func (h domainHost) awaitLogin(ctx context.Context, ns string, n domainNames) { + deadline := h.now().Add(h.loginWait) + for { + if envOK, ready, err := h.loginPodState(ctx, ns, n); err == nil && envOK && ready { + fmt.Fprintln(h.out, " - login gate: running on the new names") + return + } + if !h.now().Before(deadline) { + fmt.Fprintf(h.out, " - login gate: not running on the new names after %s (the check below says where it stands)\n", h.loginWait) + return + } + select { + case <-ctx.Done(): + return + case <-time.After(h.pollEvery): + } + } +} + +// domainCheck is one surface's line in `felis domain check`. +type domainCheck struct { + status string // ok, FAIL, warn, or "-" (not on this host) + surface string + detail string +} + +func (h domainHost) check(ctx context.Context) int { + cfg, err := config.Load(h.paths.hostTOML) + if err != nil { + fmt.Fprintf(h.out, "felis domain check: %v\n", err) + return 1 + } + want := effectiveDomainNames(cfg.Server.RootDomain, cfg.Auth.PanelHostname, cfg.Auth.AdminHostname) + fmt.Fprintf(h.out, "felis domain check: every surface against %s (panel %s, admin console %s)\n", want.root, want.panel, want.admin) + checks := h.checks(ctx, cfg, want) + failed := 0 + for _, c := range checks { + fmt.Fprintf(h.out, " %-4s %s: %s\n", c.status, c.surface, c.detail) + if c.status == "FAIL" { + failed++ + } + } + if failed > 0 { + fmt.Fprintf(h.out, "%d surface(s) are behind; sudo felis domain set %s converges what it owns, and each line above says what else to do.\n", failed, want.root) + return 1 + } + fmt.Fprintf(h.out, "Every surface is on %s.\n", want.root) + return 0 +} + +func namesDetail(n domainNames) string { + return fmt.Sprintf("root %s, panel %s, admin %s", n.root, n.panel, n.admin) +} + +func (h domainHost) checks(ctx context.Context, cfg *config.Config, want domainNames) []domainCheck { + var out []domainCheck + add := func(status, surface, format string, a ...any) { + out = append(out, domainCheck{status, surface, fmt.Sprintf(format, a...)}) + } + match := func(surface string, got domainNames) { + if got == want { + add("ok", surface, "on the names") + } else { + add("FAIL", surface, "has %s", namesDetail(got)) + } + } + + targets, err := h.tomlTargets() + if err != nil { + add("FAIL", "config files", "%v", err) + } + for _, t := range targets { + raw, err := os.ReadFile(t.real) + if err == nil { + var got domainNames + if got, err = tomlDomainNames(raw); err == nil { + match(t.path, got) + continue + } + } + add("FAIL", t.path, "%v", err) + } + + for _, ns := range h.configNamespaces(cfg.K8s.Namespace) { + surface := "Secret " + ns + "/" + platform.ConfigSecretName + var sec corev1.Secret + if err := h.cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: platform.ConfigSecretName}, &sec); err != nil { + add("FAIL", surface, "%v", err) + continue + } + got, err := tomlDomainNames(sec.Data[platform.ConfigSecretKey]) + if err != nil { + add("FAIL", surface, "%v", err) + continue + } + match(surface, got) + } + + certOK := false + switch c, err := readCertFile(h.paths.cert); { + case err != nil: + add("FAIL", "panel certificate", "%v", err) + case !certCovers(c, want.panel, want.admin): + add("FAIL", "panel certificate", "%s names %s, not both %s and %s", h.paths.cert, strings.Join(c.DNSNames, ", "), want.panel, want.admin) + default: + certOK = true + add("ok", "panel certificate", "covers both names, valid until %s", c.NotAfter.UTC().Format("2006-01-02")) + } + var tlsSec corev1.Secret + certPEM, _ := os.ReadFile(h.paths.cert) + keyPEM, _ := os.ReadFile(h.paths.key) + switch err := h.cl.Get(ctx, client.ObjectKey{Namespace: h.controlNS, Name: platform.APITLSSecretName}, &tlsSec); { + case err != nil: + add("FAIL", "Secret "+h.controlNS+"/"+platform.APITLSSecretName, "%v", err) + case !bytes.Equal(tlsSec.Data[corev1.TLSCertKey], certPEM) || !bytes.Equal(tlsSec.Data[corev1.TLSPrivateKeyKey], keyPEM): + add("FAIL", "Secret "+h.controlNS+"/"+platform.APITLSSecretName, "differs from %s / %s", h.paths.cert, h.paths.key) + default: + add("ok", "Secret "+h.controlNS+"/"+platform.APITLSSecretName, "matches the certificate files") + } + + switch live, err := h.liveAPI(ctx, want.panel); { + case err != nil: + add("FAIL", "felis-api", "%v", err) + case live.names != want: + add("FAIL", "felis-api", "serves %s (still on the old config: roll it)", namesDetail(live.names)) + case live.cert == nil || !certCovers(live.cert, want.panel, want.admin): + add("FAIL", "felis-api", "serves the names but a certificate that does not cover them (roll it after the Secret is right)") + case certOK && !bytes.Equal(live.cert.Raw, mustCertDER(certPEM)): + add("warn", "felis-api", "serves the names with a certificate other than %s", h.paths.cert) + default: + add("ok", "felis-api", "serves the names and a certificate that covers them") + } + + h.checkProxy(ctx, want, add) + + var ms v1alpha1.MinecraftServer + switch err := h.cl.Get(ctx, client.ObjectKey{Namespace: cfg.K8s.Namespace, Name: naming.SystemLoginServer}, &ms); { + case apierrors.IsNotFound(err): + add("-", "login gate", "not installed") + case err != nil: + add("FAIL", "login gate", "%v", err) + default: + env := map[string]string{} + for _, e := range ms.Spec.Env { + env[e.Name] = e.Value + } + if env[envRootDomain] != want.root || env[envPanelHostname] != want.panel { + add("FAIL", "login gate", "the MinecraftServer env has %s=%q, %s=%q", envRootDomain, env[envRootDomain], envPanelHostname, env[envPanelHostname]) + break + } + switch envOK, ready, err := h.loginPodState(ctx, cfg.K8s.Namespace, want); { + case err != nil: + add("FAIL", "login gate", "env is right; its pod: %v", err) + case !envOK: + add("FAIL", "login gate", "env is right but the pod still runs the old one (the operator has not restarted it yet)") + case !ready: + add("warn", "login gate", "the pod has the new env and is not Ready yet") + default: + add("ok", "login gate", "env and running pod on the names") + } + } + + h.checkTunnel(want, add) + h.checkDNS(ctx, want, add) + return out +} + +func mustCertDER(certPEM []byte) []byte { + if b, _ := pem.Decode(certPEM); b != nil { + return b.Bytes + } + return nil +} + +func (h domainHost) checkProxy(ctx context.Context, want domainNames, add func(status, surface, format string, a ...any)) { + keys := []string{"root-domain", "panel-hostname", "admin-hostname"} + have, err := readKeyValues(h.paths.linkProps, keys...) + if errors.Is(err, fs.ErrNotExist) { + add("-", "proxy", "no felis-link.properties on this host; a proxy elsewhere needs root-domain=%s, panel-hostname=%s, admin-hostname=%s", want.root, want.panel, want.admin) + return + } + if err != nil { + add("FAIL", "proxy", "%v", err) + return + } + for _, kv := range linkPropsDomain(want) { + if have[kv[0]] != kv[1] { + add("FAIL", "proxy", "%s has %s=%q", h.paths.linkProps, kv[0], have[kv[0]]) + return + } + } + info, err := os.Stat(h.paths.linkProps) + if err != nil { + add("FAIL", "proxy", "%v", err) + return + } + switch st, err := h.unitState(ctx, velocityUnit); { + case err != nil: + add("warn", "proxy", "felis-link.properties is right; could not ask systemd about %s: %v", velocityUnit, err) + case !st.loaded: + add("warn", "proxy", "felis-link.properties is right; no %s unit here, so restart your proxy if it has not been", velocityUnit) + case !st.active: + add("FAIL", "proxy", "felis-link.properties is right; %s is not running", velocityUnit) + case st.since.IsZero(): + add("warn", "proxy", "felis-link.properties is right; could not tell when %s started", velocityUnit) + case st.since.Before(info.ModTime()): + add("FAIL", "proxy", "%s has run since before felis-link.properties changed: sudo systemctl restart %s", velocityUnit, velocityUnit) + default: + add("ok", "proxy", "felis-link.properties on the names, %s started after it changed", velocityUnit) + } +} + +func (h domainHost) checkTunnel(want domainNames, add func(status, surface, format string, a ...any)) { + raw, err := os.ReadFile(h.paths.tunnelConfig) + if errors.Is(err, fs.ErrNotExist) { + add("-", "Cloudflare tunnel", "not set up on this host") + return + } + if err != nil { + add("FAIL", "Cloudflare tunnel", "%v", err) + return + } + var tc struct { + Ingress []struct { + Hostname string `json:"hostname"` + } `json:"ingress"` + } + if err := yaml.Unmarshal(raw, &tc); err != nil { + add("FAIL", "Cloudflare tunnel", "%s: %v", h.paths.tunnelConfig, err) + return + } + var routed []string + for _, r := range tc.Ingress { + if r.Hostname != "" { + routed = append(routed, r.Hostname) + } + } + for _, host := range []string{want.panel, want.admin} { + if !containsString(routed, host) { + add("FAIL", "Cloudflare tunnel", "routes %s, not %s: re-run the Cloudflare step of sudo felis setup", strings.Join(routed, ", "), host) + return + } + } + add("ok", "Cloudflare tunnel", "routes both names") +} + +func (h domainHost) checkDNS(ctx context.Context, want domainNames, add func(status, surface, format string, a ...any)) { + var missing []string + for _, host := range []string{want.root, want.panel, want.admin, dnsProbeLabel + "." + want.root} { + lctx, cancel := context.WithTimeout(ctx, 5*time.Second) + _, err := h.lookupHost(lctx, host) + cancel() + if err != nil { + if strings.HasPrefix(host, dnsProbeLabel+".") { + host = "*." + want.root + } + missing = append(missing, host) + } + } + if len(missing) > 0 { + verb := "does not resolve" + if len(missing) > 1 { + verb = "do not resolve" + } + // The admin name is the one a wildcard-only zone misses; say why only then. + note := "" + if containsString(missing, want.admin) && !containsString(missing, "*."+want.root) { + note = fmt.Sprintf(": the *.%s wildcard does not cover %s, which needs its own record", want.root, want.admin) + } + add("warn", "DNS", "%s %s from this host%s", strings.Join(missing, ", "), verb, note) + return + } + add("ok", "DNS", "the root, both hostnames and *.%s resolve", want.root) +} diff --git a/cmd/felis/domain_test.go b/cmd/felis/domain_test.go new file mode 100644 index 0000000..3016b8f --- /dev/null +++ b/cmd/felis/domain_test.go @@ -0,0 +1,892 @@ +package main + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "math/big" + "net" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "felis.lolicon.best/internal/apis/felis/v1alpha1" + "felis.lolicon.best/internal/naming" + "felis.lolicon.best/internal/platform" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +// installerTOML is felis.toml as deploy/bootstrap.sh write_felis_toml renders it, +// comments included: the domain move has to leave all of it but three values alone. +func installerTOML(root, dbHost string) string { + return `# Generated by deploy/bootstrap.sh; rerun the installer to regenerate. Hand edits are +# overwritten, except [smtp], [[auth_source]], [offsite], and the operator-owned +# [registry] / [archive] overrides, which carry forward. +[server] +listen = "0.0.0.0:8080" +root_domain = "` + root + `" + +[database] +url = "postgres://felis:pw@` + dbHost + `:5432/felis?sslmode=disable" + +[k8s] +namespace = "minecraft" +egress_mode = "nodeport" + +[velocity] +# The two always-on system servers that felis setup provisions. +login_image = "felis/limbo:1" +lobby_image = "felis/lobby:1" +game_port = 25565 + +[registry] +url = "registry.felis.svc:5000" +build_namespace = "felis-build" + +[archive] +store = "tarLocal" +local_path = "/var/lib/felis/archives" + +[auth] +admin_hostname = "op.console.` + root + `" +panel_hostname = "console.` + root + `" +access_jwt_aud = "aud123" + +# Third-party Yggdrasil sources federated by the hasJoined multiplexer. +[[auth_source]] +tag = "littleskin" +prefix = "LS" +url = "https://littleskin.cn/api/yggdrasil/sessionserver/session/minecraft/hasJoined" +` +} + +const linkPropsBody = `# Generated by deploy/bootstrap.sh — do not edit by hand; rerun the installer. +api-base-url=http://10.43.0.9:8081 +service-token=TOKEN-NOT-TO-TOUCH +root-domain=old.example +panel-hostname=console.old.example +admin-hostname=op.console.old.example +login-server=login +lobby-server=lobby +` + +var oldNames = domainNames{root: "old.example", panel: "console.old.example", admin: "op.console.old.example"} +var newNames = domainNames{root: "new.example", panel: "console.new.example", admin: "op.console.new.example"} + +// domainRig models the host: the files, the cluster, and a felis-api, proxy and +// operator that pick up config the way the real ones do — the api serves what it +// read at its last restart, the proxy runs since its last restart, and the login +// pod carries the env its MinecraftServer had when it was last rolled. +type domainRig struct { + h domainHost + cl client.Client + out *bytes.Buffer + dir string + events []string + + served domainNames + servedCert *x509.Certificate + proxySince time.Time + proxyLoaded bool + unresolved map[string]bool + // The fake operator: the CR env the login pod was last rolled to, and how + // many looks at the pod since the CR moved on. + rolledTo string + pending int +} + +func (rig *domainRig) path(name string) string { return filepath.Join(rig.dir, name) } + +func newDomainRig(t *testing.T) *domainRig { + t.Helper() + rig := &domainRig{out: &bytes.Buffer{}, dir: t.TempDir(), proxyLoaded: true, unresolved: map[string]bool{}} + writeTestFile(t, rig.path("felis.host.toml"), installerTOML("old.example", "127.0.0.1"), 0o600) + writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600) + if err := os.Symlink(rig.path("felis.host.toml"), rig.path("felis.toml")); err != nil { + t.Fatal(err) + } + certPEM, keyPEM, err := issuePanelCert(oldNames, []net.IP{net.ParseIP("10.211.55.6")}, time.Now()) + if err != nil { + t.Fatal(err) + } + writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644) + writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600) + writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640) + // The proxy started before its config was last written, which is how it + // stands after an install. + rig.proxySince = time.Now().Add(-time.Hour) + + pod := []byte(installerTOML("old.example", "10.211.55.6")) + login, err := loginSystemServer("felis/limbo:1", "minecraft", platform.InternalAPIBaseURL("felis"), oldNames.root, oldNames.panel) + if err != nil { + t.Fatal(err) + } + lobby, err := lobbySystemServer("felis/lobby:1", "minecraft") + if err != nil { + t.Fatal(err) + } + loginPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "minecraft", Env: podEnv(login.Spec.Env)}}}, + Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}, + } + rig.rolledTo = envKey(login.Spec.Env) + rig.cl = fake.NewClientBuilder().WithScheme(haltScheme(t)).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Name == naming.SystemLoginServer+"-0" { + rig.operatorTick(t, c) + } + return c.Get(ctx, key, obj, opts...) + }, + }).WithObjects( + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.ConfigSecretName}, + Data: map[string][]byte{platform.ConfigSecretKey: pod}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "minecraft", Name: platform.ConfigSecretName}, + Data: map[string][]byte{platform.ConfigSecretKey: pod}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "felis", Name: platform.APITLSSecretName}, + Type: corev1.SecretTypeTLS, Data: map[string][]byte{corev1.TLSCertKey: certPEM, corev1.TLSPrivateKeyKey: keyPEM}}, + login, lobby, loginPod, + ).Build() + rig.served = oldNames + rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM)) + + rig.h = domainHost{ + paths: domainPaths{ + hostTOML: rig.path("felis.host.toml"), podTOML: rig.path("felis.pod.toml"), defaultTOML: rig.path("felis.toml"), + cert: rig.path("panel-tls.crt"), key: rig.path("panel-tls.key"), + linkProps: rig.path("felis-link.properties"), tunnelConfig: rig.path("cloudflared.yml"), + }, + cl: rig.cl, + controlNS: "felis", + rollAPI: func(ctx context.Context) error { + rig.events = append(rig.events, "roll-api") + rig.restartAPI(t) + return nil + }, + restartUnit: func(_ context.Context, unit string) error { + rig.events = append(rig.events, "restart "+unit) + rig.proxySince = time.Now().Add(time.Second) + return nil + }, + unitState: func(context.Context, string) (unitStatus, error) { + return unitStatus{loaded: rig.proxyLoaded, active: rig.proxyLoaded, since: rig.proxySince}, nil + }, + liveAPI: func(context.Context, string) (liveAPIView, error) { + return liveAPIView{names: rig.served, cert: rig.servedCert}, nil + }, + lookupHost: func(_ context.Context, host string) ([]string, error) { + if rig.unresolved[host] { + return nil, errors.New("no such host") + } + return []string{"10.211.55.6"}, nil + }, + passkeys: func(context.Context) (int, int, error) { return 3, 2, nil }, + now: time.Now, + out: rig.out, + loginWait: 50 * time.Millisecond, + pollEvery: time.Millisecond, + } + return rig +} + +func podEnv(env []v1alpha1.EnvVar) []corev1.EnvVar { + out := make([]corev1.EnvVar, len(env)) + for i, e := range env { + out[i] = corev1.EnvVar{Name: e.Name, Value: e.Value} + } + return out +} + +// restartAPI makes the fake felis-api load the config and certificate its +// Secrets hold now. +func (rig *domainRig) restartAPI(t *testing.T) { + t.Helper() + var cfg, tlsSec corev1.Secret + ctx := context.Background() + if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.ConfigSecretName}, &cfg); err != nil { + t.Fatal(err) + } + if err := rig.cl.Get(ctx, client.ObjectKey{Namespace: "felis", Name: platform.APITLSSecretName}, &tlsSec); err != nil { + t.Fatal(err) + } + names, err := tomlDomainNames(cfg.Data[platform.ConfigSecretKey]) + if err != nil { + t.Fatal(err) + } + rig.served = names + rig.servedCert, err = x509.ParseCertificate(mustCertDER(tlsSec.Data[corev1.TLSCertKey])) + if err != nil { + t.Fatal(err) + } +} + +// rollLoginPod is the operator restarting the login pod onto its CR's env. +// operatorTick is the operator as the login pod is watched: once the CR's env +// changes it takes operatorLag looks at the pod before the restarted pod +// carries the new env, the way a real rollout lags the CR. +func (rig *domainRig) operatorTick(t *testing.T, c client.Client) { + t.Helper() + ctx := context.Background() + var ms v1alpha1.MinecraftServer + if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil { + t.Fatal(err) + } + if envKey(ms.Spec.Env) == rig.rolledTo { + return + } + if rig.pending++; rig.pending < operatorLag { + return + } + var pod corev1.Pod + if err := c.Get(ctx, client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer + "-0"}, &pod); err != nil { + t.Fatal(err) + } + pod.Spec.Containers[0].Env = podEnv(ms.Spec.Env) + if err := c.Update(ctx, &pod); err != nil { + t.Fatal(err) + } + rig.rolledTo, rig.pending = envKey(ms.Spec.Env), 0 +} + +const operatorLag = 3 + +func envKey(env []v1alpha1.EnvVar) string { + var b strings.Builder + for _, e := range env { + b.WriteString(e.Name + "=" + e.Value + "\n") + } + return b.String() +} + +func (rig *domainRig) read(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(rig.path(name)) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func (rig *domainRig) secret(t *testing.T, ns, name string) map[string][]byte { + t.Helper() + var s corev1.Secret + if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil { + t.Fatal(err) + } + return s.Data +} + +func (rig *domainRig) crEnv(t *testing.T, name string) map[string]string { + t.Helper() + var ms v1alpha1.MinecraftServer + if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: name}, &ms); err != nil { + t.Fatal(err) + } + env := map[string]string{} + for _, e := range ms.Spec.Env { + env[e.Name] = e.Value + } + return env +} + +// snapshot is every byte `set` may touch, for proving a refused or dry run +// touched none of it. +func (rig *domainRig) snapshot(t *testing.T) string { + t.Helper() + var b strings.Builder + entries, _ := os.ReadDir(rig.dir) + for _, e := range entries { + b.WriteString(e.Name() + "\n" + rig.read(t, e.Name()) + "\n") + } + var lines []string + for _, s := range []struct{ ns, name string }{{"felis", platform.ConfigSecretName}, {"minecraft", platform.ConfigSecretName}, {"felis", platform.APITLSSecretName}} { + for k, v := range rig.secret(t, s.ns, s.name) { + lines = append(lines, s.ns+"/"+s.name+"/"+k+"\n"+string(v)) + } + } + for k, v := range rig.crEnv(t, naming.SystemLoginServer) { + lines = append(lines, "env "+k+"="+v) + } + sort.Strings(lines) + b.WriteString(strings.Join(lines, "\n")) + return b.String() +} + +func TestNormalizeRootDomain(t *testing.T) { + for in, want := range map[string]string{ + "Example.COM.": "example.com", + " mc.example.org ": "mc.example.org", + "10.211.55.6.nip.io": "10.211.55.6.nip.io", + "xn--bcher-kva.example": "xn--bcher-kva.example", + } { + got, err := normalizeRootDomain(in) + if err != nil || got != want { + t.Errorf("normalizeRootDomain(%q) = %q, %v; want %q", in, got, err, want) + } + } + for _, in := range []string{"", "https://example.com", "example.com:443", "example.com/x", "10.0.0.1", "::1", + "localhost", "a_b.example", "-a.example", "a-.example", strings.Repeat("a", 64) + ".example", + strings.Repeat("abcdefghi.", 25) + "example"} { + if got, err := normalizeRootDomain(in); err == nil { + t.Errorf("normalizeRootDomain(%q) = %q, want an error", in, got) + } + } +} + +func TestPlanDomainChangeMovesDefaultsAndKeepsHandSetNames(t *testing.T) { + p := planDomainChange(oldNames, "new.example") + if p.to != newNames || p.customPanel || p.customAdmin { + t.Fatalf("defaults: %+v", p) + } + p = planDomainChange(domainNames{root: "old.example", panel: "play.corp.net", admin: "op.console.old.example"}, "new.example") + if p.to.panel != "play.corp.net" || !p.customPanel || p.to.admin != "op.console.new.example" || p.customAdmin { + t.Fatalf("hand-set panel: %+v", p) + } + p = planDomainChange(domainNames{root: "old.example", panel: "console.old.example", admin: "admin.corp.net"}, "new.example") + if p.to.admin != "admin.corp.net" || !p.customAdmin || p.to.panel != "console.new.example" { + t.Fatalf("hand-set admin: %+v", p) + } +} + +func TestEditTOMLStringsChangesOnlyTheDomainLines(t *testing.T) { + orig := installerTOML("old.example", "127.0.0.1") + out, err := editTOMLStrings([]byte(orig), domainTOMLEdits(newNames)) + if err != nil { + t.Fatal(err) + } + a, b := strings.Split(orig, "\n"), strings.Split(string(out), "\n") + if len(a) != len(b) { + t.Fatalf("line count %d → %d:\n%s", len(a), len(b), out) + } + changed := map[string]string{} + for i := range a { + if a[i] != b[i] { + changed[a[i]] = b[i] + } + } + want := map[string]string{ + `root_domain = "old.example"`: `root_domain = "new.example"`, + `admin_hostname = "op.console.old.example"`: `admin_hostname = "op.console.new.example"`, + `panel_hostname = "console.old.example"`: `panel_hostname = "console.new.example"`, + } + if len(changed) != len(want) { + t.Fatalf("changed lines %v, want %v", changed, want) + } + for k, v := range want { + if changed[k] != v { + t.Errorf("%q → %q, want %q", k, changed[k], v) + } + } +} + +func TestEditTOMLStringsAddsMissingKeysInTheirTable(t *testing.T) { + in := "[server]\nroot_domain = \"old.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\n\n[smtp]\nhost = \"relay\"\n" + out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames)) + if err != nil { + t.Fatal(err) + } + want := "[server]\nroot_domain = \"new.example\"\n\n[auth]\naccess_jwt_aud = \"x\"\npanel_hostname = \"console.new.example\"\nadmin_hostname = \"op.console.new.example\"\n\n[smtp]\nhost = \"relay\"\n" + if string(out) != want { + t.Fatalf("got:\n%s\nwant:\n%s", out, want) + } + + out, err = editTOMLStrings([]byte("[server]\nroot_domain = \"old.example\"\n\n[[auth_source]]\ntag = \"ls\"\n"), domainTOMLEdits(newNames)) + if err != nil { + t.Fatal(err) + } + got, err := tomlDomainNames(out) + if err != nil || got != newNames || !strings.Contains(string(out), "[[auth_source]]\ntag = \"ls\"\n") { + t.Fatalf("no [auth] table: %v %+v\n%s", err, got, out) + } +} + +func TestEditTOMLStringsRefusesWhatItCannotEditExactly(t *testing.T) { + for name, in := range map[string]string{ + "multi-line value": "[server]\nroot_domain = \"\"\"\nold.example\"\"\"\n[auth]\n", + // The key's line sits inside another value; the real key is absent. + "key inside a string": "[server]\nmotd = \"\"\"\nroot_domain = \"old.example\"\n\"\"\"\n[auth]\n", + "quoted header": "[server]\nroot_domain = \"old.example\"\n[\"auth\"]\npanel_hostname = \"console.old.example\"\n", + "dotted key": "server.root_domain = \"old.example\"\n", + "inline table": "server = { root_domain = \"old.example\" }\n", + } { + if out, err := editTOMLStrings([]byte(in), domainTOMLEdits(newNames)); err == nil { + t.Errorf("%s: edited instead of refusing:\n%s", name, out) + } + } +} + +// caSignedCert is an operator's certificate from their own CA; it names +// localhost too, so only the issuer tells it apart from the installer's. +func caSignedCert(t *testing.T, hosts ...string) (certPEM, keyPEM []byte) { + t.Helper() + caKey, _ := rsa.GenerateKey(rand.Reader, 2048) + ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "Corp CA"}, IsCA: true, + BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + caCert, _ := x509.ParseCertificate(caDER) + key, _ := rsa.GenerateKey(rand.Reader, 2048) + leaf := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: hosts[0]}, + DNSNames: append(hosts, "localhost"), IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour)} + der, err := x509.CreateCertificate(rand.Reader, leaf, caCert, &key.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + pk, _ := x509.MarshalPKCS8PrivateKey(key) + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pk}) +} + +func TestFelisIssuedCert(t *testing.T) { + mine, _, err := issuePanelCert(oldNames, nil, time.Now()) + if err != nil { + t.Fatal(err) + } + c, _ := x509.ParseCertificate(mustCertDER(mine)) + if !felisIssuedCert(c) { + t.Error("the installer's kind of certificate is not recognised as Felis-issued") + } + theirs, _ := caSignedCert(t, "console.old.example") + c, _ = x509.ParseCertificate(mustCertDER(theirs)) + if felisIssuedCert(c) { + t.Error("a CA-signed certificate is taken for Felis-issued") + } + + // Self-signed but without one of the installer's localhost names: someone + // else's. + key, _ := rsa.GenerateKey(rand.Reader, 2048) + for name, self := range map[string]*x509.Certificate{ + "no localhost": {DNSNames: []string{"console.old.example"}, IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}}, + "no 127.0.0.1": {DNSNames: []string{"console.old.example", "localhost"}}, + } { + self.SerialNumber, self.Subject = big.NewInt(3), pkix.Name{CommonName: "x"} + self.NotBefore, self.NotAfter = time.Now().Add(-time.Hour), time.Now().Add(time.Hour) + der, _ := x509.CreateCertificate(rand.Reader, self, self, &key.PublicKey, key) + c, _ = x509.ParseCertificate(der) + if felisIssuedCert(c) { + t.Errorf("%s: a self-signed certificate is taken for Felis-issued", name) + } + } +} + +func TestIssuePanelCertIsTheInstallersShape(t *testing.T) { + now := time.Now() + certPEM, keyPEM, err := issuePanelCert(newNames, []net.IP{net.ParseIP("10.211.55.6"), net.IPv4(127, 0, 0, 1)}, now) + if err != nil { + t.Fatal(err) + } + if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil { + t.Fatalf("key does not match the certificate: %v", err) + } + if b, _ := pem.Decode(keyPEM); b == nil || b.Type != "PRIVATE KEY" { + t.Fatalf("key is not PKCS#8 PEM like openssl writes") + } + c, _ := x509.ParseCertificate(mustCertDER(certPEM)) + if !certCovers(c, newNames.panel, newNames.admin, "localhost") || !felisIssuedCert(c) { + t.Fatalf("names %v", c.DNSNames) + } + if len(c.IPAddresses) != 2 || !c.IPAddresses[1].Equal(net.ParseIP("10.211.55.6")) { + t.Fatalf("addresses %v, want 127.0.0.1 and the node address once each", c.IPAddresses) + } + if c.Subject.CommonName != newNames.admin || c.NotAfter.Sub(now) < 824*24*time.Hour || c.NotAfter.Sub(now) > 826*24*time.Hour { + t.Fatalf("CN %q, valid until %s", c.Subject.CommonName, c.NotAfter) + } + if len(c.ExtKeyUsage) != 1 || c.ExtKeyUsage[0] != x509.ExtKeyUsageServerAuth || c.IsCA { + t.Fatalf("usage %v, CA %v", c.ExtKeyUsage, c.IsCA) + } +} + +func TestDomainSetMovesEverySurface(t *testing.T) { + rig := newDomainRig(t) + hostBefore := rig.read(t, "felis.host.toml") + code, err := rig.h.set(context.Background(), "New.Example", true) + if err != nil || code != 0 { + t.Fatalf("set = %d, %v\n%s", code, err, rig.out) + } + + // The configs: the three values moved, everything else — comments, the + // database host of each copy, the Access audience — is as it was. + host := rig.read(t, "felis.host.toml") + if want := strings.NewReplacer("old.example", "new.example").Replace(hostBefore); host != want { + t.Fatalf("host toml:\n%s", host) + } + pod := rig.read(t, "felis.pod.toml") + if got, _ := tomlDomainNames([]byte(pod)); got != newNames || !strings.Contains(pod, "@10.211.55.6:5432") { + t.Fatalf("pod toml:\n%s", pod) + } + if link, err := os.Readlink(rig.path("felis.toml")); err != nil || link != rig.path("felis.host.toml") { + t.Fatalf("felis.toml is no longer the link to the host copy: %q %v", link, err) + } + if st, _ := os.Stat(rig.path("felis.host.toml")); st.Mode().Perm() != 0o600 { + t.Fatalf("host toml mode %v", st.Mode().Perm()) + } + + // The certificate: reissued for the new names, the node address kept, the old + // pair beside it. + c, err := readCertFile(rig.path("panel-tls.crt")) + if err != nil || !certCovers(c, newNames.panel, newNames.admin) || !felisIssuedCert(c) { + t.Fatalf("certificate: %v %v", err, c.DNSNames) + } + if !c.IPAddresses[len(c.IPAddresses)-1].Equal(net.ParseIP("10.211.55.6")) { + t.Fatalf("addresses %v", c.IPAddresses) + } + if _, err := tls.LoadX509KeyPair(rig.path("panel-tls.crt"), rig.path("panel-tls.key")); err != nil { + t.Fatalf("new pair: %v", err) + } + if st, _ := os.Stat(rig.path("panel-tls.key")); st.Mode().Perm() != 0o600 { + t.Fatalf("key mode %v", st.Mode().Perm()) + } + backups, _ := filepath.Glob(rig.path("panel-tls.*.pre-domain-*")) + if len(backups) != 2 { + t.Fatalf("old pair kept as %v", backups) + } + for _, b := range backups { + if st, _ := os.Stat(b); strings.Contains(b, ".key.") && st.Mode().Perm() != 0o600 { + t.Fatalf("kept key %s has mode %v", b, st.Mode().Perm()) + } + old, _ := os.ReadFile(b) + if strings.Contains(b, ".crt.") { + oc, _ := x509.ParseCertificate(mustCertDER(old)) + if oc == nil || !certCovers(oc, oldNames.panel) { + t.Fatalf("kept certificate is not the old one") + } + } + } + + // The Secrets carry the files. + for _, ns := range []string{"felis", "minecraft"} { + if got := rig.secret(t, ns, platform.ConfigSecretName)[platform.ConfigSecretKey]; string(got) != pod { + t.Fatalf("%s/felis-config is not felis.pod.toml", ns) + } + } + tlsData := rig.secret(t, "felis", platform.APITLSSecretName) + if string(tlsData[corev1.TLSCertKey]) != rig.read(t, "panel-tls.crt") || string(tlsData[corev1.TLSPrivateKeyKey]) != rig.read(t, "panel-tls.key") { + t.Fatal("felis-api-tls does not hold the new pair") + } + + // The login gate's env moved and nothing else did. + env := rig.crEnv(t, naming.SystemLoginServer) + if env[envRootDomain] != newNames.root || env[envPanelHostname] != newNames.panel || env[envAPIBaseURL] != platform.InternalAPIBaseURL("felis") { + t.Fatalf("login env %v", env) + } + + // The proxy's file: the three keys moved, its token and mode did not. + props := rig.read(t, "felis-link.properties") + if want := strings.NewReplacer("old.example", "new.example").Replace(linkPropsBody); props != want { + t.Fatalf("felis-link.properties:\n%s", props) + } + if st, _ := os.Stat(rig.path("felis-link.properties")); st.Mode().Perm() != 0o640 { + t.Fatalf("felis-link.properties mode %v", st.Mode().Perm()) + } + + if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" { + t.Fatalf("events %v", rig.events) + } + if !strings.Contains(rig.out.String(), "Every surface is on new.example.") { + t.Fatalf("the closing check did not pass:\n%s", rig.out) + } +} + +func TestDomainSetWithoutYesChangesNothing(t *testing.T) { + rig := newDomainRig(t) + before := rig.snapshot(t) + code, err := rig.h.set(context.Background(), "new.example", false) + if err != nil || code != 0 { + t.Fatalf("set = %d, %v", code, err) + } + if rig.snapshot(t) != before || len(rig.events) != 0 { + t.Fatalf("a dry run changed something (events %v)", rig.events) + } + out := rig.out.String() + for _, want := range []string{ + "console.old.example → console.new.example", + "3 passkey(s) of 2 user(s) are bound to console.old.example", + "does not cover op.console.new.example", + "No [smtp] relay is configured", + "sudo felis domain set -yes new.example", + } { + if !strings.Contains(out, want) { + t.Errorf("plan lacks %q:\n%s", want, out) + } + } +} + +func TestDomainSetKeepsAHandSetPanelHostname(t *testing.T) { + rig := newDomainRig(t) + for _, f := range []string{"felis.host.toml", "felis.pod.toml"} { + writeTestFile(t, rig.path(f), strings.Replace(rig.read(t, f), `panel_hostname = "console.old.example"`, `panel_hostname = "play.corp.net"`, 1), 0o600) + } + code, err := rig.h.set(context.Background(), "new.example", true) + if err != nil { + t.Fatalf("set: %v\n%s", err, rig.out) + } + got, _ := tomlDomainNames([]byte(rig.read(t, "felis.host.toml"))) + if got != (domainNames{root: "new.example", panel: "play.corp.net", admin: "op.console.new.example"}) { + t.Fatalf("names %+v", got) + } + c, _ := readCertFile(rig.path("panel-tls.crt")) + if !certCovers(c, "play.corp.net", "op.console.new.example") { + t.Fatalf("certificate names %v", c.DNSNames) + } + if env := rig.crEnv(t, naming.SystemLoginServer); env[envPanelHostname] != "play.corp.net" { + t.Fatalf("login env %v", env) + } + if code != 0 || !strings.Contains(rig.out.String(), "play.corp.net (set by hand, kept") { + t.Fatalf("code %d:\n%s", code, rig.out) + } +} + +func TestDomainSetRefusesAnOperatorCertificateForOtherNames(t *testing.T) { + rig := newDomainRig(t) + certPEM, keyPEM := caSignedCert(t, "console.old.example", "op.console.old.example") + writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644) + writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600) + before := rig.snapshot(t) + if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "not issued by Felis") { + t.Fatalf("err = %v", err) + } + if rig.snapshot(t) != before || len(rig.events) != 0 { + t.Fatal("a refused move changed something") + } + + // The operator's certificate for the new names is kept as it is. + certPEM, keyPEM = caSignedCert(t, "console.new.example", "op.console.new.example") + writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644) + writeTestFile(t, rig.path("panel-tls.key"), string(keyPEM), 0o600) + if _, err := rig.h.set(context.Background(), "new.example", true); err != nil { + t.Fatalf("set: %v", err) + } + if rig.read(t, "panel-tls.crt") != string(certPEM) { + t.Fatal("the operator's certificate was replaced") + } +} + +func TestDomainSetRefusesAConfigItCannotEdit(t *testing.T) { + rig := newDomainRig(t) + writeTestFile(t, rig.path("felis.pod.toml"), strings.Replace(rig.read(t, "felis.pod.toml"), "[auth]", "[\"auth\"]", 1), 0o600) + before := rig.snapshot(t) + if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "by hand") { + t.Fatalf("err = %v", err) + } + if rig.snapshot(t) != before || len(rig.events) != 0 { + t.Fatal("a refused move changed something") + } +} + +func TestDomainSetAgainOnlyConvergesWhatIsBehind(t *testing.T) { + rig := newDomainRig(t) + if _, err := rig.h.set(context.Background(), "new.example", true); err != nil { + t.Fatal(err) + } + rig.events, rig.out = nil, &bytes.Buffer{} + rig.h.out = rig.out + before := rig.snapshot(t) + code, err := rig.h.set(context.Background(), "new.example", true) + if err != nil || code != 0 { + t.Fatalf("second set = %d, %v\n%s", code, err, rig.out) + } + if len(rig.events) != 0 || rig.snapshot(t) != before { + t.Fatalf("a converged install was touched again: %v\n%s", rig.events, rig.out) + } + + // A proxy that was not restarted after the move is restarted by a re-run, and + // an api still on the old config is rolled. + rig.proxySince = time.Now().Add(-time.Hour) + rig.served = oldNames + if _, err := rig.h.set(context.Background(), "new.example", true); err != nil { + t.Fatal(err) + } + if strings.Join(rig.events, ",") != "roll-api,restart felis-velocity" { + t.Fatalf("events %v", rig.events) + } + + // An api on the new names that still presents the old certificate is rolled. + rig.events = nil + oldCert, _, _ := issuePanelCert(oldNames, nil, time.Now()) + rig.servedCert, _ = x509.ParseCertificate(mustCertDER(oldCert)) + if _, err := rig.h.set(context.Background(), "new.example", true); err != nil { + t.Fatal(err) + } + if strings.Join(rig.events, ",") != "roll-api" { + t.Fatalf("events %v", rig.events) + } +} + +func TestDomainSetRefusesALoginServerItDoesNotOwn(t *testing.T) { + rig := newDomainRig(t) + var ms v1alpha1.MinecraftServer + if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms); err != nil { + t.Fatal(err) + } + delete(ms.Labels, v1alpha1.LabelSystemRole) + if err := rig.cl.Update(context.Background(), &ms); err != nil { + t.Fatal(err) + } + if _, err := rig.h.set(context.Background(), "new.example", true); err == nil || !strings.Contains(err.Error(), "system role") { + t.Fatalf("err = %v", err) + } + if env := rig.crEnv(t, naming.SystemLoginServer); env[envRootDomain] != oldNames.root { + t.Fatalf("a server not marked as the login gate was changed: %v", env) + } +} + +func TestDomainCheckNamesTheSurfaceThatIsBehind(t *testing.T) { + cases := []struct { + surface string + breakIt func(t *testing.T, rig *domainRig) + }{ + {"felis.pod.toml", func(t *testing.T, rig *domainRig) { + writeTestFile(t, rig.path("felis.pod.toml"), installerTOML("old.example", "10.211.55.6"), 0o600) + }}, + {"Secret minecraft/felis-config", func(t *testing.T, rig *domainRig) { + rig.putSecret(t, "minecraft", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x")) + }}, + {"Secret felis/felis-config", func(t *testing.T, rig *domainRig) { + rig.putSecret(t, "felis", platform.ConfigSecretName, platform.ConfigSecretKey, installerTOML("old.example", "x")) + }}, + {"panel certificate", func(t *testing.T, rig *domainRig) { + // The Secret follows the file, so only the certificate's names are wrong. + certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now()) + writeTestFile(t, rig.path("panel-tls.crt"), string(certPEM), 0o644) + rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM)) + }}, + {"Secret felis/felis-api-tls", func(t *testing.T, rig *domainRig) { + certPEM, _, _ := issuePanelCert(newNames, nil, time.Now()) + rig.putSecret(t, "felis", platform.APITLSSecretName, corev1.TLSCertKey, string(certPEM)) + }}, + {"felis-api", func(t *testing.T, rig *domainRig) { rig.served = oldNames }}, + {"felis-api", func(t *testing.T, rig *domainRig) { + certPEM, _, _ := issuePanelCert(oldNames, nil, time.Now()) + rig.servedCert, _ = x509.ParseCertificate(mustCertDER(certPEM)) + }}, + {"proxy", func(t *testing.T, rig *domainRig) { + writeTestFile(t, rig.path("felis-link.properties"), linkPropsBody, 0o640) + rig.proxySince = time.Now().Add(time.Hour) + }}, + {"proxy", func(t *testing.T, rig *domainRig) { rig.proxySince = time.Now().Add(-time.Hour) }}, + {"login gate", func(t *testing.T, rig *domainRig) { + var ms v1alpha1.MinecraftServer + _ = rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: naming.SystemLoginServer}, &ms) + for i := range ms.Spec.Env { + if ms.Spec.Env[i].Name == envRootDomain { + ms.Spec.Env[i].Value = "old.example" + } + } + if err := rig.cl.Update(context.Background(), &ms); err != nil { + t.Fatal(err) + } + }}, + {"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envRootDomain, "old.example") }}, + {"login gate", func(t *testing.T, rig *domainRig) { rig.setPodEnv(t, envPanelHostname, "console.old.example") }}, + {"Cloudflare tunnel", func(t *testing.T, rig *domainRig) { + writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n service: https://127.0.0.1:30443\n- hostname: op.console.old.example\n service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644) + }}, + } + for _, tc := range cases { + rig := newDomainRig(t) + if _, err := rig.h.set(context.Background(), "new.example", true); err != nil { + t.Fatal(err) + } + rig.out.Reset() + tc.breakIt(t, rig) + if code := rig.h.check(context.Background()); code != 1 { + t.Errorf("%s behind: check = %d\n%s", tc.surface, code, rig.out) + continue + } + var failed []string + for _, ln := range strings.Split(rig.out.String(), "\n") { + if strings.HasPrefix(ln, " FAIL ") { + failed = append(failed, ln) + } + } + if len(failed) != 1 || !strings.Contains(failed[0], tc.surface) { + t.Errorf("%s behind: FAIL lines %q", tc.surface, failed) + } + } +} + +func TestDomainCheckPassesAConvergedInstallAndWarnsOnDNS(t *testing.T) { + rig := newDomainRig(t) + if _, err := rig.h.set(context.Background(), "new.example", true); err != nil { + t.Fatal(err) + } + writeTestFile(t, rig.path("cloudflared.yml"), "tunnel: abc\ningress:\n- hostname: console.new.example\n service: https://127.0.0.1:30443\n- hostname: op.console.new.example\n service: https://127.0.0.1:30443\n- service: http_status:404\n", 0o644) + rig.unresolved["op.console.new.example"] = true + rig.unresolved[dnsProbeLabel+".new.example"] = true + rig.out.Reset() + if code := rig.h.check(context.Background()); code != 0 { + t.Fatalf("check = %d\n%s", code, rig.out) + } + out := rig.out.String() + for _, want := range []string{" ok Cloudflare tunnel: routes both names", " warn DNS: op.console.new.example, *.new.example do not resolve from this host\n"} { + if !strings.Contains(out, want) { + t.Errorf("check lacks %q:\n%s", want, out) + } + } + if strings.Contains(out, "TOKEN-NOT-TO-TOUCH") { + t.Fatal("check printed the proxy's service token") + } + + // A zone with only the wildcard: the admin name alone is missing, and why is said. + delete(rig.unresolved, dnsProbeLabel+".new.example") + rig.out.Reset() + rig.h.check(context.Background()) + if want := " warn DNS: op.console.new.example does not resolve from this host: the *.new.example wildcard does not cover op.console.new.example, which needs its own record\n"; !strings.Contains(rig.out.String(), want) { + t.Errorf("check lacks %q:\n%s", want, rig.out) + } +} + +func (rig *domainRig) setPodEnv(t *testing.T, name, value string) { + t.Helper() + var pod corev1.Pod + if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: "minecraft", Name: "login-0"}, &pod); err != nil { + t.Fatal(err) + } + for i, e := range pod.Spec.Containers[0].Env { + if e.Name == name { + pod.Spec.Containers[0].Env[i].Value = value + } + } + if err := rig.cl.Update(context.Background(), &pod); err != nil { + t.Fatal(err) + } +} + +func (rig *domainRig) putSecret(t *testing.T, ns, name, key, val string) { + t.Helper() + var s corev1.Secret + if err := rig.cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name}, &s); err != nil { + t.Fatal(err) + } + s.Data[key] = []byte(val) + if err := rig.cl.Update(context.Background(), &s); err != nil { + t.Fatal(err) + } +} + +func TestParseUnitShow(t *testing.T) { + st := parseUnitShow("LoadState=loaded\nActiveState=active\nActiveEnterTimestamp=@1790000000\n") + if !st.loaded || !st.active || !st.since.Equal(time.Unix(1790000000, 0)) { + t.Fatalf("%+v", st) + } + st = parseUnitShow("LoadState=not-found\nActiveState=inactive\nActiveEnterTimestamp=\n") + if st.loaded || st.active || !st.since.IsZero() { + t.Fatalf("%+v", st) + } +} diff --git a/cmd/felis/rotatetoken.go b/cmd/felis/rotatetoken.go index 4a72034..9c3e44b 100644 --- a/cmd/felis/rotatetoken.go +++ b/cmd/felis/rotatetoken.go @@ -259,6 +259,11 @@ func writeTokenSecret(ctx context.Context, cl client.Client, namespace, name, to // file is replaced atomically and keeps its mode and owner: felis-link.properties // is root:felis-velocity 0640, and the proxy must still be able to read it. func setKeyValueLine(path, key, sep, value string) error { + return setKeyValueLines(path, sep, [][2]string{{key, value}}) +} + +// setKeyValueLines is setKeyValueLine for several keys in one rewrite. +func setKeyValueLines(path, sep string, kv [][2]string) error { info, err := os.Stat(path) if err != nil { return err @@ -268,17 +273,27 @@ func setKeyValueLine(path, key, sep, value string) error { return err } lines := strings.Split(strings.TrimRight(string(raw), "\n"), "\n") - found := false - for i, ln := range lines { - k, _, ok := strings.Cut(ln, sep) - if ok && strings.TrimSpace(k) == key { - lines[i] = key + sep + value - found = true + for _, p := range kv { + key, value := p[0], p[1] + found := false + for i, ln := range lines { + k, _, ok := strings.Cut(ln, sep) + if ok && strings.TrimSpace(k) == key { + lines[i] = key + sep + value + found = true + } + } + if !found { + lines = append(lines, key+sep+value) } } - if !found { - lines = append(lines, key+sep+value) - } + return replaceFileKeepingMode(path, info, []byte(strings.Join(lines, "\n")+"\n")) +} + +// replaceFileKeepingMode atomically replaces path with data, keeping the mode and +// owner info describes: these files are read by other users (the proxy's) and +// some hold credentials, so a rewrite must not widen or narrow who can read them. +func replaceFileKeepingMode(path string, info os.FileInfo, data []byte) error { tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*") if err != nil { return err @@ -294,7 +309,7 @@ func setKeyValueLine(path, key, sep, value string) error { return err } } - if _, err := tmp.WriteString(strings.Join(lines, "\n") + "\n"); err != nil { + if _, err := tmp.Write(data); err != nil { tmp.Close() return err } diff --git a/cmd/felis/run.go b/cmd/felis/run.go index 3b1aced..f8ff00e 100644 --- a/cmd/felis/run.go +++ b/cmd/felis/run.go @@ -32,6 +32,7 @@ Commands: setup Run host bootstrap + first-run setup console (TUI; requires root/sudo) converge Fill in fields a newer desired spec added to already-installed system servers rotate-token Replace one internal caller's token and restart what holds it (velocity|limbo|build|ops; requires root/sudo) + domain Move the install to a new root domain on every surface that carries it, or check each one (set|check; requires root/sudo) watchdog Check the platform once and mail the owners what has gone wrong (run by felis-watchdog.timer) version Print the build stamp of this binary update Report which platform components have updates available @@ -71,6 +72,7 @@ var commands = map[string]func(args []string, stdout, stderr io.Writer) int{ "setup": cmdSetup, "converge": cmdConverge, "rotate-token": cmdRotateToken, + "domain": cmdDomain, "breakGlass": cmdBreakGlass, "bootstrap-assets": cmdBootstrapAssets, "init-forwarding": cmdInitForwarding, diff --git a/docs/operations.md b/docs/operations.md index ef49937..aaa694d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -444,3 +444,65 @@ production install: then log in and restore one world. `felis offsite status` and `felis db check` exit non-zero when the copy or the newest bundle is stale; wire them into your monitoring, or rely on the watchdog's mail. + +## 6. Changing the root domain [VM-VERIFIED] [GO-TESTED] [SH-TESTED] + +The root domain is written into more places than the installer's config: the panel +certificate (`/etc/felis/panel-tls.crt`), the `felis-config` Secret in both namespaces, +the `felis-api-tls` Secret, the proxy's `felis-link.properties`, the login gate's +`MinecraftServer` env (`FELIS_ROOT_DOMAIN`, `FELIS_PANEL_HOSTNAME`), the Cloudflare tunnel +and DNS. `felis domain set` moves every one of them that lives on the host, in that +order, then restarts what reads them; `felis domain check` reports each surface on its +own line. The installer keeps the installed domain: a rerun with a different +`FELIS_ROOT_DOMAIN` stops and names this command. + +```sh +sudo felis domain set new.example.net # the plan: every surface, what it moves to, what it costs +sudo felis domain set -yes new.example.net # do it +sudo felis domain check # one line per surface; exits 1 while any is behind +``` + +What it keeps: + +- A panel or admin-console hostname set by hand in `[auth]` (anything other than + `console.` / `op.console.`) stays as it is; change it in + `/etc/felis/felis.host.toml` yourself if it should move, then run `set` again. +- The other `[auth]` keys (`access_jwt_aud`, `client_ip_header`) and every other line of + both config files. The edit refuses a file it cannot change line for line (a multi-line + value, a quoted or dotted key) and names what to fix. +- An operator's own certificate. The installer's self-signed certificate is reissued for + the new names (same shape, the old pair saved beside it as `*.pre-domain-