Felis/cmd/felis/watchdog_heartbeat.go

package main

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"felis.lolicon.best/internal/offsite"
)

// defaultHeartbeatFile holds the heartbeat URL deploy/bootstrap.sh writes from
// FELIS_WATCHDOG_HEARTBEAT_URL. It lives in /etc/felis, so a host rebuilt from
// a bundle pings the same check once it takes the off-site bucket over.
const defaultHeartbeatFile = "/etc/felis/watchdog-heartbeat-url"

const (
	// heartbeatTimeout bounds one ping; a monitoring service slower than that
	// is as good as down for the run.
	heartbeatTimeout = 10 * time.Second
	// heartbeatReportMax caps the report a failure ping carries: monitoring
	// services keep about the first 10 KB of a ping's body.
	heartbeatReportMax = 10000
)

// heartbeat is the ping one run sends to a dead man's switch at a monitoring
// service (Healthchecks.io, and the services that copy its API), which alerts
// its own users when the pings stop: the host down, the timer gone, the
// watchdog failing before it can mail. No check on the host can report those.
// A run whose alerts reach the owners GETs url; one whose alerts reach no one
// POSTs report to url/fail, or withholds the ping when url has a query, where
// no /fail can be added and the missing ping trips the check instead.
type heartbeat struct {
	url    string // "" pings nothing
	fail   bool
	report string
	// standby is a host standing by for another host's off-site bucket: a
	// rehearsal, or a rebuild not taken over. It pings nothing, or it would
	// keep the check of the host that writes the bucket green after that host
	// died.
	standby bool
	// quiet is the installer's quiet window, which restarts things on
	// purpose: no failure is pinged.
	quiet bool
}

// send pings the heartbeat and logs the outcome.
func (b heartbeat) send(cl *http.Client, stdout, stderr io.Writer) {
	switch {
	case b.url == "":
		return
	case b.standby:
		fmt.Fprintln(stdout, "felis watchdog: this host stands by for the off-site bucket; pinging no heartbeat (the host that writes the bucket pings it)")
		return
	case b.fail && b.quiet:
		fmt.Fprintln(stdout, "felis watchdog: quiet while the installer runs; withholding the failure ping")
		return
	}
	sent, err := b.ping(cl)
	switch {
	case err != nil:
		fmt.Fprintf(stderr, "felis watchdog: heartbeat: %v\n", err)
	case !sent:
		fmt.Fprintf(stdout, "felis watchdog: withholding the heartbeat ping (the URL has a query, so it has no /fail endpoint)\n")
	case b.fail:
		fmt.Fprintf(stdout, "felis watchdog: pinged the heartbeat's failure endpoint at %s\n", redactURL(b.url))
	}
}

// ping sends the request; sent is false when a failure withholds it.
func (b heartbeat) ping(cl *http.Client) (sent bool, err error) {
	ctx, cancel := context.WithTimeout(context.Background(), heartbeatTimeout)
	defer cancel()
	method, target, body := http.MethodGet, b.url, io.Reader(nil)
	if b.fail {
		if strings.Contains(b.url, "?") {
			return false, nil
		}
		method, target = http.MethodPost, strings.TrimSuffix(b.url, "/")+"/fail"
		body = strings.NewReader(clipUTF8(b.report, heartbeatReportMax))
	}
	req, err := http.NewRequestWithContext(ctx, method, target, body)
	if err != nil {
		return false, fmt.Errorf("%s %s: bad URL", method, redactURL(target))
	}
	resp, err := cl.Do(req)
	if err != nil {
		// A url.Error quotes the whole URL, the check's key among it.
		var ue *url.Error
		if errors.As(err, &ue) {
			err = ue.Err
		}
		return false, fmt.Errorf("%s %s: %w", method, redactURL(target), err)
	}
	io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
	resp.Body.Close()
	if resp.StatusCode/100 != 2 {
		return false, fmt.Errorf("%s %s: %s", method, redactURL(target), resp.Status)
	}
	return true, nil
}

// clipUTF8 cuts s to at most n bytes without splitting a character.
func clipUTF8(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return strings.ToValidUTF8(s[:n], "")
}

// readHeartbeatURL reads the heartbeat URL from path; no file is no heartbeat.
func readHeartbeatURL(path string) (string, error) {
	if path == "" {
		return "", nil
	}
	raw, err := os.ReadFile(path)
	if errors.Is(err, os.ErrNotExist) {
		return "", nil
	}
	if err != nil {
		return "", err
	}
	s := strings.TrimSpace(string(raw))
	if err := checkHeartbeatURL(s); err != nil {
		return "", fmt.Errorf("%s: %w", path, err)
	}
	return s, nil
}

// checkHeartbeatURL accepts an http(s) URL with a host. Its error leaves the
// URL out: the path is the check's key, which anyone who reads it can ping in
// the host's name.
func checkHeartbeatURL(s string) error {
	u, err := url.Parse(s)
	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || strings.ContainsAny(s, " \t\r\n") {
		return errors.New("the heartbeat URL is not an http:// or https:// URL")
	}
	return nil
}

// redactURL is a heartbeat URL as logs show it: its scheme and host.
func redactURL(s string) string {
	u, err := url.Parse(s)
	if err != nil || u.Host == "" {
		return "(the heartbeat URL)"
	}
	return u.Scheme + "://" + u.Host + "/..."
}

// standsBy reports whether this host stands by for another host's off-site
// bucket (offsite.Status.Standby), however old that record is: the
// installer's take-over or the host's first write ends it.
func standsBy(offsiteOn bool, statusPath string) bool {
	if !offsiteOn {
		return false
	}
	st, err := offsite.ReadStatus(statusPath)
	return err == nil && st != nil && st.Standby
}