FelisMC/Felis
2
0 Fork 0
Code Issues Pull Requests 6 Actions Packages Projects Releases Wiki Activity

feat(offsite): 世界归档与数据库备份加密同步到异地 S3，reaper 确认异地副本后才删除世界

Browse Source
This commit is contained in:
Lemon-miaow committed 2026-09-24 19:25:16 +08:00
1 parent fa8db4c7e8
commit 47890ca913
27 files changed
+2928 -51
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

deploy/bootstrap.sh
+226 -4
View File
@@ -153,6 +153,19 @@ FELIS_DB_BACKUP_METRICS="${FELIS_DB_BACKUP_METRICS:-/var/lib/node_exporter/textf
# 0 migrates without the pre-migration snapshot, e.g. against an external database newer
# than this host's pg_dump. The upgrade stops if the snapshot fails and this is not set.
FELIS_PRE_MIGRATE_BACKUP="${FELIS_PRE_MIGRATE_BACKUP:-1}"
# The off-site copy (felis offsite, troubleshooting §16). Every hour the host encrypts each
# world archive and the newest database bundles and copies them to an S3-compatible bucket,
# and the reaper deletes an idle world only once its archive is there. Without a bucket
# every backup lives on this one machine, and losing its disk loses them all. Set the
# bucket and its endpoint to turn it on; FELIS_OFFSITE_ACCESS_KEY / FELIS_OFFSITE_SECRET_KEY
# (and optionally a FELIS_OFFSITE_KEY from `felis offsite keygen`) go into
# /etc/felis/offsite.env, mode 0600, with the encryption key generated when there is none.
# A re-run without these keeps the [offsite] felis.host.toml already has.
FELIS_OFFSITE_ENDPOINT="${FELIS_OFFSITE_ENDPOINT:-}"
FELIS_OFFSITE_BUCKET="${FELIS_OFFSITE_BUCKET:-}"
FELIS_OFFSITE_REGION="${FELIS_OFFSITE_REGION:-}"
FELIS_OFFSITE_PREFIX="${FELIS_OFFSITE_PREFIX:-}"
FELIS_OFFSITE_DB_KEEP="${FELIS_OFFSITE_DB_KEEP:-}"
INSTALL_MODE="${FELIS_INSTALL_MODE:-}"
# Loopback by default: hasJoined is an unauthenticated endpoint by protocol (Velocity
# sends no token), so a public bind is a free auth relay — anyone can point their own
@@ -255,6 +268,9 @@ DB_BACKUP_TIMER="/etc/systemd/system/felis-db-backup.timer"
WATCHDOG_SERVICE="/etc/systemd/system/felis-watchdog.service"
WATCHDOG_TIMER="/etc/systemd/system/felis-watchdog.timer"
WATCHDOG_STATE="/var/lib/felis/watchdog/state.json"
OFFSITE_ENV="${STATE_DIR}/offsite.env"
OFFSITE_SERVICE="/etc/systemd/system/felis-offsite.service"
OFFSITE_TIMER="/etc/systemd/system/felis-offsite.timer"
# While this marker holds a future Unix time, felis watchdog mails nothing: an install
# restarts the control plane and the system servers on purpose. cleanup removes it; the
# time in it is the backstop for an installer killed before its EXIT trap runs.
@@ -600,6 +616,34 @@ validate_settings() {
validate_nodeport FELIS_PANEL_NODEPORT "$FELIS_PANEL_NODEPORT"
validate_listen FELIS_NANO_LISTEN "$FELIS_NANO_LISTEN"
validate_cidr FELIS_NANO_PROXY_CIDR "$FELIS_NANO_PROXY_CIDR"
validate_offsite_settings
}
# validate_offsite_settings checks the FELIS_OFFSITE_* inputs before anything is
# installed. They are written into felis.toml as TOML strings and the secrets into a
# single-quoted env file, so a quote, a backslash or a line break is refused outright.
validate_offsite_settings() {
local name value
for name in FELIS_OFFSITE_ENDPOINT FELIS_OFFSITE_BUCKET FELIS_OFFSITE_REGION FELIS_OFFSITE_PREFIX \
FELIS_OFFSITE_ACCESS_KEY FELIS_OFFSITE_SECRET_KEY FELIS_OFFSITE_KEY; do
value="${!name:-}"
case "$value" in
*\"* | *\'* | *\\* | *$'\n'* | *$'\r'*) die "${name} must not contain quotes, backslashes or line breaks" ;;
esac
done
if [ -z "$FELIS_OFFSITE_BUCKET" ]; then
if [ -n "$FELIS_OFFSITE_ENDPOINT$FELIS_OFFSITE_REGION$FELIS_OFFSITE_PREFIX$FELIS_OFFSITE_DB_KEEP" ]; then
die "FELIS_OFFSITE_* is set without FELIS_OFFSITE_BUCKET; name the bucket too"
fi
return 0
fi
[ -n "$FELIS_OFFSITE_ENDPOINT" ] || die "FELIS_OFFSITE_BUCKET needs FELIS_OFFSITE_ENDPOINT (https://host[:port] of the S3-compatible store)"
case "$FELIS_OFFSITE_BUCKET" in
*/* | *' '*) die "FELIS_OFFSITE_BUCKET is a bucket name; put a path inside it in FELIS_OFFSITE_PREFIX" ;;
esac
if [ -n "$FELIS_OFFSITE_DB_KEEP" ] && ! [[ "$FELIS_OFFSITE_DB_KEEP" =~ ^[1-9][0-9]*$ ]]; then
die "FELIS_OFFSITE_DB_KEEP must be a positive number of bundles, got: ${FELIS_OFFSITE_DB_KEEP}"
fi
}
# ---------------------------------------------------------------------------
@@ -2319,8 +2363,52 @@ persisted_registry_block() {
done
}
# persisted_offsite_block echoes the [offsite] section an earlier run (or the operator)
# left behind: after the first install the bucket is configured by editing felis.host.toml
# or by re-running with FELIS_OFFSITE_*, and a plain re-run must keep it. Deleting the
# section and re-running is how the off-site copy is turned off. Same first-readable-file
# rule as persisted_smtp_block.
persisted_offsite_block() {
local f out
for f in "${STATE_DIR}/felis.host.toml" "${STATE_DIR}/felis.pod.toml"; do
[ -r "$f" ] || continue
out="$(awk '
/^[[:space:]]*\[/ {
if (inoff) exit
inoff = ($0 ~ /^[[:space:]]*\[offsite\][[:space:]]*$/)
if (inoff) print
next
}
inoff && /^[[:space:]]*(endpoint|region|bucket|prefix|access_key_ref|secret_key_ref|key_ref|db_keep)[[:space:]]*=/ { print }
' "$f")"
[ -n "$out" ] || continue
printf '%s\n' "$out"
return 0
done
}
# offsite_block is the [offsite] section this run writes: from FELIS_OFFSITE_* when the
# bucket is given, else the one an earlier run left.
offsite_block() {
if [ -z "$FELIS_OFFSITE_BUCKET" ]; then
persisted_offsite_block
return 0
fi
printf '[offsite]\n'
printf 'endpoint = "%s"\n' "$FELIS_OFFSITE_ENDPOINT"
printf 'bucket = "%s"\n' "$FELIS_OFFSITE_BUCKET"
if [ -n "$FELIS_OFFSITE_REGION" ]; then printf 'region = "%s"\n' "$FELIS_OFFSITE_REGION"; fi
if [ -n "$FELIS_OFFSITE_PREFIX" ]; then printf 'prefix = "%s"\n' "$FELIS_OFFSITE_PREFIX"; fi
if [ -n "$FELIS_OFFSITE_DB_KEEP" ]; then printf 'db_keep = %s\n' "$FELIS_OFFSITE_DB_KEEP"; fi
}
# offsite_enabled: the [offsite] section this run writes names a bucket.
offsite_enabled() {
offsite_block | grep -Eq '^[[:space:]]*bucket[[:space:]]*=[[:space:]]*"[^"]+"'
}
write_felis_toml() {
local target="$1" db_host="$2" smtp_block auth_source_blocks registry_block archive_block
local target="$1" db_host="$2" smtp_block auth_source_blocks registry_block archive_block offsite_section
smtp_block="$(persisted_smtp_block)"
if [ -n "$smtp_block" ]; then
log "carrying forward the configured [smtp] relay"
@@ -2337,10 +2425,16 @@ write_felis_toml() {
log "carrying forward the configured [archive] overrides"
archive_block="${archive_block}"$'\n' # keep a blank line before the next section
fi
# The pod copy needs it as much as the host one: the reaper reads [offsite] from the
# config Secret to know it must wait for each archive's off-site copy.
offsite_section="$(offsite_block)"
if [ -n "$offsite_section" ]; then
offsite_section="${offsite_section}"$'\n\n' # keep a blank line before the next section
fi
cat > "$target" <<EOF
# Generated by deploy/bootstrap.sh; rerun the installer to regenerate. Hand edits are
# overwritten, except [smtp], [[auth_source]], and the operator-owned [registry] /
# [archive] overrides, which carry forward.
# overwritten, except [smtp], [[auth_source]], [offsite], and the operator-owned
# [registry] / [archive] overrides, which carry forward.
[server]
listen = "0.0.0.0:8080"
root_domain = "${FELIS_ROOT_DOMAIN}"
@@ -2366,7 +2460,7 @@ ${registry_block}
store = "tarLocal"
local_path = "${FELIS_ARCHIVE_LOCAL_PATH}"
${archive_block}
[auth]
${offsite_section}[auth]
admin_hostname = "op.console.${FELIS_ROOT_DOMAIN}"
panel_hostname = "console.${FELIS_ROOT_DOMAIN}"
@@ -2432,6 +2526,82 @@ quiet_watchdog() {
printf '%s\n' "$(( $(date +%s) + 7200 ))" > "$WATCHDOG_QUIET_FILE"
}
# The hourly off-site copy. The bucket is checked now, in the install, so wrong
# credentials or an unreachable endpoint show up here; the first copy itself runs in the
# background, since a host with many archives can take a long while to upload them.
# With no bucket configured the timer is removed (the operator deleted [offsite]) and the
# install says loudly that every backup is on this machine only.
install_offsite_timer() {
if [ "${OFFSITE_ENABLED:-0}" != 1 ]; then
if [ -e "$OFFSITE_TIMER" ] || [ -e "$OFFSITE_SERVICE" ]; then
systemctl disable --now felis-offsite.timer >/dev/null 2>&1 || true
rm -f "$OFFSITE_TIMER" "$OFFSITE_SERVICE"
systemctl daemon-reload
warn "no [offsite] bucket is configured any more; the off-site copy timer was removed"
fi
return 0
fi
cat > "$OFFSITE_SERVICE" <<EOF
[Unit]
Description=Felis off-site copy (world archives and database bundles, encrypted, to the [offsite] bucket)
After=network-online.target k3s.service postgresql.service felis-db-backup.service
Wants=network-online.target
[Service]
Type=oneshot
EnvironmentFile=${OFFSITE_ENV}
ExecStart=${HOST_BIN} offsite sync -config ${STATE_DIR}/felis.host.toml -env-file ${OFFSITE_ENV} -db-dir ${FELIS_DB_BACKUP_DIR} -backup-pvc "${FELIS_BACKUP_PVC}"
TimeoutStartSec=55min
Nice=10
IOSchedulingClass=idle
PrivateTmp=yes
NoNewPrivileges=yes
EOF
cat > "$OFFSITE_TIMER" <<EOF
[Unit]
Description=Hourly Felis off-site copy
[Timer]
OnCalendar=hourly
RandomizedDelaySec=10min
Persistent=true
[Install]
WantedBy=timers.target
EOF
systemctl daemon-reload
systemctl enable --now felis-offsite.timer
if "$HOST_BIN" offsite list -config "${STATE_DIR}/felis.host.toml" -env-file "$OFFSITE_ENV" >/dev/null; then
systemctl start --no-block felis-offsite.service
ok "off-site copy: hourly to the [offsite] bucket, first copy started (sudo felis offsite status; journalctl -u felis-offsite)"
else
warn "the [offsite] bucket did not answer (error above); nothing is copied off this machine until it does: fix ${OFFSITE_ENV} or [offsite] in ${STATE_DIR}/felis.host.toml, then sudo systemctl start felis-offsite.service"
fi
}
# summary_offsite is the installer's last word on where the backups live.
summary_offsite() {
if [ "${OFFSITE_ENABLED:-0}" != 1 ]; then
warn "NO OFF-SITE COPY: every world archive and database backup is on this machine only."
warn "Losing its disk loses them all. Set FELIS_OFFSITE_BUCKET, FELIS_OFFSITE_ENDPOINT,"
warn "FELIS_OFFSITE_ACCESS_KEY and FELIS_OFFSITE_SECRET_KEY and re-run (docs/troubleshooting.md §16)."
return 0
fi
if [ "${OFFSITE_KEY_NEW:-0}" = 1 ]; then
echo
warn "================================================================================"
warn "The off-site copies are encrypted with this key. Store it NOW somewhere other than"
warn "this machine (a password manager): without it nothing in the bucket can be read."
warn ""
warn " FELIS_OFFSITE_KEY=${FELIS_OFFSITE_KEY}"
warn ""
warn "It is also in ${OFFSITE_ENV}, which is lost with this machine."
warn "================================================================================"
else
log "Off-site copy: the encryption key is in ${OFFSITE_ENV}; keep a copy of it off this machine."
fi
}
# The platform watchdog: every two minutes it checks the control plane, the login gate,
# the fleet, PostgreSQL, the game proxy, the database backups and the host's disks and
# memory, and mails the owners (their verified addresses, over the [smtp] relay) what
@@ -2520,6 +2690,53 @@ EOF
fi
}
# configure_offsite writes OFFSITE_ENV, the secrets behind [offsite]: the bucket's access
# keys and the key every off-site object is sealed with. Values in this run's environment
# replace the file's (rotating the bucket credentials is a re-run); the encryption key is
# generated when neither has one. A different key than the file's is refused: every object
# already in the bucket is sealed with the old one, and swapping it would make them
# unreadable without a word.
configure_offsite() {
OFFSITE_ENABLED=0
OFFSITE_KEY_NEW=0
offsite_enabled || return 0
OFFSITE_ENABLED=1
local env_ak="${FELIS_OFFSITE_ACCESS_KEY:-}" env_sk="${FELIS_OFFSITE_SECRET_KEY:-}" env_key="${FELIS_OFFSITE_KEY:-}"
local file_key=""
FELIS_OFFSITE_ACCESS_KEY="" FELIS_OFFSITE_SECRET_KEY="" FELIS_OFFSITE_KEY=""
if [ -f "$OFFSITE_ENV" ]; then
# shellcheck disable=SC1090
. "$OFFSITE_ENV"
file_key="$FELIS_OFFSITE_KEY"
fi
FELIS_OFFSITE_ACCESS_KEY="${env_ak:-$FELIS_OFFSITE_ACCESS_KEY}"
FELIS_OFFSITE_SECRET_KEY="${env_sk:-$FELIS_OFFSITE_SECRET_KEY}"
if [ -n "$env_key" ] && [ -n "$file_key" ] && [ "$env_key" != "$file_key" ]; then
die "FELIS_OFFSITE_KEY differs from the key in ${OFFSITE_ENV}; the objects already in the bucket are sealed with that one. Unset FELIS_OFFSITE_KEY to keep it (docs/troubleshooting.md §16)"
fi
FELIS_OFFSITE_KEY="${env_key:-$file_key}"
if [ -z "$FELIS_OFFSITE_ACCESS_KEY" ] || [ -z "$FELIS_OFFSITE_SECRET_KEY" ]; then
die "[offsite] names a bucket but there are no credentials for it: set FELIS_OFFSITE_ACCESS_KEY and FELIS_OFFSITE_SECRET_KEY (they are kept in ${OFFSITE_ENV})"
fi
if [ -z "$FELIS_OFFSITE_KEY" ]; then
FELIS_OFFSITE_KEY="$(openssl rand -base64 32)"
OFFSITE_KEY_NEW=1
fi
(
umask 077
cat > "$OFFSITE_ENV" <<EOF
# The off-site copy's secrets (felis offsite, docs/troubleshooting.md §16). Keep a copy of
# FELIS_OFFSITE_KEY somewhere other than this machine: without it the copies in the
# bucket cannot be read, and this file goes with the machine.
FELIS_OFFSITE_ACCESS_KEY='${FELIS_OFFSITE_ACCESS_KEY}'
FELIS_OFFSITE_SECRET_KEY='${FELIS_OFFSITE_SECRET_KEY}'
FELIS_OFFSITE_KEY='${FELIS_OFFSITE_KEY}'
EOF
)
chmod 0600 "$OFFSITE_ENV"
ok "off-site copy: secrets in ${OFFSITE_ENV}"
}
deploy_bundle() {
local had_api=0 had_operator=0
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
@@ -2767,6 +2984,8 @@ summary() {
log "Mojang account rather than a password."
log "Use 'sudo felis breakGlass' only for emergency local Owner recovery/reset."
echo
summary_offsite
echo
}
# ---------------------------------------------------------------------------
@@ -3099,6 +3318,7 @@ main() {
bootstrap_from_tui || [ -n "${FELIS_SKIP_FETCH:-}" ] || resolve_install_ref
install_cloudflared
load_or_make_secrets
configure_offsite
ensure_panel_tls_cert
install_docker
install_k3s
@@ -3138,6 +3358,8 @@ main() {
install_velocity
# After deploy_bundle: the bundle's MinecraftServer export reads the cluster.
install_db_backup_timer
# After the backup timer: its first bundle is part of the first copy.
install_offsite_timer
# Last: its first run should see the platform as this install leaves it.
install_watchdog_timer
mark_bootstrap_done
deploy/bootstrap_test.sh
+167 -4
View File
@@ -1020,12 +1020,14 @@ rm -f "$kubcalls"
wrblock="$(awk '/^write_felis_toml\(\) \{/,/^}/' "$BS")"
prblock="$(awk '/^persisted_registry_block\(\) \{/,/^}/' "$BS")"
pablock="$(awk '/^persisted_archive_block\(\) \{/,/^}/' "$BS")"
{ [ -n "$wrblock" ] && [ -n "$prblock" ] && [ -n "$pablock" ]; } \
|| { echo "FAIL: write_felis_toml / persisted_{registry,archive}_block not found in $BS"; exit 1; }
poblock="$(awk '/^persisted_offsite_block\(\) \{/,/^}/' "$BS")"
oblock="$(awk '/^offsite_block\(\) \{/,/^}/' "$BS")"
{ [ -n "$wrblock" ] && [ -n "$prblock" ] && [ -n "$pablock" ] && [ -n "$poblock" ] && [ -n "$oblock" ]; } \
|| { echo "FAIL: write_felis_toml / persisted_{registry,archive,offsite}_block / offsite_block not found in $BS"; exit 1; }
# The blocks quote themselves (the awk program uses single quotes), so they are
# sourced from a file instead of being spliced into a single-quoted bash -c.
fnfile="$(mktemp)"
printf '%s\n%s\n%s\n' "$prblock" "$pablock" "$wrblock" > "$fnfile"
printf '%s\n%s\n%s\n%s\n%s\n' "$prblock" "$pablock" "$poblock" "$oblock" "$wrblock" > "$fnfile"
rdir="$(mktemp -d)"
cat > "$rdir/felis.host.toml" <<'TOML'
@@ -1044,6 +1046,10 @@ region = "us-east-1"
store = "tarLocal"
local_path = "/stale/path"
retention = "30d"
[offsite]
endpoint = "https://objects.example"
bucket = "felis-offsite"
TOML
run_write() { # out-file
@@ -1055,7 +1061,7 @@ run_write() { # out-file
FELIS_ROOT_DOMAIN=r.example.com DB_USER=u DB_PASSWORD=p DB_NAME=d MINECRAFT_NS=minecraft \
FELIS_EGRESS_MODE=nodeport FELIS_LIMBO_IMAGE=li FELIS_LOBBY_IMAGE=lo \
REGISTRY_URL=registry.felis.svc:5000 BUILD_NS=felis-build FELIS_ARCHIVE_LOCAL_PATH=/a \
write_felis_toml "$OUT_TOML" 127.0.0.1'
FELIS_OFFSITE_BUCKET= write_felis_toml "$OUT_TOML" 127.0.0.1'
}
run_write "$rdir/out.toml"
@@ -1071,6 +1077,11 @@ expect "the carried subtable keeps its keys" 'endpoint = "https://s3.example"' "
expect "url stays installer-owned" 'url = "registry.felis.svc:5000"' "$out"
expect "a re-run carries the archive retention window" 'retention = "30d"' "$out"
expect "the archive mount stays installer-owned" 'local_path = "/a"' "$out"
expect "a re-run keeps the off-site bucket, set apart from the next section" '[offsite]
endpoint = "https://objects.example"
bucket = "felis-offsite"
[auth]' "$out"
case "$out" in
*stale.invalid* | *stale-ns* | *stale/path*)
echo "FAIL: stale installer-owned values survived the re-run"; fails=$((fails + 1)) ;;
@@ -1220,6 +1231,158 @@ case "$main_block" in
echo "PASS main quiets the watchdog first and installs it last" ;;
*) echo "FAIL main must call quiet_watchdog before any restart and install_watchdog_timer after the backup timer"; fails=$((fails + 1)) ;;
esac
case "$main_block" in
*load_or_make_secrets*configure_offsite*run_migrations*install_db_backup_timer*install_offsite_timer*install_watchdog_timer*)
echo "PASS main configures the off-site copy before the toml is written and starts it after the first backup" ;;
*) echo "FAIL main must call configure_offsite before run_migrations and install_offsite_timer between the backup and watchdog timers"; fails=$((fails + 1)) ;;
esac
# --- the off-site copy: [offsite], its secrets file, the hourly timer ---------------------
# Without it every backup is on one disk. A re-run must keep the bucket, the encryption key
# must never change under objects sealed with the old one, and an install without a bucket
# must say so loudly.
ofile="$(mktemp)"
for fn in validate_offsite_settings persisted_offsite_block offsite_block offsite_enabled configure_offsite install_offsite_timer summary_offsite; do
blk="$(awk "/^${fn}\\(\\) \\{/,/^}/" "$BS")"
[ -n "$blk" ] || { echo "FAIL: no ${fn} found in $BS"; exit 1; }
[ "$(printf '%s\n' "$blk" | wc -l)" -lt 90 ] \
|| { echo "FAIL: the extracted block is not ${fn} -- did its closing brace move?"; exit 1; }
printf '%s\n' "$blk" >> "$ofile"
done
odir="$(mktemp -d)"
run_offsite() { # script; runs with the off-site functions sourced
STATE_DIR="$odir" OFFSITE_ENV="$odir/offsite.env" OFFSITE_SERVICE="$odir/felis-offsite.service" \
OFFSITE_TIMER="$odir/felis-offsite.timer" FNFILE="$ofile" HOST_BIN=fakefelis \
FELIS_DB_BACKUP_DIR=/var/lib/felis/db-backups FELIS_BACKUP_PVC=felis-backups bash -c '
set -Eeuo pipefail
die() { printf "DIE: %s\n" "$*"; exit 1; }
log() { printf "LOG: %s\n" "$*"; }; ok() { printf "OK: %s\n" "$*"; }; warn() { printf "WARN: %s\n" "$*"; }
systemctl() { printf "SYSTEMCTL: %s\n" "$*" >&2; }
fakefelis() { printf "RUN: %s\n" "$*" >&2; [ -z "${CHECK_FAILS:-}" ] || { echo "bucket: access denied" >&2; return 1; }; }
FELIS_OFFSITE_ENDPOINT="${FELIS_OFFSITE_ENDPOINT:-}" FELIS_OFFSITE_BUCKET="${FELIS_OFFSITE_BUCKET:-}"
FELIS_OFFSITE_REGION="${FELIS_OFFSITE_REGION:-}" FELIS_OFFSITE_PREFIX="${FELIS_OFFSITE_PREFIX:-}"
FELIS_OFFSITE_DB_KEEP="${FELIS_OFFSITE_DB_KEEP:-}"
. "$FNFILE"
'"$1" 2>&1
}
out="$(FELIS_OFFSITE_BUCKET=b run_offsite validate_offsite_settings)"
expect "a bucket without an endpoint is refused" "DIE: FELIS_OFFSITE_BUCKET needs FELIS_OFFSITE_ENDPOINT" "$out"
out="$(FELIS_OFFSITE_ENDPOINT=https://s3.example FELIS_OFFSITE_BUCKET='b"x' run_offsite validate_offsite_settings)"
expect "a quote cannot reach the generated toml" "DIE: FELIS_OFFSITE_BUCKET must not contain quotes" "$out"
out="$(FELIS_OFFSITE_ENDPOINT=https://s3.example FELIS_OFFSITE_BUCKET=b FELIS_OFFSITE_SECRET_KEY="se'cret" run_offsite validate_offsite_settings)"
expect "a quote cannot reach the secrets file" "DIE: FELIS_OFFSITE_SECRET_KEY must not contain quotes" "$out"
out="$(FELIS_OFFSITE_ENDPOINT=https://s3.example FELIS_OFFSITE_BUCKET=b/sub run_offsite validate_offsite_settings)"
expect "a path in the bucket name is refused" "DIE: FELIS_OFFSITE_BUCKET is a bucket name" "$out"
out="$(FELIS_OFFSITE_ENDPOINT=https://s3.example run_offsite validate_offsite_settings)"
expect "an endpoint without a bucket is refused" "DIE: FELIS_OFFSITE_* is set without FELIS_OFFSITE_BUCKET" "$out"
out="$(FELIS_OFFSITE_ENDPOINT=https://s3.example FELIS_OFFSITE_BUCKET=b FELIS_OFFSITE_DB_KEEP=0 run_offsite validate_offsite_settings)"
expect "db_keep must be positive" "DIE: FELIS_OFFSITE_DB_KEEP must be a positive number" "$out"
out="$(run_offsite 'validate_offsite_settings; echo fine')"
expect "no off-site settings is a valid install" "fine" "$out"
out="$(FELIS_OFFSITE_ENDPOINT=https://s3.example FELIS_OFFSITE_BUCKET=felis FELIS_OFFSITE_PREFIX=host1 run_offsite offsite_block)"
expect "the environment writes [offsite]" '[offsite]
endpoint = "https://s3.example"
bucket = "felis"
prefix = "host1"' "$out"
cat > "$odir/felis.host.toml" <<'TOML'
[archive]
store = "tarLocal"
[offsite]
endpoint = "https://s3.example"
bucket = "kept"
key_ref = "MY_KEY"
[auth]
admin_hostname = "x"
TOML
out="$(run_offsite offsite_block)"
expect "a re-run keeps the configured bucket" 'bucket = "kept"' "$out"
expect "a re-run keeps the key reference" 'key_ref = "MY_KEY"' "$out"
case "$out" in
*admin_hostname*) echo "FAIL the carried [offsite] swallowed the next section"; fails=$((fails + 1)) ;;
*) echo "PASS the carried [offsite] stops at the next section" ;;
esac
# First configured install: the key is generated, the file is private, the key is shown once.
rm -f "$odir/felis.host.toml"
out="$(FELIS_OFFSITE_ENDPOINT=https://s3.example FELIS_OFFSITE_BUCKET=felis \
FELIS_OFFSITE_ACCESS_KEY=AK FELIS_OFFSITE_SECRET_KEY=SK run_offsite 'configure_offsite; summary_offsite')"
envf="$(cat "$odir/offsite.env" 2>/dev/null)"
expect "the access key is kept for the unit" "FELIS_OFFSITE_ACCESS_KEY='AK'" "$envf"
expect "an encryption key is generated" "FELIS_OFFSITE_KEY='" "$envf"
key="$(sed -n "s/^FELIS_OFFSITE_KEY='\(.*\)'$/\1/p" "$odir/offsite.env")"
if [ "$(printf '%s' "$key" | base64 -d 2>/dev/null | wc -c | tr -d ' ')" = 32 ]; then
echo "PASS the generated key is 32 random bytes in base64"
else
echo "FAIL the generated key '$key' is not 32 bytes of base64"; fails=$((fails + 1))
fi
if [ "$(stat -c %a "$odir/offsite.env" 2>/dev/null || stat -f %Lp "$odir/offsite.env")" = 600 ]; then
echo "PASS the off-site secrets file is private"
else
echo "FAIL offsite.env must be 0600"; fails=$((fails + 1))
fi
expect "a new key is shown once, with the warning to keep it elsewhere" "WARN: FELIS_OFFSITE_KEY=${key}" "$out"
# A re-run with new credentials keeps the key; a different key is refused.
out="$(FELIS_OFFSITE_ENDPOINT=https://s3.example FELIS_OFFSITE_BUCKET=felis \
FELIS_OFFSITE_ACCESS_KEY=AK2 run_offsite 'configure_offsite; summary_offsite')"
expect "rotated credentials replace the old ones" "FELIS_OFFSITE_ACCESS_KEY='AK2'" "$(cat "$odir/offsite.env")"
expect "the secret key the re-run did not give is kept" "FELIS_OFFSITE_SECRET_KEY='SK'" "$(cat "$odir/offsite.env")"
expect "the key survives a re-run" "FELIS_OFFSITE_KEY='${key}'" "$(cat "$odir/offsite.env")"
case "$out" in
*"FELIS_OFFSITE_KEY="*) echo "FAIL a re-run printed the key again"; fails=$((fails + 1)) ;;
*) echo "PASS a re-run does not print the key again" ;;
esac
out="$(FELIS_OFFSITE_ENDPOINT=https://s3.example FELIS_OFFSITE_BUCKET=felis \
FELIS_OFFSITE_KEY=c29tZXRoaW5nIGVsc2UgZW50aXJlbHkgZGlmZmVyZW50IQ== run_offsite configure_offsite)"
expect "a different key is refused" "DIE: FELIS_OFFSITE_KEY differs from the key in" "$out"
expect "the refusal leaves the key alone" "FELIS_OFFSITE_KEY='${key}'" "$(cat "$odir/offsite.env")"
rm -f "$odir/offsite.env"
out="$(FELIS_OFFSITE_ENDPOINT=https://s3.example FELIS_OFFSITE_BUCKET=felis run_offsite configure_offsite)"
expect "a bucket without credentials is refused" "DIE: [offsite] names a bucket but there are no credentials" "$out"
out="$(run_offsite 'configure_offsite; install_offsite_timer; summary_offsite; echo "enabled=$OFFSITE_ENABLED"')"
expect "no bucket leaves the off-site copy off" "enabled=0" "$out"
expect "no bucket is a loud warning" "WARN: NO OFF-SITE COPY" "$out"
out="$(OFFSITE_ENABLED=1 run_offsite install_offsite_timer)"
unit="$(cat "$odir/felis-offsite.service")"
timer="$(cat "$odir/felis-offsite.timer")"
expect "the unit loads the secrets" "EnvironmentFile=$odir/offsite.env" "$unit"
expect "the unit syncs from the host config" \
"ExecStart=fakefelis offsite sync -config $odir/felis.host.toml -env-file $odir/offsite.env -db-dir /var/lib/felis/db-backups -backup-pvc \"felis-backups\"" "$unit"
expect "a run ends before the next hour's" "TimeoutStartSec=55min" "$unit"
expect "the copy runs hourly" "OnCalendar=hourly" "$timer"
expect "a missed run catches up at boot" "Persistent=true" "$timer"
expect "the timer is enabled" "SYSTEMCTL: enable --now felis-offsite.timer" "$out"
expect "the bucket is checked during the install" "RUN: offsite list -config $odir/felis.host.toml -env-file $odir/offsite.env" "$out"
expect "the first copy runs in the background" "SYSTEMCTL: start --no-block felis-offsite.service" "$out"
out="$(OFFSITE_ENABLED=1 CHECK_FAILS=1 run_offsite install_offsite_timer)"
expect "an unreachable bucket shows why" "bucket: access denied" "$out"
expect "an unreachable bucket is a loud warning" "WARN: the [offsite] bucket did not answer" "$out"
case "$out" in
*"start --no-block"*) echo "FAIL a failed check still started the copy"; fails=$((fails + 1)) ;;
*) echo "PASS a failed check does not start the copy" ;;
esac
out="$(OFFSITE_ENABLED=0 run_offsite 'systemctl() { printf "SYSTEMCTL: %s\n" "$*" >> "$STATE_DIR/systemctl.log"; }; install_offsite_timer')"
expect "removing [offsite] disables the timer" "SYSTEMCTL: disable --now felis-offsite.timer" "$(cat "$odir/systemctl.log")"
expect "removing [offsite] says so" "WARN: no [offsite] bucket is configured any more" "$out"
if [ -e "$odir/felis-offsite.service" ] || [ -e "$odir/felis-offsite.timer" ]; then
echo "FAIL removing [offsite] left the units behind"; fails=$((fails + 1))
else
echo "PASS removing [offsite] removes the units"
fi
rm -rf "$odir" "$ofile"
# ---------------------------------------------------------------------------------------
if [ "$fails" -eq 0 ]; then