227 lines
9.6 KiB
Bash
227 lines
9.6 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
unset KNOT_MASTER_KEY
|
|
|
|
# These paths are deliberately fixed to this host and its rehearsed migration.
|
|
stage=/home/anish/tangled-v1.20.0-preflight
|
|
repo=/etc/commonscomputing-nix
|
|
state=/var/lib/knot
|
|
source_dir=/home/git
|
|
backup_dir=/var/backups
|
|
profile=/nix/var/nix/profiles/system
|
|
current_system=/run/current-system
|
|
ssh_key=/etc/ssh/ssh_host_ed25519_key
|
|
lock_file=/run/lock/commonscomputing-update.lock
|
|
runtime_template=/run/knot-upgrade.XXXXXX
|
|
system_root=/var/lib/knot-upgrade-system
|
|
tools_root=/var/lib/knot-upgrade-tools
|
|
|
|
if [ "$(id -u)" != 0 ] || [ "$(hostname)" != asusmininix ]; then
|
|
printf 'Run this script as root on asusmininix.\n' >&2
|
|
exit 1
|
|
fi
|
|
if [ "$#" -gt 1 ] || { [ "$#" -eq 1 ] && [ "$1" != --yes ]; }; then
|
|
printf 'Usage: bash scripts/upgrade-knot.sh [--yes]\n' >&2
|
|
exit 1
|
|
fi
|
|
exec 9>"$lock_file"
|
|
flock --nonblock 9 || { printf 'Another update is running.\n' >&2; exit 1; }
|
|
if systemctl is-active --quiet auto-update.service; then
|
|
printf 'Wait for auto-update.service to finish before migrating.\n' >&2
|
|
exit 1
|
|
fi
|
|
|
|
system=$(<"$stage/system-path")
|
|
tools=$(<"$stage/tools-path")
|
|
case "$system" in /nix/store/*-nixos-system-asusmininix-*) ;; *) exit 1 ;; esac
|
|
case "$tools" in /nix/store/*-knot-migration-tools) ;; *) exit 1 ;; esac
|
|
if ! nix-store --check-validity "$system" "$tools"; then
|
|
read -r expected _ < "$stage/upgrade.export.gz.sha256"
|
|
actual=$(sha256sum "$stage/upgrade.export.gz")
|
|
test "${actual%% *}" = "$expected"
|
|
gzip -dc "$stage/upgrade.export.gz" | nix-store --import --option require-sigs false
|
|
fi
|
|
nix-store --realise "$system" --add-root "$system_root" >/dev/null
|
|
nix-store --realise "$tools" --add-root "$tools_root" >/dev/null
|
|
export PATH="$tools/bin:$PATH"
|
|
test -x "$system/bin/switch-to-configuration"
|
|
test -x "$system/sw/bin/knot-server"
|
|
# An older staged build intentionally disabled the timer; do not install it.
|
|
grep -Fq 'ConditionPathExists=/var/lib/knot/.upgrade-verified' \
|
|
"$system/etc/systemd/system/auto-update.timer"
|
|
grep -Fq 'ConditionPathExists=/var/lib/knot/.upgrade-verified' \
|
|
"$system/etc/systemd/system/auto-update.service"
|
|
|
|
smoke() {
|
|
for unit in knot-rs.service spindle.service sshd.service; do
|
|
systemctl is-active --quiet "$unit"
|
|
done
|
|
curl --fail --silent --show-error --max-time 10 --retry 5 --retry-connrefused \
|
|
--retry-delay 1 http://127.0.0.1:5555/xrpc/_health | jq -e '.version | length > 0'
|
|
curl --fail --silent --show-error --max-time 10 --retry 5 --retry-connrefused \
|
|
--retry-delay 1 http://127.0.0.1:6555/_health | jq -e '.status == "ok"'
|
|
curl --fail --silent --show-error --max-time 15 \
|
|
https://knot.commonscomputer.com/xrpc/org.tangled.knot.describeKnot \
|
|
| jq -e '.did == "did:web:knot.commonscomputer.com" and .contact.email == "pds@email.commonscomputer.com"'
|
|
read -r _ expected_key _ < <(ssh-keygen -lf "$ssh_key.pub")
|
|
scanned=$(ssh-keyscan -T 5 -p 2223 -t ed25519 127.0.0.1 2>/dev/null)
|
|
read -r _ actual_key _ < <(printf '%s\n' "$scanned" | ssh-keygen -lf -)
|
|
test "$expected_key" = "$actual_key"
|
|
# The upgraded endpoint is a long-lived WebSocket: timeout after the handshake.
|
|
status=$(curl --silent --max-time 3 --output /dev/null --write-out '%{http_code}' \
|
|
-H 'Connection: Upgrade' -H 'Upgrade: websocket' -H 'Sec-WebSocket-Version: 13' \
|
|
-H 'Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==' \
|
|
https://knot.commonscomputer.com/xrpc/com.atproto.sync.subscribeRepos || true)
|
|
test "$status" = 101
|
|
}
|
|
|
|
export GIT_TERMINAL_PROMPT=0
|
|
export GIT_SSH_COMMAND='ssh -o BatchMode=yes -o ConnectTimeout=15'
|
|
git -C "$repo" ls-remote --exit-code origin HEAD >/dev/null
|
|
git -C "$repo" push --dry-run origin HEAD:main
|
|
git -C "$repo" var GIT_AUTHOR_IDENT >/dev/null
|
|
if ! git -C "$repo" diff --cached --quiet || \
|
|
! git -C "$repo" diff --quiet -- '*.nix' flake.lock; then
|
|
printf 'Commit/stash staged changes and Nix/lockfile edits before cutover.\n' >&2
|
|
exit 1
|
|
fi
|
|
if [ -e "$state/.migration-complete" ]; then
|
|
printf 'Migration marker already exists; checking the live Rust service only.\n'
|
|
live_system=$(readlink -f "$current_system")
|
|
grep -Fq 'ConditionPathExists=/var/lib/knot/.upgrade-verified' \
|
|
"$live_system/etc/systemd/system/auto-update.timer"
|
|
grep -Fq 'ConditionPathExists=/var/lib/knot/.upgrade-verified' \
|
|
"$live_system/etc/systemd/system/auto-update.service"
|
|
smoke
|
|
touch "$state/.upgrade-verified"
|
|
systemctl unmask --runtime auto-update.timer
|
|
systemctl start auto-update.timer
|
|
systemctl is-active --quiet auto-update.timer
|
|
printf 'Already migrated; smoke tests passed and automatic updates are active.\n'
|
|
exit 0
|
|
fi
|
|
if [ -e "$state" ]; then
|
|
printf '%s exists without a completed migration. Inspect/quarantine it before retrying.\n' "$state" >&2
|
|
exit 1
|
|
fi
|
|
systemctl is-active --quiet knot.service
|
|
test -r "$source_dir/knotserver.db"
|
|
test -s "$ssh_key"
|
|
old_system=$(readlink -f "$current_system")
|
|
runtime=""
|
|
backup=""
|
|
downtime=0
|
|
activation_attempted=0
|
|
migration_started=0
|
|
verified=0
|
|
|
|
cleanup() {
|
|
result=$?
|
|
trap - EXIT INT TERM
|
|
set +e
|
|
if [ "$result" -ne 0 ] && [ "$downtime" -eq 1 ] && [ "$verified" -eq 0 ]; then
|
|
systemctl stop auto-update.timer
|
|
systemctl mask --runtime auto-update.timer
|
|
systemctl stop knot-rs.service
|
|
rm -f "$state/.migration-complete" "$state/.upgrade-verified"
|
|
if [ "$activation_attempted" -eq 1 ]; then
|
|
printf 'Activation was attempted: state may have changed. Updates/Rust knot are stopped.\n' >&2
|
|
printf 'Do NOT blindly roll back; inspect %s and %s before reopening writes.\n' "$state" "$backup" >&2
|
|
else
|
|
printf 'Migration failed before activation; restarting the unchanged Go knot.\n' >&2
|
|
if [ "$migration_started" -eq 1 ] && [ -e "$state" ]; then
|
|
mv "$state" "$backup/failed-target"
|
|
fi
|
|
if ! systemctl start knot.service; then
|
|
printf 'URGENT: Go knot could not be restarted; inspect %s.\n' "$backup" >&2
|
|
fi
|
|
fi
|
|
printf 'Backups retained in %s. Timer runtime mask does not survive reboot.\n' "$backup" >&2
|
|
fi
|
|
if [ -n "$runtime" ]; then
|
|
rm -f "$runtime/knot.env"
|
|
rmdir "$runtime"
|
|
fi
|
|
unset master_key
|
|
exit "$result"
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
umask 077
|
|
runtime=$(mktemp -d "$runtime_template")
|
|
agenix -d "$repo/secrets/knot-env.age" -i "$ssh_key" > "$runtime/knot.env"
|
|
chmod 0400 "$runtime/knot.env"
|
|
master_key=""
|
|
export -n master_key
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
case "$line" in
|
|
KNOT_MASTER_KEY=*) master_key=${line#KNOT_MASTER_KEY=} ;;
|
|
''|\#*) ;;
|
|
*) printf 'Unexpected entry in knot-env.age: expected only KNOT_MASTER_KEY.\n' >&2; exit 1 ;;
|
|
esac
|
|
done < "$runtime/knot.env"
|
|
if ! [[ "$master_key" =~ ^[A-Za-z0-9+/]{43}=$ ]]; then
|
|
printf 'KNOT_MASTER_KEY must be an unquoted base64 encoding of 32 random bytes.\n' >&2
|
|
exit 1
|
|
fi
|
|
migration_args=(
|
|
--source-db "$source_dir/knotserver.db" --source-repos "$source_dir"
|
|
--hostname knot.commonscomputer.com --host-key "$ssh_key"
|
|
--plc-url https://plc.directory --contact-email pds@email.commonscomputer.com
|
|
--master-key-env KNOT_MASTER_KEY --object-format sha1 --target "$state"
|
|
)
|
|
KNOT_MASTER_KEY="$master_key" knot-migrate "${migration_args[@]}" --dry-run
|
|
printf '\nThis will stop Go knot, back it up, COPY its data, and install the staged Rust system.\n'
|
|
printf 'Git SSH moves to port 2223; administrative SSH stays on 22. Use Tailscale or arrange WAN forwarding.\n'
|
|
printf 'Confirm the master key is backed up off-host and keep a second administrative session open.\n'
|
|
if [ "${1:-}" != --yes ]; then
|
|
read -r -p 'Type MIGRATE to approve downtime: ' approval
|
|
test "$approval" = MIGRATE
|
|
fi
|
|
|
|
backup=$(mktemp -d "$backup_dir/knot-cutover.XXXXXX")
|
|
printf '%s\n' "$old_system" > "$backup/old-system"
|
|
nix-store --realise "$old_system" --add-root "$backup/system" >/dev/null
|
|
systemctl stop auto-update.timer
|
|
downtime=1
|
|
systemctl stop knot.service
|
|
sqlite3 "file:$source_dir/knotserver.db?mode=ro" ".backup '$backup/knotserver.db'"
|
|
tar --acls --xattrs --numeric-owner -czf "$backup/go-knot.tar.gz" \
|
|
"$source_dir" "$ssh_key" "$ssh_key.pub"
|
|
migration_started=1
|
|
KNOT_MASTER_KEY="$master_key" knot-migrate "${migration_args[@]}"
|
|
test -f "$state/ssh_host_key" && test -s "$state/ssh_host_key"
|
|
test -f "$state/sealed-keys" && test -s "$state/sealed-keys"
|
|
read -r _ expected_key _ < <(ssh-keygen -lf "$ssh_key.pub")
|
|
read -r _ imported_key _ < <(ssh-keygen -lf "$state/ssh_host_key")
|
|
test "$expected_key" = "$imported_key"
|
|
KNOT_MASTER_KEY="$master_key" "$system/sw/bin/knot-server" --config "$system/etc/knot/config.toml" validate
|
|
if [ -f "$state/repo-signing-keys.json" ]; then
|
|
mv "$state/repo-signing-keys.json" "$backup/"
|
|
fi
|
|
|
|
activation_attempted=1
|
|
nix-env --profile "$profile" --set "$system"
|
|
"$system/bin/switch-to-configuration" switch
|
|
chown -R knot:knot "$state"
|
|
chmod 0600 "$state/ssh_host_key" "$state/sealed-keys"
|
|
cmp --silent "$runtime/knot.env" /run/agenix/knot-env
|
|
touch "$state/.migration-complete"
|
|
systemctl start knot-rs.service
|
|
smoke
|
|
verified=1
|
|
touch "$state/.upgrade-verified"
|
|
systemctl unmask --runtime auto-update.timer
|
|
systemctl start auto-update.timer
|
|
systemctl is-active --quiet auto-update.timer
|
|
if ! curl --fail --silent --show-error --max-time 15 -X POST \
|
|
https://index.tangled.network/xrpc/com.atproto.sync.requestCrawl \
|
|
-H 'Content-Type: application/json' -d '{"hostname":"knot.commonscomputer.com"}'; then
|
|
printf 'Warning: request the indexer crawl again later; the local migration passed.\n' >&2
|
|
fi
|
|
printf '\nSUCCESS: Rust knot and spindle are healthy; weekly automatic updates are active.\n'
|
|
printf 'Backups: %s (includes plaintext repository signing keys; back up encrypted off-host).\n' "$backup"
|
|
printf 'Next: test an authenticated clone/push over Git SSH port 2223.\n'
|