asusmini: script Tangled cutover and resume checked updates

scripts/upgrade-knot.sh

@@ -0,0 +1,226 @@
+#!/usr/bin/env bash
+set -euo pipefail
+unset KNOT_MASTER_KEY
+
+# These paths are deliberately fixed to this host and its rehearsed migration.
+stage=/home/anish/tangled-v1.20.0-preflight
+repo=/etc/commonscomputing-nix
+state=/var/lib/knot
+source_dir=/home/git
+backup_dir=/var/backups
+profile=/nix/var/nix/profiles/system
+current_system=/run/current-system
+ssh_key=/etc/ssh/ssh_host_ed25519_key
+lock_file=/run/lock/commonscomputing-update.lock
+runtime_template=/run/knot-upgrade.XXXXXX
+system_root=/var/lib/knot-upgrade-system
+tools_root=/var/lib/knot-upgrade-tools
+
+if [ "$(id -u)" != 0 ] || [ "$(hostname)" != asusmininix ]; then
+  printf 'Run this script as root on asusmininix.\n' >&2
+  exit 1
+fi
+if [ "$#" -gt 1 ] || { [ "$#" -eq 1 ] && [ "$1" != --yes ]; }; then
+  printf 'Usage: bash scripts/upgrade-knot.sh [--yes]\n' >&2
+  exit 1
+fi
+exec 9>"$lock_file"
+flock --nonblock 9 || { printf 'Another update is running.\n' >&2; exit 1; }
+if systemctl is-active --quiet auto-update.service; then
+  printf 'Wait for auto-update.service to finish before migrating.\n' >&2
+  exit 1
+fi
+
+system=$(<"$stage/system-path")
+tools=$(<"$stage/tools-path")
+case "$system" in /nix/store/*-nixos-system-asusmininix-*) ;; *) exit 1 ;; esac
+case "$tools" in /nix/store/*-knot-migration-tools) ;; *) exit 1 ;; esac
+if ! nix-store --check-validity "$system" "$tools"; then
+  read -r expected _ < "$stage/upgrade.export.gz.sha256"
+  actual=$(sha256sum "$stage/upgrade.export.gz")
+  test "${actual%% *}" = "$expected"
+  gzip -dc "$stage/upgrade.export.gz" | nix-store --import --option require-sigs false
+fi
+nix-store --realise "$system" --add-root "$system_root" >/dev/null
+nix-store --realise "$tools" --add-root "$tools_root" >/dev/null
+export PATH="$tools/bin:$PATH"
+test -x "$system/bin/switch-to-configuration"
+test -x "$system/sw/bin/knot-server"
+# An older staged build intentionally disabled the timer; do not install it.
+grep -Fq 'ConditionPathExists=/var/lib/knot/.upgrade-verified' \
+  "$system/etc/systemd/system/auto-update.timer"
+grep -Fq 'ConditionPathExists=/var/lib/knot/.upgrade-verified' \
+  "$system/etc/systemd/system/auto-update.service"
+
+smoke() {
+  for unit in knot-rs.service spindle.service sshd.service; do
+    systemctl is-active --quiet "$unit"
+  done
+  curl --fail --silent --show-error --max-time 10 --retry 5 --retry-connrefused \
+    --retry-delay 1 http://127.0.0.1:5555/xrpc/_health | jq -e '.version | length > 0'
+  curl --fail --silent --show-error --max-time 10 --retry 5 --retry-connrefused \
+    --retry-delay 1 http://127.0.0.1:6555/_health | jq -e '.status == "ok"'
+  curl --fail --silent --show-error --max-time 15 \
+    https://knot.commonscomputer.com/xrpc/org.tangled.knot.describeKnot \
+    | jq -e '.did == "did:web:knot.commonscomputer.com" and .contact.email == "pds@email.commonscomputer.com"'
+  read -r _ expected_key _ < <(ssh-keygen -lf "$ssh_key.pub")
+  scanned=$(ssh-keyscan -T 5 -p 2223 -t ed25519 127.0.0.1 2>/dev/null)
+  read -r _ actual_key _ < <(printf '%s\n' "$scanned" | ssh-keygen -lf -)
+  test "$expected_key" = "$actual_key"
+  # The upgraded endpoint is a long-lived WebSocket: timeout after the handshake.
+  status=$(curl --silent --max-time 3 --output /dev/null --write-out '%{http_code}' \
+    -H 'Connection: Upgrade' -H 'Upgrade: websocket' -H 'Sec-WebSocket-Version: 13' \
+    -H 'Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==' \
+    https://knot.commonscomputer.com/xrpc/com.atproto.sync.subscribeRepos || true)
+  test "$status" = 101
+}
+
+export GIT_TERMINAL_PROMPT=0
+export GIT_SSH_COMMAND='ssh -o BatchMode=yes -o ConnectTimeout=15'
+git -C "$repo" ls-remote --exit-code origin HEAD >/dev/null
+git -C "$repo" push --dry-run origin HEAD:main
+git -C "$repo" var GIT_AUTHOR_IDENT >/dev/null
+if ! git -C "$repo" diff --cached --quiet || \
+   ! git -C "$repo" diff --quiet -- '*.nix' flake.lock; then
+  printf 'Commit/stash staged changes and Nix/lockfile edits before cutover.\n' >&2
+  exit 1
+fi
+if [ -e "$state/.migration-complete" ]; then
+  printf 'Migration marker already exists; checking the live Rust service only.\n'
+  live_system=$(readlink -f "$current_system")
+  grep -Fq 'ConditionPathExists=/var/lib/knot/.upgrade-verified' \
+    "$live_system/etc/systemd/system/auto-update.timer"
+  grep -Fq 'ConditionPathExists=/var/lib/knot/.upgrade-verified' \
+    "$live_system/etc/systemd/system/auto-update.service"
+  smoke
+  touch "$state/.upgrade-verified"
+  systemctl unmask --runtime auto-update.timer
+  systemctl start auto-update.timer
+  systemctl is-active --quiet auto-update.timer
+  printf 'Already migrated; smoke tests passed and automatic updates are active.\n'
+  exit 0
+fi
+if [ -e "$state" ]; then
+  printf '%s exists without a completed migration. Inspect/quarantine it before retrying.\n' "$state" >&2
+  exit 1
+fi
+systemctl is-active --quiet knot.service
+test -r "$source_dir/knotserver.db"
+test -s "$ssh_key"
+old_system=$(readlink -f "$current_system")
+runtime=""
+backup=""
+downtime=0
+activation_attempted=0
+migration_started=0
+verified=0
+
+cleanup() {
+  result=$?
+  trap - EXIT INT TERM
+  set +e
+  if [ "$result" -ne 0 ] && [ "$downtime" -eq 1 ] && [ "$verified" -eq 0 ]; then
+    systemctl stop auto-update.timer
+    systemctl mask --runtime auto-update.timer
+    systemctl stop knot-rs.service
+    rm -f "$state/.migration-complete" "$state/.upgrade-verified"
+    if [ "$activation_attempted" -eq 1 ]; then
+      printf 'Activation was attempted: state may have changed. Updates/Rust knot are stopped.\n' >&2
+      printf 'Do NOT blindly roll back; inspect %s and %s before reopening writes.\n' "$state" "$backup" >&2
+    else
+      printf 'Migration failed before activation; restarting the unchanged Go knot.\n' >&2
+      if [ "$migration_started" -eq 1 ] && [ -e "$state" ]; then
+        mv "$state" "$backup/failed-target"
+      fi
+      if ! systemctl start knot.service; then
+        printf 'URGENT: Go knot could not be restarted; inspect %s.\n' "$backup" >&2
+      fi
+    fi
+    printf 'Backups retained in %s. Timer runtime mask does not survive reboot.\n' "$backup" >&2
+  fi
+  if [ -n "$runtime" ]; then
+    rm -f "$runtime/knot.env"
+    rmdir "$runtime"
+  fi
+  unset master_key
+  exit "$result"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+umask 077
+runtime=$(mktemp -d "$runtime_template")
+agenix -d "$repo/secrets/knot-env.age" -i "$ssh_key" > "$runtime/knot.env"
+chmod 0400 "$runtime/knot.env"
+master_key=""
+export -n master_key
+while IFS= read -r line || [ -n "$line" ]; do
+  case "$line" in
+    KNOT_MASTER_KEY=*) master_key=${line#KNOT_MASTER_KEY=} ;;
+    ''|\#*) ;;
+    *) printf 'Unexpected entry in knot-env.age: expected only KNOT_MASTER_KEY.\n' >&2; exit 1 ;;
+  esac
+done < "$runtime/knot.env"
+if ! [[ "$master_key" =~ ^[A-Za-z0-9+/]{43}=$ ]]; then
+  printf 'KNOT_MASTER_KEY must be an unquoted base64 encoding of 32 random bytes.\n' >&2
+  exit 1
+fi
+migration_args=(
+  --source-db "$source_dir/knotserver.db" --source-repos "$source_dir"
+  --hostname knot.commonscomputer.com --host-key "$ssh_key"
+  --plc-url https://plc.directory --contact-email pds@email.commonscomputer.com
+  --master-key-env KNOT_MASTER_KEY --object-format sha1 --target "$state"
+)
+KNOT_MASTER_KEY="$master_key" knot-migrate "${migration_args[@]}" --dry-run
+printf '\nThis will stop Go knot, back it up, COPY its data, and install the staged Rust system.\n'
+printf 'Git SSH moves to port 2223; administrative SSH stays on 22. Use Tailscale or arrange WAN forwarding.\n'
+printf 'Confirm the master key is backed up off-host and keep a second administrative session open.\n'
+if [ "${1:-}" != --yes ]; then
+  read -r -p 'Type MIGRATE to approve downtime: ' approval
+  test "$approval" = MIGRATE
+fi
+
+backup=$(mktemp -d "$backup_dir/knot-cutover.XXXXXX")
+printf '%s\n' "$old_system" > "$backup/old-system"
+nix-store --realise "$old_system" --add-root "$backup/system" >/dev/null
+systemctl stop auto-update.timer
+downtime=1
+systemctl stop knot.service
+sqlite3 "file:$source_dir/knotserver.db?mode=ro" ".backup '$backup/knotserver.db'"
+tar --acls --xattrs --numeric-owner -czf "$backup/go-knot.tar.gz" \
+  "$source_dir" "$ssh_key" "$ssh_key.pub"
+migration_started=1
+KNOT_MASTER_KEY="$master_key" knot-migrate "${migration_args[@]}"
+test -f "$state/ssh_host_key" && test -s "$state/ssh_host_key"
+test -f "$state/sealed-keys" && test -s "$state/sealed-keys"
+read -r _ expected_key _ < <(ssh-keygen -lf "$ssh_key.pub")
+read -r _ imported_key _ < <(ssh-keygen -lf "$state/ssh_host_key")
+test "$expected_key" = "$imported_key"
+KNOT_MASTER_KEY="$master_key" "$system/sw/bin/knot-server" --config "$system/etc/knot/config.toml" validate
+if [ -f "$state/repo-signing-keys.json" ]; then
+  mv "$state/repo-signing-keys.json" "$backup/"
+fi
+
+activation_attempted=1
+nix-env --profile "$profile" --set "$system"
+"$system/bin/switch-to-configuration" switch
+chown -R knot:knot "$state"
+chmod 0600 "$state/ssh_host_key" "$state/sealed-keys"
+cmp --silent "$runtime/knot.env" /run/agenix/knot-env
+touch "$state/.migration-complete"
+systemctl start knot-rs.service
+smoke
+verified=1
+touch "$state/.upgrade-verified"
+systemctl unmask --runtime auto-update.timer
+systemctl start auto-update.timer
+systemctl is-active --quiet auto-update.timer
+if ! curl --fail --silent --show-error --max-time 15 -X POST \
+  https://index.tangled.network/xrpc/com.atproto.sync.requestCrawl \
+  -H 'Content-Type: application/json' -d '{"hostname":"knot.commonscomputer.com"}'; then
+  printf 'Warning: request the indexer crawl again later; the local migration passed.\n' >&2
+fi
+printf '\nSUCCESS: Rust knot and spindle are healthy; weekly automatic updates are active.\n'
+printf 'Backups: %s (includes plaintext repository signing keys; back up encrypted off-host).\n' "$backup"
+printf 'Next: test an authenticated clone/push over Git SSH port 2223.\n'