From 114bed13ba232c821f4d9af052e6035c96335045 Mon Sep 17 00:00:00 2001 From: Anish Lakhwara Date: Sun, 1 Nov 2026 15:01:46 -0700 Subject: [PATCH] asusmini: prepare guarded Tangled v1.20.0 cutover --- README.md | 11 +- TANGLED-MIGRATION.md | 295 +++++++++++++++++++++++++++++++++ flake.lock | 27 +-- flake.nix | 21 ++- hosts/asusmini/atproto.nix | 41 ++++- hosts/asusmini/auto-update.nix | 9 +- secrets/knot-env.age | 19 +++ secrets/secrets.nix | 1 + 8 files changed, 398 insertions(+), 26 deletions(-) create mode 100644 TANGLED-MIGRATION.md create mode 100644 secrets/knot-env.age diff --git a/README.md b/README.md index 3703c13..ec7e883 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,15 @@ Infrastructure-as-code for Commons Computing's ATProto services. This flake manages the following services on `asusmini`: - **PDS** (Personal Data Server) - `pds.commonscomputer.com:5556` -- **Tangled Knot** (Git forge) - `knot.commonscomputer.com:5555` +- **Tangled Knot** (Rust Git forge) - HTTP `5555`, Git SSH `2223` - **Tangled Spindle** (CI runner) - `spindle.commonscomputer.com:6555` All services are proxied through Caddy with automatic HTTPS. +The supervised Go-to-Rust cutover is documented in [TANGLED-MIGRATION.md](TANGLED-MIGRATION.md). +Do not activate the new configuration before following its backup and migration steps. +Administrative SSH stays on port 22; Git SSH on 2223 needs explicit client routing. + ## Setup ### Prerequisites @@ -80,6 +84,11 @@ sudo nixos-rebuild switch --flake .#asusmini #### Auto-Update +The timer is currently disabled for the Tangled migration. Tangled is pinned +to v1.20.0 and uses its own locked dependencies so the tested binaries can +be copied to the server without recompilation. Re-enable updates deliberately +only after cutover and verification. + A systemd timer runs weekly to: 1. Pull latest git changes 2. Run `nix flake update` diff --git a/TANGLED-MIGRATION.md b/TANGLED-MIGRATION.md new file mode 100644 index 0000000..3fd7f6a --- /dev/null +++ b/TANGLED-MIGRATION.md @@ -0,0 +1,295 @@ +# Tangled v1.20.0 Preflight + +Status: local and server smoke tests passed. The operator completed the +real-data dry-run successfully: 23 repositories, 16 members, 12 collaborator +grants, zero skips, and no ACL/table disagreement. The master key has an +off-host backup. Nothing has been deployed or migrated. + +The operator created `secrets/knot-env.age` on asusmini. The encrypted file +and its recipient rule have been synchronized into this checkout; the key +has not been decrypted here. The new configuration wires it into agenix at +`/run/agenix/knot-env`, root-owned with mode `0400`. + +## Release And Build Outputs + +Verified upstream tag: `v1.20.0`, commit +`2096d06d938e680297adeadd71b8320c9f3491d4`. + +Built on the workstation using the release's own locked inputs, not this +repository's `nixpkgs` override: + +| Package | Exact Store Path | +| --- | --- | +| Rust knot | `/nix/store/7wv0kkflzpihyzm9k7mahxdci97r8k0h-knot-server-1.20.0` | +| Migrator | `/nix/store/h3d5c7wad0y6nmlkllrjq95g15j6zgpw-knot-migrate-1.20.0` | +| Spindle | `/nix/store/id8ma5qaj5q6msfjw2llppsz7w46jy9i-spindle-1.20.0` | + +These outputs have local GC roots under `/tmp/opencode/`. Do not delete them +until the outputs have durable roots on the server. + +## Completed Checks + +- All three packages built successfully. +- Rust knot configuration validation and HTTP health passed at v1.20.0. +- Knot describe/contact, DID document, and SSH host-key matching passed. +- Unauthenticated repository creation was rejected with 401/403. +- Knot signing identity and SSH identity persisted across restart. +- Spindle started with disposable state; health, DID, and contact passed. +- Synthetic Go-database migration passed both dry-run and copy migration. +- Synthetic migration preserved the source database and bare repository. +- A temporary NixOS configuration evaluated with Rust knot and spindle at + 1.20.0, Go knot disabled, and Rust `wantedBy = []`. +- Uploaded the closure archive and test scripts to the server and verified + that the archive checksum matches the local copy. + +The upstream v1.20.0 spindle source hardcodes `wireVersion = "v1.16.1"` in +`spindle/xrpc/org_tangled_spindle_describe_spindle.go`. Its describe endpoint +is not a reliable indicator of the package release; verify the store path. + +These are startup and migration smoke tests, not a full CI workflow test or +an authenticated end-to-end clone/push test against migrated production data. + +## Server Artifacts + +SSH: `anish@pds.commonscomputer.com`. + +Directory: `/home/anish/tangled-v1.20.0-preflight`. + +Archive: `tangled-v1.20.0-closure.export.gz` (137 MiB compressed), containing +the package closures plus Python/OpenSSL for the test harness. + +SHA-256: +`aa50f7edf3f2e4721f27abc86e4cdd6476d66eeed3dd013ddc5252dcc228a4d3`. + +The plain `nix copy --to ssh-ng://anish@pds.commonscomputer.com` attempt was +rejected because the build outputs are unsigned and only root is trusted. +No signature policy or trusted-user configuration was changed. + +## Network Plan + +Administrative OpenSSH remains on port **22**. Rust knot uses **2223**; +the upstream module opens that port in the NixOS firewall. Public port 2222 +already belongs to the separate Gitea service and must not be repurposed. + +The Ansible Caddy template already proxies all requests, including WebSocket +upgrades, from `knot.commonscomputer.com` to `192.168.1.249:5555`. No path-specific +`/events` rule needs replacing. Knot trusts forwarded addresses only from +the Caddy LXC, `192.168.1.11/32`. + +WAN forwarding is not managed by either repository. Before cutover, either +arrange external TCP 2223 -> `192.168.1.249:2223`, or use the existing Tailscale +address `100.122.40.27:2223`. SSH Git clients must specify the new port. The +old port-22 Git SSH URLs will not work after removing the Go module; do not +redirect public port 22 if that is your only administrative access path. + +## Stage The Built System (No Downtime) + +The system build and migration-tools bundle are exported separately from +the earlier smoke archive. The staging directory contains `system-path`, +`tools-path`, `upgrade.export.gz`, and `upgrade.export.gz.sha256`. + +As root on asusmini, before stopping anything: + +```sh +sudo -i +``` + +In that root Bash session: + +```sh +set -euo pipefail +test "$EUID" -eq 0 +cd /home/anish/tangled-v1.20.0-preflight +sha256sum --check upgrade.export.gz.sha256 +gzip -dc upgrade.export.gz | nix-store --import --option require-sigs false +system=$(cat system-path) +tools=$(cat tools-path) +nix-store --realise "$system" --add-root /var/lib/knot-upgrade-system +nix-store --realise "$tools" --add-root /var/lib/knot-upgrade-tools +test -x "$system/bin/switch-to-configuration" +test -x "$tools/bin/knot-migrate" +``` + +The archive contains only paths missing from the server at staging time. +Do not garbage-collect the server store between staging and activation. +The signature exception applies to this one root-authorized import, not to +persistent Nix configuration. Importing and rooting paths do not activate them. + +After committing the encrypted secret on the server if necessary, pull the +configuration with `git pull --ff-only` in `/etc/commonscomputing-nix`. Do not +overwrite unrelated DNS edits. Activation below uses the exact built path, +not a fresh flake evaluation or update. + +## Decrypt And Rehearse With The Persistent Key + +Continue in the same root Bash session. Do not enable shell tracing (`set -x`). + +```sh +export PATH="$tools/bin:$PATH" +umask 077 +runtime=$(mktemp -d /run/knot-migration.XXXXXX) +trap 'rm -f "$runtime/knot.env"; rmdir "$runtime"; unset KNOT_MASTER_KEY' EXIT +agenix -d /etc/commonscomputing-nix/secrets/knot-env.age \ + -i /etc/ssh/ssh_host_ed25519_key > "$runtime/knot.env" +chmod 0400 "$runtime/knot.env" +set -a +. "$runtime/knot.env" +set +a +test -n "${KNOT_MASTER_KEY:-}" + +migration_args=( + --source-db /home/git/knotserver.db + --source-repos /home/git + --hostname knot.commonscomputer.com + --host-key /etc/ssh/ssh_host_ed25519_key + --plc-url https://plc.directory + --contact-email pds@email.commonscomputer.com + --master-key-env KNOT_MASTER_KEY + --object-format sha1 + --target /var/lib/knot +) +knot-migrate "${migration_args[@]}" --dry-run +``` + +Check the expected 23 repositories, 16 members, 12 grants, zero skips, and +no ownership drift. SQLite may create WAL/SHM files for read-only access. +This uses the real backed-up key, not the disposable preflight script's key. +If it fails, stop here; production is still running. + +## Cut Over (Downtime Starts Here) + +Run only after explicitly approving downtime and confirming the SSH plan. +Keep a second administrative SSH connection open. Do not reboot midway. + +```sh +test ! -e /var/lib/knot/.migration-complete +test ! -e /var/lib/knot/sealed-keys +old_system=$(readlink -f /run/current-system) +backup=/var/backups/knot-cutover-$(date +%Y%m%dT%H%M%S) +install -d -m 0700 "$backup" +printf '%s\n' "$old_system" > "$backup/old-system" +nix-store --realise "$old_system" --add-root "$backup/system" + +systemctl stop knot.service +sqlite3 'file:/home/git/knotserver.db?mode=ro' ".backup '$backup/knotserver.db'" +tar --acls --xattrs --numeric-owner -C / -czf "$backup/go-knot.tar.gz" \ + home/git etc/ssh/ssh_host_ed25519_key etc/ssh/ssh_host_ed25519_key.pub +knot-migrate "${migration_args[@]}" +``` + +Inspect the migration report before continuing. The default is **copy** mode; +never add `--consume-source`. Source repositories/database remain in place. +On any error, do not create the marker or start Rust knot. The old service +can be restarted while investigating (`systemctl start knot.service`). A +failed migration may leave partial target state; inspect it and quarantine +the entire target before a retry. Never blindly reuse it or delete the source. + +```sh +test -s /var/lib/knot/sealed-keys +test -f /var/lib/knot/sealed-keys +test -s /var/lib/knot/ssh_host_key +test -f /var/lib/knot/ssh_host_key +ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub +ssh-keygen -y -f /var/lib/knot/ssh_host_key | ssh-keygen -lf - +``` + +The two fingerprints must match. Then validate the actual deployment config: + +```sh +/nix/store/7wv0kkflzpihyzm9k7mahxdci97r8k0h-knot-server-1.20.0/bin/knot-server \ + --config "$system/etc/knot/config.toml" validate +if [ -f /var/lib/knot/repo-signing-keys.json ]; then + mv /var/lib/knot/repo-signing-keys.json "$backup/" +fi +``` + +`repo-signing-keys.json` contains plaintext repository private keys. Keep the +backup directory root-only and back it up encrypted off-host. Do not commit it. + +Install the prebuilt system. The absent marker keeps Rust stopped while +activation creates the `knot` service user and decrypts the agenix secret: + +```sh +nix-env --profile /nix/var/nix/profiles/system --set "$system" +"$system/bin/switch-to-configuration" switch +chown -R knot:knot /var/lib/knot +chmod 0600 /var/lib/knot/ssh_host_key /var/lib/knot/sealed-keys +test -s /run/agenix/knot-env +cmp --silent "$runtime/knot.env" /run/agenix/knot-env +touch /var/lib/knot/.migration-complete +if ! systemctl start knot-rs.service || \ + ! curl --fail --silent --show-error --retry 10 --retry-connrefused \ + --retry-delay 1 http://127.0.0.1:5555/xrpc/_health; then + rm -f /var/lib/knot/.migration-complete + systemctl stop knot-rs.service + exit 1 +fi +rm -f "$runtime/knot.env" +rmdir "$runtime" +unset KNOT_MASTER_KEY +trap - EXIT +``` + +Stop on any activation error rather than continuing to create the marker. +The marker allows the service to start normally on subsequent boots. +Auto-update remains disabled until deliberately re-enabled after verification. + +## Final Checks + +```sh +systemctl is-active knot-rs spindle sshd +systemctl is-active knot.service || true +journalctl -u knot-rs -n 80 --no-pager +curl --fail http://127.0.0.1:5555/xrpc/_health +curl --fail https://knot.commonscomputer.com/xrpc/_health +curl --fail https://knot.commonscomputer.com/xrpc/org.tangled.knot.describeKnot +curl --fail https://knot.commonscomputer.com/.well-known/did.json +ssh-keyscan -p 2223 -t ed25519 127.0.0.1 2>/dev/null | ssh-keygen -lf - +``` + +Expect v1.20.0, the original knot hostname/DID and contact, and the original +SSH fingerprint. From a client, clone a known migrated repository, push a +temporary branch using port 2223, then delete that branch. Check membership +and collaborator access in the web UI. Test WebSocket upgrade for +`/xrpc/com.atproto.sync.subscribeRepos` through Caddy; a successful HTTP health +request alone does not prove the event stream works. + +Only after verification, request an indexer crawl: + +```sh +curl --fail -X POST https://index.tangled.network/xrpc/com.atproto.sync.requestCrawl \ + -H 'Content-Type: application/json' \ + -d '{"hostname":"knot.commonscomputer.com"}' +``` + +## Roll Back If Needed + +Before activation, just restart `knot.service`; the source was copied, not +moved. After activation, use the preserved generation: + +```sh +systemctl stop knot-rs.service +rm -f /var/lib/knot/.migration-complete +old_system=$(cat "$backup/old-system") +nix-env --profile /nix/var/nix/profiles/system --set "$old_system" +"$old_system/bin/switch-to-configuration" switch +systemctl stop auto-update.timer +systemctl mask --runtime auto-update.timer +systemctl start knot.service +``` + +The old generation may re-enable the timer, hence the explicit stop and +runtime mask. The mask does not survive reboot: do not reboot the rolled-back +generation until a declarative configuration with the timer disabled is ready. +Retain all backups and migrated state. Writes made after cutover are not copied +back to the Go database by rollback; coordinate with users before reopening +writes. Never overwrite the source database unless recovery actually needs it. + +At the last check, production knot and spindle were active, the auto-update +timer was inactive, and `/run/current-system` was unchanged: +`/nix/store/sv9r0h1xs6fsfjvbqv2vwslj42wm08v4-nixos-system-asusmininix-26.11.20261027.31ecbac`. + +## Upstream Guides + +- https://tangled.org/docs/upgrading/knot-2/ +- https://tangled.org/docs/upgrading/knots-and-spindles/#upgrading-to-v1200 diff --git a/flake.lock b/flake.lock index 93bab8f..5e7cf42 100644 --- a/flake.lock +++ b/flake.lock @@ -373,18 +373,18 @@ "type": "github" } }, - "nixpkgs-pds": { + "nixpkgs_2": { "locked": { - "lastModified": 1791734076, - "narHash": "sha256-i5qk1kLiit98rRbh1AXmWBNT9QXC/y4l5Wk/u5EdpDQ=", + "lastModified": 1781216227, + "narHash": "sha256-9mUW6gNwoN2SWc/l0fW4svPNOulXLl8ijqKyeSOGgJE=", "owner": "nixos", "repo": "nixpkgs", - "rev": "7fd4004d0511991fff3a0524856a9be09e28c5f0", + "rev": "a0374025a863d007d98e3297f6aa46cc3141c2f0", "type": "github" }, "original": { "owner": "nixos", - "ref": "nixos-unstable", + "ref": "nixos-26.05", "repo": "nixpkgs", "type": "github" } @@ -505,24 +505,25 @@ "mathjax-src": "mathjax-src", "mermaid-src": "mermaid-src", "microvm": "microvm", - "nixpkgs": [ - "nixpkgs" - ], + "nixpkgs": "nixpkgs_2", "nixpkgs-pds": "nixpkgs-pds", "sqlite-lib-src": "sqlite-lib-src", "tailwindcss-animated-src": "tailwindcss-animated-src", "timeless-fonts-src": "timeless-fonts-src" }, "locked": { - "lastModified": 1793301352, - "narHash": "sha256-fl6toQZeetHRBEGlYgEwdVXnOz7wTuWhb8XbZbnIzdw=", - "ref": "refs/heads/master", - "rev": "dfcc014ecfdce4e4fc5e4b33691c8ba17ea61a47", - "revCount": 6003, + "lastModified": 1793354951, + "narHash": "sha256-5Q5bJvpE/eB0FJirmsL7YXXN+FXjtrQmXaOXJ3f1rEQ=", + "ref": "refs/tags/v1.20.0", + "rev": "2096d06d938e680297adeadd71b8320c9f3491d4", + "shallow": true, "type": "git", "url": "https://tangled.org/@tangled.org/core" }, "original": { + "ref": "refs/tags/v1.20.0", + "rev": "2096d06d938e680297adeadd71b8320c9f3491d4", + "shallow": true, "type": "git", "url": "https://tangled.org/@tangled.org/core" } diff --git a/flake.nix b/flake.nix index 86d8ae1..104489f 100644 --- a/flake.nix +++ b/flake.nix @@ -4,8 +4,8 @@ inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; tangled = { - url = "git+https://tangled.org/@tangled.org/core"; - inputs.nixpkgs.follows = "nixpkgs"; + # Keep the release's own package set so deployment reuses tested binaries. + url = "git+https://tangled.org/@tangled.org/core?ref=refs/tags/v1.20.0&rev=2096d06d938e680297adeadd71b8320c9f3491d4&shallow=1"; }; agenix = { url = "github:ryantm/agenix"; @@ -36,7 +36,7 @@ system = "x86_64-linux"; modules = [ ./hosts/asusmini - tangled.nixosModules.knot + tangled.nixosModules.knot-rs tangled.nixosModules.spindle agenix.nixosModules.default inputs."automerge-repo".nixosModules.default @@ -45,6 +45,21 @@ }; }; + packages.x86_64-linux.knot-migration-tools = + let + pkgs = nixpkgs.legacyPackages.x86_64-linux; + in + pkgs.buildEnv { + name = "knot-migration-tools"; + paths = [ + tangled.packages.x86_64-linux.knot-migrate + agenix.packages.x86_64-linux.default + pkgs.sqlite + pkgs.openssl + pkgs.git + ]; + }; + devShells = { aarch64-darwin.default = let diff --git a/hosts/asusmini/atproto.nix b/hosts/asusmini/atproto.nix index 3245635..a6c7e92 100644 --- a/hosts/asusmini/atproto.nix +++ b/hosts/asusmini/atproto.nix @@ -6,6 +6,12 @@ owner = "pds"; }; + age.secrets.knot-env = { + file = ../../secrets/knot-env.age; + mode = "0400"; + owner = "root"; + }; + services.bluesky-pds = { enable = true; environmentFiles = [ config.age.secrets.pds-env.path ]; @@ -52,14 +58,41 @@ fi ''; - services.tangled.knot = { + services.tangled.knot-rs = { enable = true; - server = { - hostname = "knot.commonscomputer.com"; - owner = "did:plc:om5yygegi4yxcbay5gemn2wm"; + environmentFile = config.age.secrets.knot-env.path; + settings = { + server = { + hostname = "knot.commonscomputer.com"; + admins = [ "did:plc:om5yygegi4yxcbay5gemn2wm" ]; + # Caddy is on another machine; administrative sshd keeps port 22. + listen_addr = "0.0.0.0:5555"; + ssh_listen_addr = "0.0.0.0:2223"; + }; + git.object_format = "sha1"; + atproto.plc_directory = "https://plc.directory"; + contact.email = "pds@email.commonscomputer.com"; + lfs.store_path = "/var/lib/knot/lfs"; + xrpc = { + trusted_proxy_header = "x-forwarded-for"; + trusted_proxies = [ "192.168.1.11/32" ]; + }; + tls.http3 = false; }; }; + # Activation may create directories, but must not mint a new empty identity. + # Create this marker only after migration and ownership checks (see runbook). + systemd.services.knot-rs.unitConfig.ConditionPathExists = [ + "/var/lib/knot/.migration-complete" + "/var/lib/knot/ssh_host_key" + "/var/lib/knot/sealed-keys" + ]; + systemd.services.knot-rs.serviceConfig.ExecStartPre = [ + "${pkgs.coreutils}/bin/test -s /var/lib/knot/ssh_host_key" + "${pkgs.coreutils}/bin/test -s /var/lib/knot/sealed-keys" + ]; + services.automerge-repo-sync-server.enable = true; # Public edge moved 2026-09-28: 205.250.24.44 -> pve Caddy LXC (192.168.1.11) diff --git a/hosts/asusmini/auto-update.nix b/hosts/asusmini/auto-update.nix index 0a7d4fb..440ede3 100644 --- a/hosts/asusmini/auto-update.nix +++ b/hosts/asusmini/auto-update.nix @@ -24,11 +24,8 @@ echo "Updating flake inputs..." nix flake update - # Explicitly update tangled so we always pull the latest knot/spindle - # builds, even if other inputs are pinned or the general update is - # later restricted. tangled.org/@tangled.org/core moves quickly and - # we want to track master. - nix flake update tangled + # Tangled stays at the release pinned in flake.nix. Upgrade it deliberately + # after migration/testing rather than tracking master automatically. # Check if there are changes to commit if ! git diff --quiet flake.lock; then @@ -53,6 +50,8 @@ }; systemd.timers.auto-update = { + # Re-enable deliberately after the supervised Tangled migration is verified. + enable = false; description = "Auto-update timer"; wantedBy = [ "timers.target" ]; timerConfig = { diff --git a/secrets/knot-env.age b/secrets/knot-env.age new file mode 100644 index 0000000..41bdf6e --- /dev/null +++ b/secrets/knot-env.age @@ -0,0 +1,19 @@ +age-encryption.org/v1 +-> ssh-ed25519 GixcEg j9ctdzN4xlfXX5Pc6yCvBGq0rus1GtNDzmjn5dF/GHk +F9zz064IPF3gFK4V5ixj3zSlWAj/U2ySZj4R7Iv9WXU +-> ssh-rsa POo0eA +mCjAL56LTqiiSb83bVdMxR9xPB9F+ySL266zlgrcA1QTU1h+CzIW9e/p1Tqhwltw +Ww3g66ODASLtAR0A/Haj0oJUpY+lNwDno9eIbsk8XuIz2RmNAouVTCn3Mf/Ugm7D +BoFHSdYLAuoTfJBSqRyPUU0aGR+9zAyQUHyC32vOekTs3EOzmhGfGt05Fo2ClJaH +LhSg1IkvBXOr8qJCr5fWNZvKx2TsCysM0X6eHQlEzlY2J8KBcYGX2c0CmlynbFrn +ECpwTsB2d7o1AYHuQbvDbPx03sLAu/Q++Dq/WseD9BfZlueJhWWaAPgmj3ulqG8g +PcE6lsjVs9JkGUezY8aJLM1zb4Zy4ENjW+dh66NVHV/FNL7CBnNPmQ/Iuw+gAfQb +2uvLPamKrGCvVAnOnmtU6a5VwGygALpavq780EK/W/ZNZs5YC10qOjdYkwhFAUEH +EacIVIJoMfjDyd+q6MIb4IkQidfS5RavVETOiu/XXnT8eJImTJHVtkPLG6PlxRTj + +-> ssh-ed25519 thu/uw NYs06NaZ3okyVjYrYynNfdC9h9a9ygNx80So7Sqc/ig +sHqd5OAdu4uLHrWl1ISkYj2FKPMRJdfUKQBSFmvXX3k +-> ssh-ed25519 HR5c7Q NRVLLirtUASQ2uDbTvs6lXN6P7AiXpNHuhAvInJ6Njg +5A2lVT7uI2oZAMPi5K/AWVQDU81bO3Ozdn286KOXym4 +--- W4jRPRoarq+RN5GTL2qZBXiIGxUjZJqkv/PDkKzfjCc +^mCy-RD 'ǥӏ#6_lz(nyұ|Kd1Ac<΂6]5tfjJ\1g| j \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index d25fd4f..b530dd1 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -11,4 +11,5 @@ in { "pds-env.age".publicKeys = allKeys; "dns-creds.age".publicKeys = allKeys; + "knot-env.age".publicKeys = allKeys; }