{ pkgs, ... }: pkgs.writeShellApplication { name = "upgrader"; runtimeInputs = with pkgs; [ ripgrep fzf jq gawk ]; text = '' CHECK_TIMEOUT=120 usage() { exit_code="''${1:-0}" echo "upgrader TARGET_HOST" echo " -h,--help Show help" echo " --wait-and-rollback GEN Wait CHECK_TIMEOUT seconds and rollback to the boot generation [REMOTE]" echo " --ignore-lock Ignore locks" exit "$exit_code" } ilog() { echo "$1" >> .upgrader.log } wait_and_rollback() { if [[ ! $EUID = 0 ]]; then exit 1 fi touch .upgrader.watcher.lck trap "" EXIT trap "echo 'watcher: exiting' >> .upgrader.log; rm -f .upgrader.watcher.lck" EXIT boot_gen="$(cat /boot/loader/loader.conf | rg 'default\s*nixos-generation-(\d+)\.conf' -r '\$1'")" ilog "watcher: boot gen is '$boot_gen'" while ! pgrep nixos-rebuild &>/dev/null; do sleep 1; done ilog 'watcher: nixos-rebuild started' while pgrep nixos-rebuild &>/dev/null; do sleep 1 done ilog 'watcher: nixos-rebuild finished' for i in $(seq 0 "$CHECK_TIMEOUT"); do ilog "watcher: wait $i" sleep 1 done ilog "watcher: was not killed, rolling back to boot gen '$boot_gen'" nix-env --switch-generation "$gen" -p /nix/var/nix/profiles/system /nix/var/nix/profiles/system/bin/switch-to-configuration switch exit 1 } [[ $# -lt 1 ]] && { printf "error: no TARGET_HOST argument provided\n" >&2; usage 1; } target_host="$1" shift ignore_locks=0 while [ "$#" -gt 0 ]; do case "$1" in --wait-and-rollback) wait_and_rollback "$@" ;; --ignore-locks) ignore_locks=1 break ;; -h | --help | help) usage 0 ;; *) echo "error: unexpected argument '$1' (if you meant to pass this to the underlying command(s), use '-- $1')" usage 1 esac shift done ssh "$target_host" true echo "SSH access to '$target_host' confirmed!" # TODO: install upgrader on remote host and know how to call it log() { echo "$1" ssh "$target_host" 'echo "$(date) $1" >> .upgrader.log' } remote_bg_job() { ssh "$target_host" 'nohup sh -c "( ( '"$1"' ) & )"' } if [[ $ignore_locks == 0 ]] && ssh "$target_host" "stat .upgrader.lck &>/dev/null" &>/dev/null; then log ".upgrader.lck exists on '$target_host', refusing to upgrade" exit 1 fi # TODO: handle boot-level failures? boot_gen="$(ssh "$target_host" "cat /boot/loader/loader.conf | rg 'default\s*nixos-generation-(\d+)\.conf' -r '\$1'")" log "Current Boot Generation for '$target_host': $boot_gen" # TODO: start some kind of "waiter" remote_bg_job 'remote_upgrader --wait-and-rollback' log "Upgrading '$target_host' (may require sudo prompt)..." ssh -t "$target_host" "touch .upgrader.lck; sudo nixos-rebuild test --flake git+https://git.lyte.dev/lytedev/nix" ssh -t "$target_host" 'remote_upgrader --kill-waiter' log 'Promoting current configuration to boot generation (may require sudo prompt)...' ssh -t "$target_host" 'sudo /run/current-system/bin/switch-to-configuration switch' ''; }