Added support for remote and local realization

Added getters to CopyOptions
feat: create colmena library
2024-12-08 19:06:41 +01:00 · 2024-12-07 15:32:48 +01:00 · 2024-10-10 21:04:01 +02:00 · 2024-10-06 01:32:25 +02:00 · 2024-10-03 18:42:52 +02:00 · 2024-09-26 12:30:14 +02:00
35 changed files with 1018 additions and 1198 deletions
--- a/.editorconfig
+++ b/.editorconfig
@ -10,7 +10,8 @@ insert_final_newline = true
 trim_trailing_whitespace = true
 charset = utf-8

-[*.json]
+# Rust
+[*.rs]
 indent_style = space
 indent_size = 2

--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -13,23 +13,19 @@ jobs:
            image: ubuntu-latest
            system: aarch64-linux
          - label: x86_64-darwin
-            image: macos-latest
-            system: x86_64-darwin
-          - label: aarch64-darwin
-            image: macos-latest
-            system: aarch64-darwin
+            image: macos-12

    name: ${{ matrix.label }}
    runs-on: ${{ matrix.image }}
    steps:
-      - uses: actions/checkout@v4.2.2
+      - uses: actions/checkout@v3.3.0

      - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@b92f66560d6f97d6576405a7bae901ab57e72b6a # v15
+        uses: DeterminateSystems/nix-installer-action@9b252454a8d70586c4ee7f163bf4bb1e9de3d763 # v2

      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3.2.0
-        if: matrix.system == 'aarch64-linux'
+        uses: docker/setup-qemu-action@v2.1.0
+        if: matrix.system != ''

      - name: Generate System Flags
        run: |
@ -43,7 +39,7 @@ jobs:
          HOST_SYSTEM: '${{ matrix.system }}'

      - name: Enable Binary Cache
-        uses: cachix/cachix-action@v15
+        uses: cachix/cachix-action@v12
        with:
          name: colmena
          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@ -53,60 +49,3 @@ jobs:

      - name: Build manual
        run: nix build .#manual -L
-
-  nix-matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@v4.2.2
-
-      - uses: DeterminateSystems/nix-installer-action@v15
-        continue-on-error: true # Self-hosted runners already have Nix installed
-
-      - name: Enable Binary Cache
-        uses: cachix/cachix-action@v15
-        with:
-          name: colmena
-          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-
-      - id: set-matrix
-        name: Generate Nix Matrix
-        run: |
-          set -Eeu
-          matrix="$(nix eval --json '.#githubActions.matrix')"
-          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
-
-  nix-matrix-job:
-    name: ${{ matrix.name }}
-    runs-on: ${{ matrix.os }}
-    needs:
-      - build
-      - nix-matrix
-    strategy:
-      matrix: ${{ fromJSON(needs.nix-matrix.outputs.matrix) }}
-    steps:
-      - name: Maximize build space
-        uses: easimon/maximize-build-space@master
-        with:
-          remove-dotnet: 'true'
-          build-mount-path: /nix
-
-      - name: Set /nix permissions
-        run: |
-          sudo chown root:root /nix
-
-      - uses: actions/checkout@v4.2.2
-
-      - uses: DeterminateSystems/nix-installer-action@v15
-        continue-on-error: true # Self-hosted runners already have Nix installed
-
-      - name: Enable Binary Cache
-        uses: cachix/cachix-action@v15
-        with:
-          name: colmena
-          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-
-      - name: Build ${{ matrix.attr }}
-        run: |
-          nix build --no-link --print-out-paths -L '.#${{ matrix.attr }}'
--- a/.github/workflows/linters.yml
+++ b/.github/workflows/linters.yml
@ -10,13 +10,13 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-      - uses: actions/checkout@v4.2.2
+      - uses: actions/checkout@v3.3.0

      - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@b92f66560d6f97d6576405a7bae901ab57e72b6a # v15
+        uses: DeterminateSystems/nix-installer-action@9b252454a8d70586c4ee7f163bf4bb1e9de3d763 # v2

      - name: Enable binary cache
-        uses: cachix/cachix-action@v15
+        uses: cachix/cachix-action@v12
        with:
          name: colmena
          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
--- a/.github/workflows/manual-stable.yml
+++ b/.github/workflows/manual-stable.yml
@ -16,13 +16,13 @@ jobs:
    if: github.repository == 'zhaofengli/colmena'

    steps:
-      - uses: actions/checkout@v4.2.2
+      - uses: actions/checkout@v3.3.0

      - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@b92f66560d6f97d6576405a7bae901ab57e72b6a # v15
+        uses: DeterminateSystems/nix-installer-action@9b252454a8d70586c4ee7f163bf4bb1e9de3d763 # v2

      - name: Enable Binary Cache
-        uses: cachix/cachix-action@v15
+        uses: cachix/cachix-action@v12
        with:
          name: colmena
          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@ -38,7 +38,7 @@ jobs:
        run: nix build .#manual -L

      - name: Deploy manual
-        uses: JamesIves/github-pages-deploy-action@v4.6.9
+        uses: JamesIves/github-pages-deploy-action@4.1.6
        with:
          branch: gh-pages
          folder: result
@ -52,7 +52,7 @@ jobs:
        if: ${{ env.api_version == env.latest_stable_api }}

      - name: Deploy redirect farm
-        uses: JamesIves/github-pages-deploy-action@v4.6.9
+        uses: JamesIves/github-pages-deploy-action@4.1.6
        with:
          branch: gh-pages
          folder: result-redirectFarm
--- a/.github/workflows/manual.yml
+++ b/.github/workflows/manual.yml
@ -16,13 +16,13 @@ jobs:
    if: github.repository == 'zhaofengli/colmena'

    steps:
-      - uses: actions/checkout@v4.2.2
+      - uses: actions/checkout@v3.3.0

      - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@b92f66560d6f97d6576405a7bae901ab57e72b6a # v15
+        uses: DeterminateSystems/nix-installer-action@9b252454a8d70586c4ee7f163bf4bb1e9de3d763 # v2

      - name: Enable Binary Cache
-        uses: cachix/cachix-action@v15
+        uses: cachix/cachix-action@v12
        with:
          name: colmena
          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
@ -32,7 +32,7 @@ jobs:
        run: nix build .#manual -L

      - name: Deploy manual
-        uses: JamesIves/github-pages-deploy-action@v4.6.9
+        uses: JamesIves/github-pages-deploy-action@v4.3.4
        with:
          branch: gh-pages
          folder: result
@ -47,7 +47,7 @@ jobs:
        run: nix build .#manual.redirectFarm -L

      - name: Deploy redirect farm
-        uses: JamesIves/github-pages-deploy-action@v4.6.9
+        uses: JamesIves/github-pages-deploy-action@4.1.6
        with:
          branch: gh-pages
          folder: result-redirectFarm
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@ -13,15 +13,15 @@ jobs:
    name: ${{ matrix.os.label }}
    runs-on: ${{ matrix.os.image }}
    steps:
-      - uses: actions/checkout@v4.2.2
+      - uses: actions/checkout@v3.3.0
        with:
          fetch-depth: 0

      - name: Install Nix
-        uses: DeterminateSystems/nix-installer-action@b92f66560d6f97d6576405a7bae901ab57e72b6a # v15
+        uses: DeterminateSystems/nix-installer-action@9b252454a8d70586c4ee7f163bf4bb1e9de3d763 # v2

      - name: Enable Binary Cache
-        uses: cachix/cachix-action@v15
+        uses: cachix/cachix-action@v12
        with:
          name: colmena
          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
--- a/.gitignore
+++ b/.gitignore
@ -1,4 +1,3 @@
 result*
 /target
 /.direnv
-/.vscode
--- a/.srcignore
+++ b/.srcignore
@ -1,18 +0,0 @@
-# Exclusions from source distribution
-#
-# Files listed here will not be part of colmena.src
-
-/.github
-/CNAME
-/renovate.json
-
-/manual
-/integration-tests
-
-/nix
-/default.nix
-/flake-compat.nix
-/package.nix
-/shell.nix
-
-# vim: set ft=gitignore:
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@ -9,17 +9,18 @@ edition = "2021"
 [dependencies]
 async-stream = "0.3.5"
 async-trait = "0.1.68"
+atty = "0.2"
 clap = { version = "4.3", features = ["derive"] }
 clap_complete = "4.3"
 clicolors-control = "1"
 console = "0.15.5"
 const_format = "0.2.30"
-env_logger = "0.11.0"
+env_logger = "0.10.0"
 futures = "0.3.28"
 glob = "0.3.1"
-hostname = "0.4.0"
+hostname = "0.3.1"
 indicatif = "0.17.3"
-itertools = "0.13.0"
+itertools = "0.11.0"
 libc = "0.2.144"
 log = "0.4.17"
 quit = "2.0.0"
@ -27,12 +28,12 @@ regex = "1"
 serde = { version = "1.0.163", features = ["derive"] }
 serde_json = "1.0"
 shell-escape = "0.1.5"
-snafu = { version = "0.8.0", features = ["backtrace", "backtraces-impl-backtrace-crate"] }
+snafu = { version = "0.7.4", features = ["backtrace", "backtraces-impl-backtrace-crate"] }
 sys-info = "0.9.1"
 tempfile = "3.5.0"
 tokio-stream = "0.1.14"
 uuid = { version = "1.3.2", features = ["serde", "v4"] }
-validator = { version = "0.19.0", features = ["derive"] }
+validator = { version = "0.16.0", features = ["derive"] }

 [dev-dependencies]
 ntest = "0.9.0"
@ -43,9 +44,18 @@ version = "1.28.1"
 features = [
 	"fs",
 	"io-util",
+	"io-std",
 	"macros",
 	"process",
 	"rt",
 	"rt-multi-thread",
 	"sync",
 ]
+
+[[bin]]
+name = "colmena"
+path = "src/main.rs"
+
+[lib]
+name = "colmena"
+path = "src/lib.rs"
--- a/README.md
+++ b/README.md
@ -3,7 +3,7 @@
 [![Matrix Channel](https://img.shields.io/badge/Matrix-%23colmena%3Anixos.org-blueviolet)](https://matrix.to/#/#colmena:nixos.org)
 [![Stable Manual](https://img.shields.io/badge/Manual-Stable-informational)](https://colmena.cli.rs/stable)
 [![Unstable Manual](https://img.shields.io/badge/Manual-Unstable-orange)](https://colmena.cli.rs/unstable)
-[![Build](https://github.com/zhaofengli/colmena/actions/workflows/build.yml/badge.svg)](https://github.com/zhaofengli/colmena/actions/workflows/build.yml)
+[![Build](https://github.com/zhaofengli/colmena/workflows/Build/badge.svg)](https://github.com/zhaofengli/colmena/actions/workflows/build.yml)

 Colmena is a simple, stateless [NixOS](https://nixos.org) deployment tool modeled after [NixOps](https://github.com/NixOS/nixops) and [morph](https://github.com/DBCDK/morph), written in Rust.
 It's a thin wrapper over Nix commands like `nix-instantiate` and `nix-copy-closure`, and supports parallel deployment.
--- a/flake.lock
+++ b/flake.lock
@ -31,33 +31,13 @@
        "type": "github"
      }
    },
-    "nix-github-actions": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1729742964,
-        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "type": "github"
-      }
-    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1730785428,
-        "narHash": "sha256-Zwl8YgTVJTEum+L+0zVAWvXAGbWAuXHax3KzuejaDyo=",
+        "lastModified": 1696019113,
+        "narHash": "sha256-X3+DKYWJm93DRSdC5M6K5hLqzSya9BjibtBsuARoPco=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "4aa36568d413aca0ea84a1684d2d46f55dbabad7",
+        "rev": "f5892ddac112a1e9b3612c39af1b72987ee5783a",
        "type": "github"
      },
      "original": {
@ -71,23 +51,22 @@
      "inputs": {
        "flake-compat": "flake-compat",
        "flake-utils": "flake-utils",
-        "nix-github-actions": "nix-github-actions",
        "nixpkgs": "nixpkgs",
        "stable": "stable"
      }
    },
    "stable": {
      "locked": {
-        "lastModified": 1730883749,
-        "narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
+        "lastModified": 1696039360,
+        "narHash": "sha256-g7nIUV4uq1TOVeVIDEZLb005suTWCUjSY0zYOlSBsyE=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
+        "rev": "32dcb45f66c0487e92db8303a798ebc548cadedc",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixos-24.05",
+        "ref": "nixos-23.05",
        "repo": "nixpkgs",
        "type": "github"
      }
--- a/flake.nix
+++ b/flake.nix
@ -3,12 +3,7 @@

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
-    stable.url = "github:NixOS/nixpkgs/nixos-24.05";
-
-    nix-github-actions = {
-      url = "github:nix-community/nix-github-actions";
-      inputs.nixpkgs.follows = "nixpkgs";
-    };
+    stable.url = "github:NixOS/nixpkgs/nixos-23.05";

    flake-utils.url = "github:numtide/flake-utils";

@ -18,41 +13,12 @@
    };
  };

-  outputs = {
-    self,
-    nixpkgs,
-    stable,
-    flake-utils,
-    nix-github-actions,
-    ...
-  } @ inputs: let
+  outputs = { self, nixpkgs, stable, flake-utils, ... } @ inputs: let
    supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
    colmenaOptions = import ./src/nix/hive/options.nix;
    colmenaModules = import ./src/nix/hive/modules.nix;
-
-    # Temporary fork of nix-eval-jobs with changes to be upstreamed
-    # Mostly for the integration test setup and not needed in most use cases
-    _evalJobsOverlay = final: prev: let
-      patched = prev.nix-eval-jobs.overrideAttrs (old: {
-        version = old.version + "-colmena";
-        patches = (old.patches or []) ++ [
-          # Allows NIX_PATH to be honored
-          (final.fetchpatch {
-            url = "https://github.com/zhaofengli/nix-eval-jobs/commit/6ff5972724230ac2b96eb1ec355cd25ca512ef57.patch";
-            hash = "sha256-2NiMYpw27N+X7Ixh2HkP3fcWvopDJWQDVjgRdhOL2QQ";
-          })
-        ];
-      });
-    in {
-      nix-eval-jobs = patched;
-    };
  in flake-utils.lib.eachSystem supportedSystems (system: let
-    pkgs = import nixpkgs {
-      inherit system;
-      overlays = [
-        _evalJobsOverlay
-      ];
-    };
+    pkgs = nixpkgs.legacyPackages.${system};
  in rec {
    # We still maintain the expression in a Nixpkgs-acceptable form
    defaultPackage = self.packages.${system}.colmena;
@ -117,21 +83,11 @@
    in if pkgs.stdenv.isLinux then import ./integration-tests {
      pkgs = import nixpkgs {
        inherit system;
-        overlays = [
-          self.overlays.default
-          inputsOverlay
-
-          _evalJobsOverlay
-        ];
+        overlays = [ self.overlays.default inputsOverlay ];
      };
      pkgsStable = import stable {
        inherit system;
-        overlays = [
-          self.overlays.default
-          inputsOverlay
-
-          _evalJobsOverlay
-        ];
+        overlays = [ self.overlays.default inputsOverlay ];
      };
    } else {};
  }) // {
@ -148,12 +104,6 @@
      inherit rawHive colmenaOptions colmenaModules;
      hermetic = true;
    };
-
-    githubActions = nix-github-actions.lib.mkGithubMatrix {
-      checks = {
-        inherit (self.checks) x86_64-linux;
-      };
-    };
  };

  nixConfig = {
--- a/garnix.yaml
+++ b/garnix.yaml
@ -0,0 +1,3 @@
+builds:
+  include:
+    - 'checks.x86_64-linux.*'
--- a/integration-tests/default.nix
+++ b/integration-tests/default.nix
@ -8,18 +8,8 @@
  apply-local = import ./apply-local { inherit pkgs; };
  build-on-target = import ./build-on-target { inherit pkgs; };
  exec = import ./exec { inherit pkgs; };
-
-  # FIXME: The old evaluation method doesn't work purely with Nix 2.21+
-  flakes = import ./flakes {
-    inherit pkgs;
-    extraApplyFlags = "--experimental-flake-eval";
-  };
-  flakes-impure = import ./flakes {
-    inherit pkgs;
-    pure = false;
-  };
-  #flakes-streaming = import ./flakes { inherit pkgs; evaluator = "streaming"; };
-
+  flakes = import ./flakes { inherit pkgs; };
+  flakes-streaming = import ./flakes { inherit pkgs; evaluator = "streaming"; };
  parallel = import ./parallel { inherit pkgs; };

  allow-apply-all = import ./allow-apply-all { inherit pkgs; };
--- a/integration-tests/flakes/default.nix
+++ b/integration-tests/flakes/default.nix
@ -1,29 +1,13 @@
 { pkgs
 , evaluator ? "chunked"
-, extraApplyFlags ? ""
-, pure ? true
 }:

 let
-  inherit (pkgs) lib;
-
  tools = pkgs.callPackage ../tools.nix {
    targets = [ "alpha" ];
  };
-
-  applyFlags = "--evaluator ${evaluator} ${extraApplyFlags}"
-    + lib.optionalString (!pure) "--impure";
-
-  # From integration-tests/nixpkgs.nix
-  colmenaFlakeInputs = pkgs._inputs;
 in tools.runTest {
-  name = "colmena-flakes-${evaluator}"
-    + lib.optionalString (!pure) "-impure";
-
-  nodes.deployer = {
-    virtualisation.additionalPaths =
-      lib.mapAttrsToList (k: v: v.outPath) colmenaFlakeInputs;
-  };
+  name = "colmena-flakes-${evaluator}";

  colmena.test = {
    bundle = ./.;
@ -32,13 +16,12 @@ in tools.runTest {
      import re

      deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix")
-      deployer.succeed("sed -i 's @colmena@ path:${tools.colmena.src} g' /tmp/bundle/flake.nix")

      with subtest("Lock flake dependencies"):
          deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock")

      with subtest("Deploy with a plain flake without git"):
-          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}")
+          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
          alpha.succeed("grep FIRST /etc/deployment")

      with subtest("Deploy with a git flake"):
@ -46,22 +29,21 @@ in tools.runTest {

          # don't put probe.nix in source control - should fail
          deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix")
-          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
-          assert re.search(r"probe.nix.*(No such file or directory|does not exist)", logs), "Expected error message not found in log"
+          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
+          assert re.search(r"probe.nix.*No such file or directory", logs)

          # now it should succeed
          deployer.succeed("cd /tmp/bundle && git add probe.nix")
-          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}")
+          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
          alpha.succeed("grep SECOND /etc/deployment")

-    '' + lib.optionalString pure ''
      with subtest("Check that impure expressions are forbidden"):
          deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix")
-          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
-          assert re.search(r"access to absolute path.*forbidden in pure (eval|evaluation) mode", logs), "Expected error message not found in log"
+          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
+          assert re.search(r"access to absolute path.*forbidden in pure eval mode", logs)

      with subtest("Check that impure expressions can be allowed with --impure"):
-          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags} --impure")
+          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator} --impure")
          alpha.succeed("grep deployer /etc/deployment")
    '';
  };
--- a/integration-tests/flakes/flake.nix
+++ b/integration-tests/flakes/flake.nix
@ -3,15 +3,13 @@

  inputs = {
    nixpkgs.url = "@nixpkgs@";
-    colmena.url = "@colmena@";
  };

-  outputs = { self, nixpkgs, colmena }: let
+  outputs = { self, nixpkgs }: let
    pkgs = import nixpkgs {
      system = "x86_64-linux";
    };
  in {
    colmena = import ./hive.nix { inherit pkgs; };
-    colmenaHive = colmena.lib.makeHive self.outputs.colmena;
  };
 }
--- a/integration-tests/tools.nix
+++ b/integration-tests/tools.nix
@ -140,7 +140,7 @@ let
    nix.settings.substituters = lib.mkForce [];

    virtualisation = {
-      memorySize = 6144;
+      memorySize = 4096;
      writableStore = true;
      additionalPaths = [
        "${pkgs.path}"
@ -165,9 +165,6 @@ let
        exec "$@" 2> >(tee /dev/stderr)
      '')
    ];
-
-    # Re-enable switch-to-configuration
-    system.switch.enable = true;
  };

  # Setup for target nodes
@ -183,9 +180,6 @@ let
      sshKeys.snakeOilPublicKey
    ];
    virtualisation.writableStore = true;
-
-    # Re-enable switch-to-configuration
-    system.switch.enable = true;
  };

  nodes = let
--- a/manual/src/tutorial/flakes.md
+++ b/manual/src/tutorial/flakes.md
@ -90,34 +90,6 @@ To build and deploy to all nodes:
 colmena apply
 ```

-## Direct Flake Evaluation (Experimental)
-
-By default, Colmena uses `nix-instantiate` to evaluate your flake which does not work purely on Nix 2.21+, necessitating the use of `--impure`.
-There is experimental support for evaluating flakes directly with `nix eval`, enabled via `--experimental-flake-eval`.
-
-To use this new evaluation mode, your flake needs to depend on Colmena itself as an input and expose a new output called `colmenaHive`:
-
-```diff
- {
-   inputs = {
-+    # ADDED: Colmena input
-+    colmena.url = "github:zhaofengli/colmena";
-
-     # ... Rest of configuration ...
-   };
-   outputs = { self, colmena, ... }: {
-+    # ADDED: New colmenaHive output
-+    colmenaHive = colmena.lib.makeHive self.outputs.colmena;
-
-     # Your existing colmena output
-     colmena = {
-       # ... Rest of configuration ...
-     };
-   };
- }
-```
-
-
 ## Next Steps

 - Head to the [Features](../features/index.md) section to see what else Colmena can do.
--- a/package.nix
+++ b/package.nix
@ -1,16 +1,13 @@
-{ lib
-, stdenv
-, rustPlatform
-, nix-gitignore
-, installShellFiles
-, nix-eval-jobs
-}:
+{ lib, stdenv, rustPlatform, installShellFiles, nix-eval-jobs }:

 rustPlatform.buildRustPackage rec {
  pname = "colmena";
  version = "0.5.0-pre";

-  src = nix-gitignore.gitignoreSource [ ./.srcignore ] ./.;
+  src = lib.cleanSourceWith {
+    filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) [ "target" "manual" "integration-tests" ]);
+    src = lib.cleanSource ./.;
+  };

  cargoLock = {
    lockFile = ./Cargo.lock;
--- a/renovate.json
+++ b/renovate.json
@ -1,14 +0,0 @@
-{
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:recommended",
-    "group:allNonMajor"
-  ],
-  "lockFileMaintenance": {
-    "enabled": true,
-    "extends": ["schedule:weekly"]
-  },
-  "nix": {
-    "enabled": true
-  }
-}
--- a/src/cli.rs
+++ b/src/cli.rs
@ -10,7 +10,7 @@ use env_logger::fmt::WriteStyle;
 use crate::{
    command::{self, apply::DeployOpts},
    error::ColmenaResult,
-    nix::{hive::EvaluationMethod, Hive, HivePath},
+    nix::{Hive, HivePath},
 };

 /// Base URL of the manual, without the trailing slash.
@ -137,21 +137,6 @@ This only works when building locally.
        value_names = ["NAME", "VALUE"],
    )]
    nix_option: Vec<String>,
-    #[arg(
-        long,
-        default_value_t,
-        help = "Use direct flake evaluation (experimental)",
-        long_help = r#"If enabled, flakes will be evaluated using `nix eval`. This requires the flake to depend on Colmena as an input and expose a compatible `colmenaHive` output:
-
-  outputs = { self, colmena, ... }: {
-    colmenaHive = colmena.lib.makeHive self.outputs.colmena;
-    colmena = ...;
-  };
-
-This is an experimental feature."#,
-        global = true
-    )]
-    experimental_flake_eval: bool,
    #[arg(
        long,
        value_name = "WHEN",
@ -277,11 +262,6 @@ async fn get_hive(opts: &Opts) -> ColmenaResult<Hive> {
        hive.set_impure(true);
    }

-    if opts.experimental_flake_eval {
-        log::warn!("Using direct flake evaluation (experimental)");
-        hive.set_evaluation_method(EvaluationMethod::DirectFlakeEval);
-    }
-
    for chunks in opts.nix_option.chunks_exact(2) {
        let [name, value] = chunks else {
            unreachable!()
--- a/src/error.rs
+++ b/src/error.rs
@ -55,6 +55,9 @@ pub enum ColmenaError {
    #[snafu(display("Unexpected active profile: {:?}", profile))]
    ActiveProfileUnexpected { profile: Profile },

+    #[snafu(display("Invalid JSON from activation program: {}", error))]
+    InvalidActivationProgramJson { error: serde_json::Error },
+
    #[snafu(display("Could not determine current profile"))]
    FailedToGetCurrentProfile,

--- a/src/job.rs
+++ b/src/job.rs
@ -874,17 +874,11 @@ fn describe_node_list(nodes: &[NodeName]) -> Option<String> {
        }

        let (idx, next) = next.unwrap();
-        let remaining_text = rough_limit - s.len();
-        let remaining_nodes = total - idx;
+        let remaining = rough_limit - s.len();

-        if next.len() + other_text.len() >= remaining_text {
-            if remaining_nodes == 1 {
-                write!(s, ", and {}", next.as_str()).unwrap();
-                break;
-            } else {
-                write!(s, ", and {} other nodes", remaining_nodes).unwrap();
-                break;
-            }
+        if next.len() + other_text.len() >= remaining {
+            write!(s, ", and {} other nodes", total - idx).unwrap();
+            break;
        }
    }

--- a/src/lib.rs
+++ b/src/lib.rs
@ -0,0 +1,16 @@
+#![allow(dead_code)]
+
+mod cli;
+mod command;
+mod error;
+mod job;
+mod nix;
+mod progress;
+mod troubleshooter;
+mod util;
+
+pub use nix::host::generic::Request as GenericRequest;
+pub use nix::host::generic::Response as GenericResponse;
+pub use nix::host::CopyOptions;
+pub use nix::key::{Key, UploadAt};
+pub use nix::store::StorePath;
--- a/src/nix/hive/assets.rs
+++ b/src/nix/hive/assets.rs
@ -51,10 +51,7 @@ impl Assets {
            // We explicitly specify `path:` instead of letting Nix resolve
            // automatically, which would involve checking parent directories
            // for a git repository.
-            let uri = format!(
-                "path:{}",
-                temp_dir.path().canonicalize().unwrap().to_str().unwrap()
-            );
+            let uri = format!("path:{}", temp_dir.path().to_str().unwrap());
            let _ = lock_flake_quiet(&uri).await;
            let assets_flake = Flake::from_uri(uri).await?;
            assets_flake_uri = Some(assets_flake.locked_uri().to_owned());
--- a/src/nix/hive/eval.nix
+++ b/src/nix/hive/eval.nix
@ -197,12 +197,12 @@ let
  ];

  serializableSystemTypeConfigKeys = [
-    "supportsDeployment"
+    "supportsDeployment" "activationProgram" "protocol"
  ];

 in rec {
  # Exported attributes
-  __schema = "v0.20241006";
+  __schema = "v0";

  nodes = listToAttrs (map (name: { inherit name; value = evalNode name (configsFor name); }) nodeNames);
  toplevel =         lib.mapAttrs (_: v: v.config.system.build.toplevel) nodes;
--- a/src/nix/hive/flake.nix
+++ b/src/nix/hive/flake.nix
@ -7,7 +7,7 @@

  outputs = { self, hive }: {
    processFlake = let
-      compatibleSchema = "v0.20241006";
+      compatibleSchema = "v0";

      # Evaluates a raw hive.
      #
--- a/src/nix/hive/mod.rs
+++ b/src/nix/hive/mod.rs
@ -8,7 +8,6 @@ use std::convert::AsRef;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;

-use const_format::formatcp;
 use tokio::process::Command;
 use tokio::sync::OnceCell;
 use validator::Validate;
@ -23,21 +22,6 @@ use crate::job::JobHandle;
 use crate::util::{CommandExecution, CommandExt};
 use assets::Assets;

-/// The version of the Hive schema we are compatible with.
-///
-/// Currently we are tied to one specific version.
-const HIVE_SCHEMA: &str = "v0.20241006";
-
-/// The snippet to be used for `nix eval --apply`.
-const FLAKE_APPLY_SNIPPET: &str = formatcp!(
-    r#"with builtins; hive: assert (hive.__schema == "{}" || throw ''
-    The colmenaHive output (schema ${{hive.__schema}}) isn't compatible with this version of Colmena.
-
-    Hint: Use the same version of Colmena as in the Flake input.
-''); "#,
-    HIVE_SCHEMA
-);
-
 #[derive(Debug, Clone)]
 pub enum HivePath {
    /// A Nix Flake.
@ -79,33 +63,11 @@ impl FromStr for HivePath {
    }
 }

-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum EvaluationMethod {
-    /// Use nix-instantiate and specify the entire Nix expression.
-    ///
-    /// This is the default method.
-    ///
-    /// For flakes, we use `builtins.getFlakes`. Pure evaluation no longer works
-    /// with this method in Nix 2.21+.
-    NixInstantiate,
-
-    /// Use `nix eval --apply` on top of a flake.
-    ///
-    /// This can be activated with --experimental-flake-eval.
-    ///
-    /// In this method, we can no longer pull in our bundled assets and
-    /// the flake must expose a compatible `colmenaHive` output.
-    DirectFlakeEval,
-}
-
 #[derive(Debug)]
 pub struct Hive {
    /// Path to the hive.
    path: HivePath,

-    /// Method to evaluate the hive with.
-    evaluation_method: EvaluationMethod,
-
    /// Path to the context directory.
    ///
    /// Normally this is directory containing the "hive.nix"
@ -174,7 +136,6 @@ impl Hive {

        Ok(Self {
            path,
-            evaluation_method: EvaluationMethod::NixInstantiate,
            context_dir,
            assets,
            show_trace: false,
@ -200,14 +161,6 @@ impl Hive {
            .await
    }

-    pub fn set_evaluation_method(&mut self, method: EvaluationMethod) {
-        if !self.is_flake() && method == EvaluationMethod::DirectFlakeEval {
-            return;
-        }
-
-        self.evaluation_method = method;
-    }
-
    pub async fn get_registry_config(&self) -> ColmenaResult<&RegistryConfig> {
        self.registry_config
            .get_or_try_init(|| async {
@ -321,24 +274,38 @@ impl Hive {
        for node in selected_nodes.into_iter() {
            let config = node_configs.remove(&node).unwrap();

-            let host = config.to_ssh_host().map(|mut host| {
-                n_ssh += 1;
-
-                if let Some(ssh_config) = &ssh_config {
-                    host.set_ssh_config(ssh_config.clone());
-                }
-
-                if self.is_flake() {
-                    host.set_use_nix3_copy(true);
-                }
-
-                host.upcast()
-            });
-            let ssh_host = host.is_some();
-            let target = TargetNode::new(node.clone(), host, config);
-
-            if !ssh_only || ssh_host {
+            if let Some(system_type) = config.system_type.as_ref() {
+                log::debug!(
+                    "Using generic host (system_type: {}) for node {}",
+                    system_type,
+                    node.0
+                );
+                let system_config = registry.systems.get(system_type).unwrap();
+                let mut generic_host = config.to_generic_host(system_config)?;
+                generic_host.connect().await?;
+                let target = TargetNode::new(node.clone(), Some(Box::new(generic_host)), config);
                targets.insert(node, target);
+            } else {
+                log::debug!("Using SSH host for node {}", node.0);
+                let host = config.to_ssh_host().map(|mut host| {
+                    n_ssh += 1;
+
+                    if let Some(ssh_config) = &ssh_config {
+                        host.set_ssh_config(ssh_config.clone());
+                    }
+
+                    if self.is_flake() {
+                        host.set_use_nix3_copy(true);
+                    }
+
+                    host.upcast()
+                });
+                let ssh_host = host.is_some();
+                let target = TargetNode::new(node.clone(), host, config);
+
+                if !ssh_only || ssh_host {
+                    targets.insert(node, target);
+                }
            }
        }

@ -503,10 +470,7 @@ impl Hive {

    /// Returns the base expression from which the evaluated Hive can be used.
    fn get_base_expression(&self) -> String {
-        match self.evaluation_method {
-            EvaluationMethod::NixInstantiate => self.assets.get_base_expression(),
-            EvaluationMethod::DirectFlakeEval => FLAKE_APPLY_SNIPPET.to_string(),
-        }
+        self.assets.get_base_expression()
    }

    /// Returns whether this Hive is a flake.
@ -529,11 +493,6 @@ impl<'hive> NixInstantiate<'hive> {
    }

    fn instantiate(&self) -> Command {
-        // TODO: Better error handling
-        if self.hive.evaluation_method == EvaluationMethod::DirectFlakeEval {
-            panic!("Instantiation is not supported with DirectFlakeEval");
-        }
-
        let mut command = Command::new("nix-instantiate");

        if self.hive.is_flake() {
@ -552,48 +511,17 @@ impl<'hive> NixInstantiate<'hive> {
    }

    fn eval(self) -> Command {
+        let mut command = self.instantiate();
        let flags = self.hive.nix_flags();
-
-        match self.hive.evaluation_method {
-            EvaluationMethod::NixInstantiate => {
-                let mut command = self.instantiate();
-
-                command
-                    .arg("--eval")
-                    .arg("--json")
-                    .arg("--strict")
-                    // Ensures the derivations are instantiated
-                    // Required for system profile evaluation and IFD
-                    .arg("--read-write-mode")
-                    .args(flags.to_args());
-
-                command
-            }
-            EvaluationMethod::DirectFlakeEval => {
-                let mut command = Command::new("nix");
-                let flake = if let HivePath::Flake(flake) = self.hive.path() {
-                    flake
-                } else {
-                    panic!("The DirectFlakeEval evaluation method only support flakes");
-                };
-
-                let hive_installable = format!("{}#colmenaHive", flake.uri());
-
-                let mut full_expression = self.hive.get_base_expression();
-                full_expression += &self.expression;
-
-                command
-                    .arg("eval") // nix eval
-                    .args(["--extra-experimental-features", "flakes nix-command"])
-                    .arg(hive_installable)
-                    .arg("--json")
-                    .arg("--apply")
-                    .arg(&full_expression)
-                    .args(flags.to_args());
-
-                command
-            }
-        }
+        command
+            .arg("--eval")
+            .arg("--json")
+            .arg("--strict")
+            // Ensures the derivations are instantiated
+            // Required for system profile evaluation and IFD
+            .arg("--read-write-mode")
+            .args(flags.to_args());
+        command
    }

    async fn instantiate_with_builders(self) -> ColmenaResult<Command> {
--- a/src/nix/hive/options.nix
+++ b/src/nix/hive/options.nix
@ -1,17 +1,18 @@
 with builtins; rec {
  keyType = { lib, name, config, ... }: let
    inherit (lib) types;
+    mdDoc = lib.mdDoc or (md: md);
  in {
    options = {
      name = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          File name of the key.
        '';
        default = name;
        type = types.str;
      };
      text = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          Content of the key.
          One of `text`, `keyCommand` and `keyFile` must be set.
        '';
@ -19,7 +20,7 @@ with builtins; rec {
        type = types.nullOr types.str;
      };
      keyFile = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          Path of the local file to read the key from.
          One of `text`, `keyCommand` and `keyFile` must be set.
        '';
@ -28,7 +29,7 @@ with builtins; rec {
        type = types.nullOr types.path;
      };
      keyCommand = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          Command to run to generate the key.
          One of `text`, `keyCommand` and `keyFile` must be set.
        '';
@ -38,14 +39,14 @@ with builtins; rec {
        in types.nullOr nonEmptyList;
      };
      destDir = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          Destination directory on the host.
        '';
        default = "/run/keys";
        type = types.path;
      };
      path = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          Full path to the destination.
        '';
        default = "${config.destDir}/${config.name}";
@ -53,28 +54,28 @@ with builtins; rec {
        internal = true;
      };
      user = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          The group that will own the file.
        '';
        default = "root";
        type = types.str;
      };
      group = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          The group that will own the file.
        '';
        default = "root";
        type = types.str;
      };
      permissions = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          Permissions to set for the file.
        '';
        default = "0600";
        type = types.str;
      };
      uploadAt = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          When to upload the keys.

          - pre-activation (default): Upload the keys before activating the new system profile.
@ -93,6 +94,7 @@ with builtins; rec {
  # Largely compatible with NixOps/Morph.
  deploymentOptions = { name, lib, ... }: let
    inherit (lib) types;
+    mdDoc = lib.mdDoc or (md: md);
  in {
    options = {
      deployment = {
@ -105,7 +107,7 @@ with builtins; rec {
          type = types.str;
        };
        targetHost = lib.mkOption {
-          description = ''
+          description = mdDoc ''
            The target SSH node for deployment.

            By default, the node's attribute name will be used.
@ -115,7 +117,7 @@ with builtins; rec {
          default = name;
        };
        targetPort = lib.mkOption {
-          description = ''
+          description = mdDoc ''
            The target SSH port for deployment.

            By default, the port is the standard port (22) or taken
@ -125,15 +127,22 @@ with builtins; rec {
          default = null;
        };
        targetUser = lib.mkOption {
-          description = ''
+          description = mdDoc ''
            The user to use to log into the remote node. If set to null, the
            target user will not be specified in SSH invocations.
          '';
          type = types.nullOr types.str;
          default = "root";
        };
+        connectionUri = lib.mkOption {
+          description = mdDoc ''
+            Connection options given to the activation program.
+          '';
+          type = types.str;
+          default = "ssh://localhost";
+        };
        allowLocalDeployment = lib.mkOption {
-          description = ''
+          description = mdDoc ''
            Allow the configuration to be applied locally on the host running
            Colmena.

@ -150,7 +159,7 @@ with builtins; rec {
          default = false;
        };
        buildOnTarget = lib.mkOption {
-          description = ''
+          description = mdDoc ''
            Whether to build the system profiles on the target node itself.

            When enabled, Colmena will copy the derivation to the target
@ -170,7 +179,7 @@ with builtins; rec {
          default = false;
        };
        tags = lib.mkOption {
-          description = ''
+          description = mdDoc ''
            A list of tags for the node.

            Can be used to select a group of nodes for deployment.
@ -179,7 +188,7 @@ with builtins; rec {
          default = [];
        };
        keys = lib.mkOption {
-          description = ''
+          description = mdDoc ''
            A set of secrets to be deployed to the node.

            Secrets are transferred to the node out-of-band and
@ -189,7 +198,7 @@ with builtins; rec {
          default = {};
        };
        replaceUnknownProfiles = lib.mkOption {
-          description = ''
+          description = mdDoc ''
            Allow a configuration to be applied to a host running a profile we
            have no knowledge of. By setting this option to false, you reduce
            the likelyhood of rolling back changes made via another Colmena user.
@ -205,7 +214,7 @@ with builtins; rec {
          default = true;
        };
        privilegeEscalationCommand = lib.mkOption {
-          description = ''
+          description = mdDoc ''
            Command to use to elevate privileges when activating the new profiles on SSH hosts.

            This is used on SSH hosts when `deployment.targetUser` is not `root`.
@ -215,7 +224,7 @@ with builtins; rec {
          default = [ "sudo" "-H" "--" ];
        };
        sshOptions = lib.mkOption {
-          description = ''
+          description = mdDoc ''
            Extra SSH options to pass to the SSH command.
          '';
          type = types.listOf types.str;
@ -256,6 +265,12 @@ with builtins; rec {
        type = types.functionTo types.unspecified;
        default = _: {};
      };
+      activationProgram = lib.mkOption {
+        description = mdDoc ''
+          Program to execute at activation time.
+        '';
+        type = types.path;
+      };
    };
  };
  registryOptions = { lib, ... }: let
@ -273,28 +288,29 @@ with builtins; rec {
  # Hive-wide options
  metaOptions = { lib, ... }: let
    inherit (lib) types;
+    mdDoc = lib.mdDoc or (md: md);
  in {
    options = {
      name = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          The name of the configuration.
        '';
        type = types.str;
        default = "hive";
      };
      description = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          A short description for the configuration.
        '';
        type = types.str;
        default = "A Colmena Hive";
      };
      nixpkgs = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          The pinned Nixpkgs package set. Accepts one of the following:

          - A path to a Nixpkgs checkout
-          - The Nixpkgs lambda (e.g., import <nixpkgs>)
+          - The Nixpkgs lambda (e.g., import \<nixpkgs\>)
          - An initialized Nixpkgs attribute set

          This option must be specified when using Flakes.
@ -303,21 +319,21 @@ with builtins; rec {
        default = null;
      };
      nodeNixpkgs = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          Node-specific Nixpkgs pins.
        '';
        type = types.attrsOf types.unspecified;
        default = {};
      };
      nodeSpecialArgs = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          Node-specific special args.
        '';
        type = types.attrsOf types.unspecified;
        default = {};
      };
      machinesFile = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          Use the machines listed in this file when building this hive configuration.

          If your Colmena host has nix configured to allow for remote builds
@ -341,7 +357,7 @@ with builtins; rec {
        type = types.nullOr types.path;
      };
      specialArgs = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          A set of special arguments to be passed to NixOS modules.

          This will be merged into the `specialArgs` used to evaluate
@ -351,7 +367,7 @@ with builtins; rec {
        type = types.attrsOf types.unspecified;
      };
      allowApplyAll = lib.mkOption {
-        description = ''
+        description = mdDoc ''
          Whether to allow deployments without a node filter set.

          If set to false, a node filter must be specified with `--on` when
--- a/src/nix/host/generic.rs
+++ b/src/nix/host/generic.rs
@ -0,0 +1,287 @@
+use std::collections::HashMap;
+use std::process::Stdio;
+
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader, BufWriter};
+use tokio::process::{Child, ChildStdin, ChildStdout, Command};
+
+use super::{CopyDirection, CopyOptions, Host, RebootOptions};
+use crate::error::{ColmenaError, ColmenaResult};
+use crate::job::JobHandle;
+use crate::nix::{self, Key, Profile, StorePath, SystemTypeConfig};
+
+pub type TransportId = String;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Transport {
+    id: TransportId,
+    long_name: String,
+    description: String,
+}
+
+pub type GoalId = String;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Goal {
+    id: GoalId,
+    long_name: String,
+    description: String,
+}
+
+/// A request to the activation program
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum Request {
+    Connection {
+        connection_uri: String,
+    },
+
+    /// Copy closure to/from host
+    CopyClosure {
+        path: StorePath,
+        to_host: bool,
+        options: CopyOptions,
+    },
+
+    /// Deploys the profile to host
+    Deploy {
+        goal: GoalId,
+        toplevel: StorePath,
+        options: CopyOptions,
+    },
+
+    /// Realizes the derivation
+    Realize {
+        path: StorePath,
+        remote: bool,
+    },
+
+    /// Uploads keys to host
+    UploadKeys {
+        keys: HashMap<String, Key>,
+        require_ownership: bool,
+    },
+
+    Activate {
+        profile: StorePath,
+        goal: GoalId,
+    },
+
+    Reboot {
+        wait_for_boot: bool,
+    },
+}
+
+/// A response from the activation program
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Response {
+    ConnectionFailed { error: String },
+    ConnectionSucceded { supported_goals: Vec<String> },
+    Progress { phase: String },
+    NewStorePath { store_path: StorePath },
+    Failed { error: String },
+    Unsupported,
+}
+
+#[derive(Debug)]
+pub struct ActivationCommand {
+    command: Child,
+    stdin: BufWriter<ChildStdin>,
+    stdout: BufReader<ChildStdout>,
+}
+
+#[derive(Debug)]
+pub struct GenericHost {
+    activation_program: ActivationCommand,
+    connection_uri: String,
+}
+
+#[async_trait]
+impl Host for GenericHost {
+    async fn copy_closure(
+        &mut self,
+        closure: &StorePath,
+        direction: CopyDirection,
+        options: CopyOptions,
+    ) -> ColmenaResult<()> {
+        self.call_default_handler(Request::CopyClosure {
+            path: closure.clone(),
+            to_host: direction == CopyDirection::ToRemote,
+            options,
+        })
+        .await
+    }
+
+    async fn realize_remote(&mut self, derivation: &StorePath) -> ColmenaResult<Vec<StorePath>> {
+        Ok(self
+            .call(
+                Request::Realize {
+                    path: derivation.clone(),
+                    remote: true
+                },
+                move |response, mut store_paths| {
+                    match response {
+                        Response::Progress { phase } => println!("{}", phase),
+                        Response::NewStorePath { store_path } => store_paths.push(store_path),
+                        _ => (),
+                    };
+                    store_paths
+                },
+                Vec::new(),
+            )
+            .await?)
+    }
+
+    async fn realize(&mut self, derivation: &StorePath) -> ColmenaResult<Vec<StorePath>> {
+        Ok(self
+            .call(
+                Request::Realize {
+                    path: derivation.clone(),
+                    remote: false
+                },
+                move |response, mut store_paths| {
+                    match response {
+                        Response::Progress { phase } => println!("{}", phase),
+                        Response::NewStorePath { store_path } => store_paths.push(store_path),
+                        _ => (),
+                    };
+                    store_paths
+                },
+                Vec::new(),
+            )
+            .await?)
+    }
+
+    fn set_job(&mut self, _: Option<JobHandle>) {}
+
+    async fn deploy(
+        &mut self,
+        profile: &Profile,
+        goal: nix::Goal,
+        copy_options: CopyOptions,
+    ) -> ColmenaResult<()> {
+        self.call_default_handler(Request::Deploy {
+            goal: goal.to_string(),
+            toplevel: profile.as_store_path().clone(),
+            options: copy_options,
+        })
+        .await
+    }
+
+    async fn upload_keys(
+        &mut self,
+        keys: &HashMap<String, Key>,
+        require_ownership: bool,
+    ) -> ColmenaResult<()> {
+        self.call_default_handler(Request::UploadKeys {
+            keys: keys.clone(),
+            require_ownership,
+        })
+        .await
+    }
+
+    async fn get_current_system_profile(&mut self) -> ColmenaResult<Profile> {
+        Err(ColmenaError::Unsupported)
+    }
+
+    async fn get_main_system_profile(&mut self) -> ColmenaResult<Profile> {
+        Err(ColmenaError::Unsupported)
+    }
+
+    async fn activate(&mut self, profile: &Profile, goal: nix::Goal) -> ColmenaResult<()> {
+        self.call_default_handler(Request::Activate {
+            profile: profile.as_store_path().clone(),
+            goal: goal.to_string(),
+        })
+        .await
+    }
+
+    async fn reboot(&mut self, options: RebootOptions) -> ColmenaResult<()> {
+        self.call_default_handler(Request::Reboot {
+            wait_for_boot: options.wait_for_boot,
+        })
+        .await
+    }
+}
+
+impl GenericHost {
+    pub fn new(system: &SystemTypeConfig, connection_uri: String) -> ColmenaResult<GenericHost> {
+        let mut command = Command::new(system.activation_program.as_path())
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::inherit())
+            .spawn()?;
+
+        let stdin = BufWriter::new(command.stdin.take().unwrap());
+        let stdout = BufReader::new(command.stdout.take().unwrap());
+
+        let activation_program = ActivationCommand {
+            command,
+            stdin,
+            stdout,
+        };
+
+        Ok(Self {
+            activation_program,
+            connection_uri,
+        })
+    }
+
+    pub async fn connect(&mut self) -> ColmenaResult<()> {
+        self.call_default_handler(Request::Connection {
+            connection_uri: self.connection_uri.clone(),
+        })
+        .await
+    }
+
+    async fn call<F, T>(
+        &mut self,
+        request: Request,
+        handler: F,
+        initial_value: T,
+    ) -> ColmenaResult<T>
+    where
+        F: Fn(Response, T) -> T,
+    {
+        let json = serde_json::to_string(&request)
+            .map_err(|error| ColmenaError::InvalidActivationProgramJson { error })?;
+
+        log::trace!("giving to activation program stdin {}", json.as_str());
+
+        let stdin = &mut self.activation_program.stdin;
+        let stdout = &mut self.activation_program.stdout;
+
+        stdin.write_all(json.as_bytes()).await?;
+        stdin.write_all(b"\n").await?;
+        stdin.flush().await?;
+
+        let mut line = String::new();
+        let mut value = initial_value;
+
+        // We're reading JSONL, so we can read all the line and parse it
+        while stdout.read_line(&mut line).await.is_ok_and(|x| x != 0) {
+            if line == "\n" {
+                log::trace!("finished receiving responses from activation program");
+                break;
+            }
+            log::trace!("receiving from activation program:\n{}", line);
+            let response: Response = serde_json::from_str(line.as_str())
+                .map_err(|error| ColmenaError::InvalidActivationProgramJson { error })?;
+
+            match response {
+                Response::Progress { phase } => {
+                    log::info!("{phase}");
+                    break;
+                }
+                Response::Unsupported => return Err(ColmenaError::Unsupported),
+                _ => value = handler(response, value),
+            }
+        }
+
+        Ok(value)
+    }
+
+    async fn call_default_handler(&mut self, request: Request) -> ColmenaResult<()> {
+        self.call(request, |_, _| {}, ()).await
+    }
+}
--- a/src/nix/host/mod.rs
+++ b/src/nix/host/mod.rs
@ -1,6 +1,7 @@
 use std::collections::HashMap;

 use async_trait::async_trait;
+use serde::{Deserialize, Serialize};

 use super::{Goal, Key, Profile, StorePath};
 use crate::error::{ColmenaError, ColmenaResult};
@ -14,13 +15,16 @@ pub use local::Local;

 mod key_uploader;

-#[derive(Copy, Clone, Debug)]
+pub mod generic;
+pub use generic::GenericHost;
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub enum CopyDirection {
    ToRemote,
    FromRemote,
 }

-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct CopyOptions {
    include_outputs: bool,
    use_substitutes: bool,
@ -61,6 +65,18 @@ impl CopyOptions {
        self.gzip = val;
        self
    }
+
+    pub fn get_include_outputs(&self) -> bool {
+        self.include_outputs
+    }
+
+    pub fn get_use_substitutes(&self) -> bool {
+        self.use_substitutes
+    }
+
+    pub fn get_gzip(&self) -> bool {
+        self.gzip
+    }
 }

 impl Default for RebootOptions {
--- a/src/nix/key.rs
+++ b/src/nix/key.rs
@ -87,14 +87,14 @@ pub struct Key {
    #[serde(flatten)]
    source: KeySource,

-    #[validate(custom(function = "validate_dest_dir"))]
+    #[validate(custom = "validate_dest_dir")]
    #[serde(rename = "destDir")]
    dest_dir: PathBuf,

-    #[validate(custom(function = "validate_unix_name"))]
+    #[validate(custom = "validate_unix_name")]
    user: String,

-    #[validate(custom(function = "validate_unix_name"))]
+    #[validate(custom = "validate_unix_name")]
    group: String,

    permissions: String,
--- a/src/nix/mod.rs
+++ b/src/nix/mod.rs
@ -10,8 +10,8 @@ use validator::{Validate, ValidationError as ValidationErrorType};
 use crate::error::{ColmenaError, ColmenaResult};

 pub mod host;
-use host::Ssh;
 pub use host::{CopyDirection, CopyOptions, Host, RebootOptions};
+use host::{GenericHost, Ssh};

 pub mod hive;
 pub use hive::{Hive, HivePath};
@ -67,6 +67,9 @@ pub struct NodeConfig {
    #[serde(rename = "targetPort")]
    target_port: Option<u16>,

+    #[serde(rename = "connectionUri")]
+    connection_uri: String,
+
    #[serde(rename = "allowLocalDeployment")]
    allow_local_deployment: bool,

@ -84,7 +87,7 @@ pub struct NodeConfig {
    #[serde(rename = "sshOptions")]
    extra_ssh_options: Vec<String>,

-    #[validate(custom(function = "validate_keys"))]
+    #[validate(custom = "validate_keys")]
    keys: HashMap<String, Key>,
 }

@ -101,6 +104,9 @@ pub struct MetaConfig {
 pub struct SystemTypeConfig {
    #[serde(rename = "supportsDeployment")]
    pub supports_deployment: bool,
+
+    #[serde(rename = "activationProgram")]
+    pub activation_program: StorePath,
 }

 #[derive(Debug, Clone, Validate, Deserialize)]
@ -208,6 +214,10 @@ impl NodeConfig {
            host
        })
    }
+
+    pub fn to_generic_host(&self, system_config: &SystemTypeConfig) -> ColmenaResult<GenericHost> {
+        GenericHost::new(system_config, self.connection_uri.clone())
+    }
 }

 impl NixFlags {
--- a/src/progress/mod.rs
+++ b/src/progress/mod.rs
@ -7,8 +7,6 @@
 pub mod plain;
 pub mod spinner;

-use std::io::IsTerminal;
-
 use async_trait::async_trait;
 use tokio::sync::mpsc::{self, UnboundedReceiver as TokioReceiver, UnboundedSender as TokioSender};

@ -92,7 +90,7 @@ pub enum LineStyle {

 impl SimpleProgressOutput {
    pub fn new(verbose: bool) -> Self {
-        let tty = std::io::stdout().is_terminal();
+        let tty = atty::is(atty::Stream::Stdout);

        if verbose || !tty {
            Self::Plain(PlainOutput::new())
Author	SHA1	Message	Date
soyouzpanda	9b530b6ff6	Added support for remote and local realization	2024-12-08 19:06:41 +01:00
soyouzpanda	eca9e7a425	Added getters to CopyOptions	2024-12-07 15:32:48 +01:00
soyouzpanda	94b1a11326	feat: create colmena library	2024-10-10 21:04:01 +02:00
soyouzpanda	e956ae403b	feat: connectionOptions -> connectionUri & removed useless options	2024-10-06 01:32:25 +02:00
soyouzpanda	c53d80aa08	feat: Oui	2024-10-03 18:42:52 +02:00
soyouzpanda	71200e5c17	feat(generic): Implemented every function of Host to GenericHost	2024-09-26 12:30:14 +02:00
soyouzpanda	ab8d8b0321	feat(generic): Added first generic host	2024-08-23 17:57:14 +02:00
soyouzpanda	596a40020b	feat(activation): Added activation program TODO: customize activation command to launch activation program	2024-08-23 14:22:42 +02:00
Ryan Lahfa	c959d643db	Merge pull request 'feat: custom evaluation' (#1 ) from custom-activation into main Reviewed-on: #1	2024-05-24 20:56:11 +02:00
Ryan Lahfa	470a0c360a	feat: generic registry and custom evaluation This PR bring custom evaluation, but does not offer yet custom activation. Therefore, you can evaluate your systems and refer to each of them, but you cannot ask Colmena to build them for you. Signed-off-by: Ryan Lahfa <ryan@dgnum.eu>	2024-05-24 20:55:31 +02:00
Ryan Lahfa	76865b3293	feat: disable key management modules Let the user opt-in… ! Signed-off-by: Ryan Lahfa <ryan@dgnum.eu>	2024-05-24 20:55:31 +02:00