Merge pull request #228 from zhaofengli/direct-flake-eval

Add direct flake evaluation support

.editorconfig

@@ -10,11 +10,6 @@ insert_final_newline = true
 trim_trailing_whitespace = true
 charset = utf-8
 
-# Rust
-[*.rs]
-indent_style = space
-indent_size = 2
-
 # Vendored
 [manual/theme/highlight.js]
 end_of_line = unset

.github/workflows/build.yml

@@ -49,3 +49,60 @@ jobs:
 
       - name: Build manual
         run: nix build .#manual -L
+
+  nix-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4.1.7
+
+      - uses: DeterminateSystems/nix-installer-action@v14
+        continue-on-error: true # Self-hosted runners already have Nix installed
+
+      - name: Enable Binary Cache
+        uses: cachix/cachix-action@v12
+        with:
+          name: colmena
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - id: set-matrix
+        name: Generate Nix Matrix
+        run: |
+          set -Eeu
+          matrix="$(nix eval --json '.#githubActions.matrix')"
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+
+  nix-matrix-job:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    needs:
+      - build
+      - nix-matrix
+    strategy:
+      matrix: ${{ fromJSON(needs.nix-matrix.outputs.matrix) }}
+    steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          remove-dotnet: 'true'
+          build-mount-path: /nix
+
+      - name: Set /nix permissions
+        run: |
+          sudo chown root:root /nix
+
+      - uses: actions/checkout@v4.1.7
+
+      - uses: DeterminateSystems/nix-installer-action@v14
+        continue-on-error: true # Self-hosted runners already have Nix installed
+
+      - name: Enable Binary Cache
+        uses: cachix/cachix-action@v12
+        with:
+          name: colmena
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - name: Build ${{ matrix.attr }}
+        run: |
+          nix build --no-link --print-out-paths -L '.#${{ matrix.attr }}'

.gitignore

@@ -1,3 +1,4 @@
 result*
 /target
 /.direnv
+/.vscode

flake.lock

@@ -31,13 +31,33 @@
         "type": "github"
       }
     },
+    "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729742964,
+        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1725103162,
-        "narHash": "sha256-Ym04C5+qovuQDYL/rKWSR+WESseQBbNAe5DsXNx5trY=",
+        "lastModified": 1730785428,
+        "narHash": "sha256-Zwl8YgTVJTEum+L+0zVAWvXAGbWAuXHax3KzuejaDyo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "12228ff1752d7b7624a54e9c1af4b222b3c1073b",
+        "rev": "4aa36568d413aca0ea84a1684d2d46f55dbabad7",
         "type": "github"
       },
       "original": {
@@ -51,17 +71,18 @@
       "inputs": {
         "flake-compat": "flake-compat",
         "flake-utils": "flake-utils",
+        "nix-github-actions": "nix-github-actions",
         "nixpkgs": "nixpkgs",
         "stable": "stable"
       }
     },
     "stable": {
       "locked": {
-        "lastModified": 1724316499,
-        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
+        "lastModified": 1730883749,
+        "narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
+        "rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
         "type": "github"
       },
       "original": {

flake.nix

@@ -5,6 +5,11 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
     stable.url = "github:NixOS/nixpkgs/nixos-24.05";
 
+    nix-github-actions = {
+      url = "github:nix-community/nix-github-actions";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
     flake-utils.url = "github:numtide/flake-utils";
 
     flake-compat = {
@@ -13,12 +18,41 @@
     };
   };
 
-  outputs = { self, nixpkgs, stable, flake-utils, ... } @ inputs: let
+  outputs = {
+    self,
+    nixpkgs,
+    stable,
+    flake-utils,
+    nix-github-actions,
+    ...
+  } @ inputs: let
     supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
     colmenaOptions = import ./src/nix/hive/options.nix;
     colmenaModules = import ./src/nix/hive/modules.nix;
+
+    # Temporary fork of nix-eval-jobs with changes to be upstreamed
+    # Mostly for the integration test setup and not needed in most use cases
+    _evalJobsOverlay = final: prev: let
+      patched = prev.nix-eval-jobs.overrideAttrs (old: {
+        version = old.version + "-colmena";
+        patches = (old.patches or []) ++ [
+          # Allows NIX_PATH to be honored
+          (final.fetchpatch {
+            url = "https://github.com/zhaofengli/nix-eval-jobs/commit/6ff5972724230ac2b96eb1ec355cd25ca512ef57.patch";
+            hash = "sha256-2NiMYpw27N+X7Ixh2HkP3fcWvopDJWQDVjgRdhOL2QQ";
+          })
+        ];
+      });
+    in {
+      nix-eval-jobs = patched;
+    };
   in flake-utils.lib.eachSystem supportedSystems (system: let
-    pkgs = nixpkgs.legacyPackages.${system};
+    pkgs = import nixpkgs {
+      inherit system;
+      overlays = [
+        _evalJobsOverlay
+      ];
+    };
   in rec {
     # We still maintain the expression in a Nixpkgs-acceptable form
     defaultPackage = self.packages.${system}.colmena;
@@ -87,7 +121,7 @@
           self.overlays.default
           inputsOverlay
 
-          self._evalJobsOverlay
+          _evalJobsOverlay
         ];
       };
       pkgsStable = import stable {
@@ -96,7 +130,7 @@
           self.overlays.default
           inputsOverlay
 
-          self._evalJobsOverlay
+          _evalJobsOverlay
         ];
       };
     } else {};
@@ -115,21 +149,10 @@
       hermetic = true;
     };
 
-    # Temporary fork of nix-eval-jobs with changes to be upstreamed
-    # Mostly for the integration test setup and not needed in most use cases
-    _evalJobsOverlay = final: prev: let
-      patched = prev.nix-eval-jobs.overrideAttrs (old: {
-        version = old.version + "-colmena";
-        patches = (old.patches or []) ++ [
-          # Allows NIX_PATH to be honored
-          (final.fetchpatch {
-            url = "https://github.com/zhaofengli/nix-eval-jobs/commit/6ff5972724230ac2b96eb1ec355cd25ca512ef57.patch";
-            hash = "sha256-2NiMYpw27N+X7Ixh2HkP3fcWvopDJWQDVjgRdhOL2QQ";
-          })
-        ];
-      });
-    in {
-      nix-eval-jobs = patched;
+    githubActions = nix-github-actions.lib.mkGithubMatrix {
+      checks = {
+        inherit (self.checks) x86_64-linux;
+      };
     };
   };
 

garnix.yaml

@@ -1,3 +0,0 @@
-builds:
-  include:
-    - 'checks.x86_64-linux.*'

integration-tests/default.nix

@@ -8,8 +8,18 @@
   apply-local = import ./apply-local { inherit pkgs; };
   build-on-target = import ./build-on-target { inherit pkgs; };
   exec = import ./exec { inherit pkgs; };
-  flakes = import ./flakes { inherit pkgs; };
-  flakes-streaming = import ./flakes { inherit pkgs; evaluator = "streaming"; };
+
+  # FIXME: The old evaluation method doesn't work purely with Nix 2.21+
+  flakes = import ./flakes {
+    inherit pkgs;
+    extraApplyFlags = "--experimental-flake-eval";
+  };
+  flakes-impure = import ./flakes {
+    inherit pkgs;
+    pure = false;
+  };
+  #flakes-streaming = import ./flakes { inherit pkgs; evaluator = "streaming"; };
+
   parallel = import ./parallel { inherit pkgs; };
 
   allow-apply-all = import ./allow-apply-all { inherit pkgs; };

integration-tests/flakes/default.nix

@@ -1,13 +1,29 @@
 { pkgs
 , evaluator ? "chunked"
+, extraApplyFlags ? ""
+, pure ? true
 }:
 
 let
+  inherit (pkgs) lib;
+
   tools = pkgs.callPackage ../tools.nix {
     targets = [ "alpha" ];
   };
+
+  applyFlags = "--evaluator ${evaluator} ${extraApplyFlags}"
+    + lib.optionalString (!pure) "--impure";
+
+  # From integration-tests/nixpkgs.nix
+  colmenaFlakeInputs = pkgs._inputs;
 in tools.runTest {
-  name = "colmena-flakes-${evaluator}";
+  name = "colmena-flakes-${evaluator}"
+    + lib.optionalString (!pure) "-impure";
+
+  nodes.deployer = {
+    virtualisation.additionalPaths =
+      lib.mapAttrsToList (k: v: v.outPath) colmenaFlakeInputs;
+  };
 
   colmena.test = {
     bundle = ./.;
@@ -16,12 +32,13 @@ in tools.runTest {
       import re
 
       deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix")
+      deployer.succeed("sed -i 's @colmena@ path:${tools.colmena.src} g' /tmp/bundle/flake.nix")
 
       with subtest("Lock flake dependencies"):
           deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock")
 
       with subtest("Deploy with a plain flake without git"):
-          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
+          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}")
           alpha.succeed("grep FIRST /etc/deployment")
 
       with subtest("Deploy with a git flake"):
@@ -29,21 +46,22 @@ in tools.runTest {
 
           # don't put probe.nix in source control - should fail
           deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix")
-          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
-          assert re.search(r"probe.nix.*No such file or directory", logs)
+          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
+          assert re.search(r"probe.nix.*(No such file or directory|does not exist)", logs), "Expected error message not found in log"
 
           # now it should succeed
           deployer.succeed("cd /tmp/bundle && git add probe.nix")
-          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
+          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}")
           alpha.succeed("grep SECOND /etc/deployment")
 
+    '' + lib.optionalString pure ''
       with subtest("Check that impure expressions are forbidden"):
           deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix")
-          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
-          assert re.search(r"access to absolute path.*forbidden in pure eval mode", logs)
+          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
+          assert re.search(r"access to absolute path.*forbidden in pure (eval|evaluation) mode", logs), "Expected error message not found in log"
 
       with subtest("Check that impure expressions can be allowed with --impure"):
-          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator} --impure")
+          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags} --impure")
           alpha.succeed("grep deployer /etc/deployment")
     '';
   };

integration-tests/flakes/flake.nix

@@ -3,13 +3,15 @@
 
   inputs = {
     nixpkgs.url = "@nixpkgs@";
+    colmena.url = "@colmena@";
   };
 
-  outputs = { self, nixpkgs }: let
+  outputs = { self, nixpkgs, colmena }: let
     pkgs = import nixpkgs {
       system = "x86_64-linux";
     };
   in {
     colmena = import ./hive.nix { inherit pkgs; };
+    colmenaHive = colmena.lib.makeHive self.outputs.colmena;
   };
 }

integration-tests/tools.nix

@@ -129,9 +129,6 @@ let
       extraDeployerConfig
     ];
 
-    # FIXME: Colmena flake support is broken with Nix 2.24
-    nix.package = pkgs.nixVersions.nix_2_18;
-
     nix.registry = lib.mkIf (pkgs ? _inputs) {
       nixpkgs.flake = pkgs._inputs.nixpkgs;
     };
@@ -168,6 +165,9 @@ let
         exec "$@" 2> >(tee /dev/stderr)
       '')
     ];
+
+    # Re-enable switch-to-configuration
+    system.switch.enable = true;
   };
 
   # Setup for target nodes
@@ -183,6 +183,9 @@ let
       sshKeys.snakeOilPublicKey
     ];
     virtualisation.writableStore = true;
+
+    # Re-enable switch-to-configuration
+    system.switch.enable = true;
   };
 
   nodes = let

manual/src/tutorial/flakes.md

@@ -90,6 +90,34 @@ To build and deploy to all nodes:
 colmena apply
 ```
 
+## Direct Flake Evaluation (Experimental)
+
+By default, Colmena uses `nix-instantiate` to evaluate your flake which does not work purely on Nix 2.21+, necessitating the use of `--impure`.
+There is experimental support for evaluating flakes directly with `nix eval`, enabled via `--experimental-flake-eval`.
+
+To use this new evaluation mode, your flake needs to depend on Colmena itself as an input and expose a new output called `colmenaHive`:
+
+```diff
+ {
+   inputs = {
++    # ADDED: Colmena input
++    colmena.url = "github:zhaofengli/colmena";
+
+     # ... Rest of configuration ...
+   };
+   outputs = { self, colmena, ... }: {
++    # ADDED: New colmenaHive output
++    colmenaHive = colmena.lib.makeHive self.outputs.colmena;
+
+     # Your existing colmena output
+     colmena = {
+       # ... Rest of configuration ...
+     };
+   };
+ }
+```
+
+
 ## Next Steps
 
 - Head to the [Features](../features/index.md) section to see what else Colmena can do.

package.nix

@@ -5,7 +5,12 @@ rustPlatform.buildRustPackage rec {
   version = "0.5.0-pre";
 
   src = lib.cleanSourceWith {
-    filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) [ "target" "manual" "integration-tests" ]);
+    filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) [
+      ".github"
+      "target"
+      "manual"
+      "integration-tests"
+    ]);
     src = lib.cleanSource ./.;
   };
 

src/cli.rs

@@ -10,7 +10,7 @@ use env_logger::fmt::WriteStyle;
 use crate::{
     command::{self, apply::DeployOpts},
     error::ColmenaResult,
-    nix::{Hive, HivePath},
+    nix::{hive::EvaluationMethod, Hive, HivePath},
 };
 
 /// Base URL of the manual, without the trailing slash.
@@ -137,6 +137,21 @@ This only works when building locally.
         value_names = ["NAME", "VALUE"],
     )]
     nix_option: Vec<String>,
+    #[arg(
+        long,
+        default_value_t,
+        help = "Use direct flake evaluation (experimental)",
+        long_help = r#"If enabled, flakes will be evaluated using `nix eval`. This requires the flake to depend on Colmena as an input and expose a compatible `colmenaHive` output:
+
+  outputs = { self, colmena, ... }: {
+    colmenaHive = colmena.lib.makeHive self.outputs.colmena;
+    colmena = ...;
+  };
+
+This is an experimental feature."#,
+        global = true
+    )]
+    experimental_flake_eval: bool,
     #[arg(
         long,
         value_name = "WHEN",
@@ -262,6 +277,11 @@ async fn get_hive(opts: &Opts) -> ColmenaResult<Hive> {
         hive.set_impure(true);
     }
 
+    if opts.experimental_flake_eval {
+        log::warn!("Using direct flake evaluation (experimental)");
+        hive.set_evaluation_method(EvaluationMethod::DirectFlakeEval);
+    }
+
     for chunks in opts.nix_option.chunks_exact(2) {
         let [name, value] = chunks else {
             unreachable!()

src/nix/hive/eval.nix

@@ -181,7 +181,7 @@ let
 
 in rec {
   # Exported attributes
-  __schema = "v0";
+  __schema = "v0.20241006";
 
   nodes = listToAttrs (map (name: { inherit name; value = evalNode name (configsFor name); }) nodeNames);
   toplevel =         lib.mapAttrs (_: v: v.config.system.build.toplevel) nodes;

src/nix/hive/flake.nix

@@ -7,7 +7,7 @@
 
   outputs = { self, hive }: {
     processFlake = let
-      compatibleSchema = "v0";
+      compatibleSchema = "v0.20241006";
 
       # Evaluates a raw hive.
       #

src/nix/hive/mod.rs

@@ -8,6 +8,7 @@ use std::convert::AsRef;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
+use const_format::formatcp;
 use tokio::process::Command;
 use tokio::sync::OnceCell;
 use validator::Validate;
@@ -22,6 +23,21 @@ use crate::job::JobHandle;
 use crate::util::{CommandExecution, CommandExt};
 use assets::Assets;
 
+/// The version of the Hive schema we are compatible with.
+///
+/// Currently we are tied to one specific version.
+const HIVE_SCHEMA: &str = "v0.20241006";
+
+/// The snippet to be used for `nix eval --apply`.
+const FLAKE_APPLY_SNIPPET: &str = formatcp!(
+    r#"with builtins; hive: assert (hive.__schema == "{}" || throw ''
+    The colmenaHive output (schema ${{hive.__schema}}) isn't compatible with this version of Colmena.
+
+    Hint: Use the same version of Colmena as in the Flake input.
+''); "#,
+    HIVE_SCHEMA
+);
+
 #[derive(Debug, Clone)]
 pub enum HivePath {
     /// A Nix Flake.
@@ -63,11 +79,33 @@ impl FromStr for HivePath {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum EvaluationMethod {
+    /// Use nix-instantiate and specify the entire Nix expression.
+    ///
+    /// This is the default method.
+    ///
+    /// For flakes, we use `builtins.getFlakes`. Pure evaluation no longer works
+    /// with this method in Nix 2.21+.
+    NixInstantiate,
+
+    /// Use `nix eval --apply` on top of a flake.
+    ///
+    /// This can be activated with --experimental-flake-eval.
+    ///
+    /// In this method, we can no longer pull in our bundled assets and
+    /// the flake must expose a compatible `colmenaHive` output.
+    DirectFlakeEval,
+}
+
 #[derive(Debug)]
 pub struct Hive {
     /// Path to the hive.
     path: HivePath,
 
+    /// Method to evaluate the hive with.
+    evaluation_method: EvaluationMethod,
+
     /// Path to the context directory.
     ///
     /// Normally this is directory containing the "hive.nix"
@@ -134,6 +172,7 @@ impl Hive {
 
         Ok(Self {
             path,
+            evaluation_method: EvaluationMethod::NixInstantiate,
             context_dir,
             assets,
             show_trace: false,
@@ -158,6 +197,14 @@ impl Hive {
             .await
     }
 
+    pub fn set_evaluation_method(&mut self, method: EvaluationMethod) {
+        if !self.is_flake() && method == EvaluationMethod::DirectFlakeEval {
+            return;
+        }
+
+        self.evaluation_method = method;
+    }
+
     pub fn set_show_trace(&mut self, value: bool) {
         self.show_trace = value;
     }
@@ -421,7 +468,10 @@ impl Hive {
 
     /// Returns the base expression from which the evaluated Hive can be used.
     fn get_base_expression(&self) -> String {
-        self.assets.get_base_expression()
+        match self.evaluation_method {
+            EvaluationMethod::NixInstantiate => self.assets.get_base_expression(),
+            EvaluationMethod::DirectFlakeEval => FLAKE_APPLY_SNIPPET.to_string(),
+        }
     }
 
     /// Returns whether this Hive is a flake.
@@ -444,6 +494,11 @@ impl<'hive> NixInstantiate<'hive> {
     }
 
     fn instantiate(&self) -> Command {
+        // TODO: Better error handling
+        if self.hive.evaluation_method == EvaluationMethod::DirectFlakeEval {
+            panic!("Instantiation is not supported with DirectFlakeEval");
+        }
+
         let mut command = Command::new("nix-instantiate");
 
         if self.hive.is_flake() {
@@ -462,17 +517,48 @@ impl<'hive> NixInstantiate<'hive> {
     }
 
     fn eval(self) -> Command {
-        let mut command = self.instantiate();
         let flags = self.hive.nix_flags();
-        command
-            .arg("--eval")
-            .arg("--json")
-            .arg("--strict")
-            // Ensures the derivations are instantiated
-            // Required for system profile evaluation and IFD
-            .arg("--read-write-mode")
-            .args(flags.to_args());
-        command
+
+        match self.hive.evaluation_method {
+            EvaluationMethod::NixInstantiate => {
+                let mut command = self.instantiate();
+
+                command
+                    .arg("--eval")
+                    .arg("--json")
+                    .arg("--strict")
+                    // Ensures the derivations are instantiated
+                    // Required for system profile evaluation and IFD
+                    .arg("--read-write-mode")
+                    .args(flags.to_args());
+
+                command
+            }
+            EvaluationMethod::DirectFlakeEval => {
+                let mut command = Command::new("nix");
+                let flake = if let HivePath::Flake(flake) = self.hive.path() {
+                    flake
+                } else {
+                    panic!("The DirectFlakeEval evaluation method only support flakes");
+                };
+
+                let hive_installable = format!("{}#colmenaHive", flake.uri());
+
+                let mut full_expression = self.hive.get_base_expression();
+                full_expression += &self.expression;
+
+                command
+                    .arg("eval") // nix eval
+                    .args(["--extra-experimental-features", "flakes nix-command"])
+                    .arg(hive_installable)
+                    .arg("--json")
+                    .arg("--apply")
+                    .arg(&full_expression)
+                    .args(flags.to_args());
+
+                command
+            }
+        }
     }
 
     async fn instantiate_with_builders(self) -> ColmenaResult<Command> {