From ec25d799ed3ab3d671868948a36171e1bba692f8 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 01/15] .gitignore: Ignore .vscode

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index ac700cc..2ddacb1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 result*
 /target
 /.direnv
+/.vscode

From 70462312f2f91f6c03a0a4ce927692cac4546f1d Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 02/15] .editorconfig: Ignore Rust

Too rigid :/ Let's just let rustfmt handle it.
---
 .editorconfig | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.editorconfig b/.editorconfig
index 607992b..e776ebb 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -10,11 +10,6 @@ insert_final_newline = true
 trim_trailing_whitespace = true
 charset = utf-8
 
-# Rust
-[*.rs]
-indent_style = space
-indent_size = 2
-
 # Vendored
 [manual/theme/highlight.js]
 end_of_line = unset

From 524cd452999beb329cf1a07cc5976ef24991de4b Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 03/15] flake.lock: Update

---
 flake.lock | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/flake.lock b/flake.lock
index 90b9601..365a9e4 100644
--- a/flake.lock
+++ b/flake.lock
@@ -33,11 +33,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1725103162,
-        "narHash": "sha256-Ym04C5+qovuQDYL/rKWSR+WESseQBbNAe5DsXNx5trY=",
+        "lastModified": 1730785428,
+        "narHash": "sha256-Zwl8YgTVJTEum+L+0zVAWvXAGbWAuXHax3KzuejaDyo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "12228ff1752d7b7624a54e9c1af4b222b3c1073b",
+        "rev": "4aa36568d413aca0ea84a1684d2d46f55dbabad7",
         "type": "github"
       },
       "original": {
@@ -57,11 +57,11 @@
     },
     "stable": {
       "locked": {
-        "lastModified": 1724316499,
-        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
+        "lastModified": 1730883749,
+        "narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
+        "rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
         "type": "github"
       },
       "original": {

From 8a8f47a1b1a7729f759969b019e48368c4d13b75 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 04/15] flake.nix: Apply nix-eval-jobs overlay on output
 package set

---
 flake.nix | 45 +++++++++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/flake.nix b/flake.nix
index 78a8301..c9ba039 100644
--- a/flake.nix
+++ b/flake.nix
@@ -17,8 +17,30 @@
     supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
     colmenaOptions = import ./src/nix/hive/options.nix;
     colmenaModules = import ./src/nix/hive/modules.nix;
+
+    # Temporary fork of nix-eval-jobs with changes to be upstreamed
+    # Mostly for the integration test setup and not needed in most use cases
+    _evalJobsOverlay = final: prev: let
+      patched = prev.nix-eval-jobs.overrideAttrs (old: {
+        version = old.version + "-colmena";
+        patches = (old.patches or []) ++ [
+          # Allows NIX_PATH to be honored
+          (final.fetchpatch {
+            url = "https://github.com/zhaofengli/nix-eval-jobs/commit/6ff5972724230ac2b96eb1ec355cd25ca512ef57.patch";
+            hash = "sha256-2NiMYpw27N+X7Ixh2HkP3fcWvopDJWQDVjgRdhOL2QQ";
+          })
+        ];
+      });
+    in {
+      nix-eval-jobs = patched;
+    };
   in flake-utils.lib.eachSystem supportedSystems (system: let
-    pkgs = nixpkgs.legacyPackages.${system};
+    pkgs = import nixpkgs {
+      inherit system;
+      overlays = [
+        _evalJobsOverlay
+      ];
+    };
   in rec {
     # We still maintain the expression in a Nixpkgs-acceptable form
     defaultPackage = self.packages.${system}.colmena;
@@ -87,7 +109,7 @@
           self.overlays.default
           inputsOverlay
 
-          self._evalJobsOverlay
+          _evalJobsOverlay
         ];
       };
       pkgsStable = import stable {
@@ -96,7 +118,7 @@
           self.overlays.default
           inputsOverlay
 
-          self._evalJobsOverlay
+          _evalJobsOverlay
         ];
       };
     } else {};
@@ -114,23 +136,6 @@
       inherit rawHive colmenaOptions colmenaModules;
       hermetic = true;
     };
-
-    # Temporary fork of nix-eval-jobs with changes to be upstreamed
-    # Mostly for the integration test setup and not needed in most use cases
-    _evalJobsOverlay = final: prev: let
-      patched = prev.nix-eval-jobs.overrideAttrs (old: {
-        version = old.version + "-colmena";
-        patches = (old.patches or []) ++ [
-          # Allows NIX_PATH to be honored
-          (final.fetchpatch {
-            url = "https://github.com/zhaofengli/nix-eval-jobs/commit/6ff5972724230ac2b96eb1ec355cd25ca512ef57.patch";
-            hash = "sha256-2NiMYpw27N+X7Ixh2HkP3fcWvopDJWQDVjgRdhOL2QQ";
-          })
-        ];
-      });
-    in {
-      nix-eval-jobs = patched;
-    };
   };
 
   nixConfig = {

From 1f7b8ab80f5e19de7c03c7852c8ee385d4229742 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 05/15] package.nix: Exclude .github from cleaned source

---
 package.nix | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/package.nix b/package.nix
index bb4142f..a4b4b9f 100644
--- a/package.nix
+++ b/package.nix
@@ -5,7 +5,12 @@ rustPlatform.buildRustPackage rec {
   version = "0.5.0-pre";
 
   src = lib.cleanSourceWith {
-    filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) [ "target" "manual" "integration-tests" ]);
+    filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) [
+      ".github"
+      "target"
+      "manual"
+      "integration-tests"
+    ]);
     src = lib.cleanSource ./.;
   };
 

From 2708c9359f116106d75f26b685490359997c8f7f Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 06/15] Revert "[FIXME] integration-tests: Temproarily pin to
 Nix 2.18"

This reverts commit 141fe82f44509d33810d8453b74e9cda222d2d49.
---
 integration-tests/tools.nix | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/integration-tests/tools.nix b/integration-tests/tools.nix
index 1c717f7..5ed43ee 100644
--- a/integration-tests/tools.nix
+++ b/integration-tests/tools.nix
@@ -129,9 +129,6 @@ let
       extraDeployerConfig
     ];
 
-    # FIXME: Colmena flake support is broken with Nix 2.24
-    nix.package = pkgs.nixVersions.nix_2_18;
-
     nix.registry = lib.mkIf (pkgs ? _inputs) {
       nixpkgs.flake = pkgs._inputs.nixpkgs;
     };

From 1f669d4c78fb0b18b8841e139bd99135ef037638 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 07/15] Bump schema version to v0.20241006

---
 src/nix/hive/eval.nix  | 2 +-
 src/nix/hive/flake.nix | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nix/hive/eval.nix b/src/nix/hive/eval.nix
index 3572907..787bf69 100644
--- a/src/nix/hive/eval.nix
+++ b/src/nix/hive/eval.nix
@@ -181,7 +181,7 @@ let
 
 in rec {
   # Exported attributes
-  __schema = "v0";
+  __schema = "v0.20241006";
 
   nodes = listToAttrs (map (name: { inherit name; value = evalNode name (configsFor name); }) nodeNames);
   toplevel =         lib.mapAttrs (_: v: v.config.system.build.toplevel) nodes;
diff --git a/src/nix/hive/flake.nix b/src/nix/hive/flake.nix
index 167d35f..f40d528 100644
--- a/src/nix/hive/flake.nix
+++ b/src/nix/hive/flake.nix
@@ -7,7 +7,7 @@
 
   outputs = { self, hive }: {
     processFlake = let
-      compatibleSchema = "v0";
+      compatibleSchema = "v0.20241006";
 
       # Evaluates a raw hive.
       #

From dc80345deeccacbec138ab3ee50bdcf97b4b9aa2 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 08/15] Add direct flake evaluation support

---
 src/cli.rs          |  22 ++++++++-
 src/nix/hive/mod.rs | 108 +++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 118 insertions(+), 12 deletions(-)

diff --git a/src/cli.rs b/src/cli.rs
index 7acc241..9068da6 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -10,7 +10,7 @@ use env_logger::fmt::WriteStyle;
 use crate::{
     command::{self, apply::DeployOpts},
     error::ColmenaResult,
-    nix::{Hive, HivePath},
+    nix::{hive::EvaluationMethod, Hive, HivePath},
 };
 
 /// Base URL of the manual, without the trailing slash.
@@ -137,6 +137,21 @@ This only works when building locally.
         value_names = ["NAME", "VALUE"],
     )]
     nix_option: Vec<String>,
+    #[arg(
+        long,
+        default_value_t,
+        help = "Use direct flake evaluation (experimental)",
+        long_help = r#"If enabled, flakes will be evaluated using `nix eval`. This requires the flake to depend on Colmena as an input and expose a compatible `colmenaHive` output:
+
+  outputs = { self, colmena, ... }: {
+    colmenaHive = colmena.lib.makeHive self.outputs.colmena;
+    colmena = ...;
+  };
+
+This is an experimental feature."#,
+        global = true
+    )]
+    experimental_flake_eval: bool,
     #[arg(
         long,
         value_name = "WHEN",
@@ -262,6 +277,11 @@ async fn get_hive(opts: &Opts) -> ColmenaResult<Hive> {
         hive.set_impure(true);
     }
 
+    if opts.experimental_flake_eval {
+        log::warn!("Using direct flake evaluation (experimental)");
+        hive.set_evaluation_method(EvaluationMethod::DirectFlakeEval);
+    }
+
     for chunks in opts.nix_option.chunks_exact(2) {
         let [name, value] = chunks else {
             unreachable!()
diff --git a/src/nix/hive/mod.rs b/src/nix/hive/mod.rs
index f9d9227..2faa88e 100644
--- a/src/nix/hive/mod.rs
+++ b/src/nix/hive/mod.rs
@@ -8,6 +8,7 @@ use std::convert::AsRef;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
+use const_format::formatcp;
 use tokio::process::Command;
 use tokio::sync::OnceCell;
 use validator::Validate;
@@ -22,6 +23,21 @@ use crate::job::JobHandle;
 use crate::util::{CommandExecution, CommandExt};
 use assets::Assets;
 
+/// The version of the Hive schema we are compatible with.
+///
+/// Currently we are tied to one specific version.
+const HIVE_SCHEMA: &str = "v0.20241006";
+
+/// The snippet to be used for `nix eval --apply`.
+const FLAKE_APPLY_SNIPPET: &str = formatcp!(
+    r#"with builtins; hive: assert (hive.__schema == "{}" || throw ''
+    The colmenaHive output (schema ${{hive.__schema}}) isn't compatible with this version of Colmena.
+
+    Hint: Use the same version of Colmena as in the Flake input.
+''); "#,
+    HIVE_SCHEMA
+);
+
 #[derive(Debug, Clone)]
 pub enum HivePath {
     /// A Nix Flake.
@@ -63,11 +79,33 @@ impl FromStr for HivePath {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum EvaluationMethod {
+    /// Use nix-instantiate and specify the entire Nix expression.
+    ///
+    /// This is the default method.
+    ///
+    /// For flakes, we use `builtins.getFlakes`. Pure evaluation no longer works
+    /// with this method in Nix 2.21+.
+    NixInstantiate,
+
+    /// Use `nix eval --apply` on top of a flake.
+    ///
+    /// This can be activated with --experimental-flake-eval.
+    ///
+    /// In this method, we can no longer pull in our bundled assets and
+    /// the flake must expose a compatible `colmenaHive` output.
+    DirectFlakeEval,
+}
+
 #[derive(Debug)]
 pub struct Hive {
     /// Path to the hive.
     path: HivePath,
 
+    /// Method to evaluate the hive with.
+    evaluation_method: EvaluationMethod,
+
     /// Path to the context directory.
     ///
     /// Normally this is directory containing the "hive.nix"
@@ -134,6 +172,7 @@ impl Hive {
 
         Ok(Self {
             path,
+            evaluation_method: EvaluationMethod::NixInstantiate,
             context_dir,
             assets,
             show_trace: false,
@@ -158,6 +197,14 @@ impl Hive {
             .await
     }
 
+    pub fn set_evaluation_method(&mut self, method: EvaluationMethod) {
+        if !self.is_flake() && method == EvaluationMethod::DirectFlakeEval {
+            return;
+        }
+
+        self.evaluation_method = method;
+    }
+
     pub fn set_show_trace(&mut self, value: bool) {
         self.show_trace = value;
     }
@@ -421,7 +468,10 @@ impl Hive {
 
     /// Returns the base expression from which the evaluated Hive can be used.
     fn get_base_expression(&self) -> String {
-        self.assets.get_base_expression()
+        match self.evaluation_method {
+            EvaluationMethod::NixInstantiate => self.assets.get_base_expression(),
+            EvaluationMethod::DirectFlakeEval => FLAKE_APPLY_SNIPPET.to_string(),
+        }
     }
 
     /// Returns whether this Hive is a flake.
@@ -444,6 +494,11 @@ impl<'hive> NixInstantiate<'hive> {
     }
 
     fn instantiate(&self) -> Command {
+        // TODO: Better error handling
+        if self.hive.evaluation_method == EvaluationMethod::DirectFlakeEval {
+            panic!("Instantiation is not supported with DirectFlakeEval");
+        }
+
         let mut command = Command::new("nix-instantiate");
 
         if self.hive.is_flake() {
@@ -462,17 +517,48 @@ impl<'hive> NixInstantiate<'hive> {
     }
 
     fn eval(self) -> Command {
-        let mut command = self.instantiate();
         let flags = self.hive.nix_flags();
-        command
-            .arg("--eval")
-            .arg("--json")
-            .arg("--strict")
-            // Ensures the derivations are instantiated
-            // Required for system profile evaluation and IFD
-            .arg("--read-write-mode")
-            .args(flags.to_args());
-        command
+
+        match self.hive.evaluation_method {
+            EvaluationMethod::NixInstantiate => {
+                let mut command = self.instantiate();
+
+                command
+                    .arg("--eval")
+                    .arg("--json")
+                    .arg("--strict")
+                    // Ensures the derivations are instantiated
+                    // Required for system profile evaluation and IFD
+                    .arg("--read-write-mode")
+                    .args(flags.to_args());
+
+                command
+            }
+            EvaluationMethod::DirectFlakeEval => {
+                let mut command = Command::new("nix");
+                let flake = if let HivePath::Flake(flake) = self.hive.path() {
+                    flake
+                } else {
+                    panic!("The DirectFlakeEval evaluation method only support flakes");
+                };
+
+                let hive_installable = format!("{}#colmenaHive", flake.uri());
+
+                let mut full_expression = self.hive.get_base_expression();
+                full_expression += &self.expression;
+
+                command
+                    .arg("eval") // nix eval
+                    .args(["--extra-experimental-features", "flakes nix-command"])
+                    .arg(hive_installable)
+                    .arg("--json")
+                    .arg("--apply")
+                    .arg(&full_expression)
+                    .args(flags.to_args());
+
+                command
+            }
+        }
     }
 
     async fn instantiate_with_builders(self) -> ColmenaResult<Command> {

From 0a836dc251b10a6c3125c2aea698ece363338c0f Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 09/15] integration-tests/tools: Re-enable
 switch-to-configuration

---
 integration-tests/tools.nix | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/integration-tests/tools.nix b/integration-tests/tools.nix
index 5ed43ee..22a395f 100644
--- a/integration-tests/tools.nix
+++ b/integration-tests/tools.nix
@@ -165,6 +165,9 @@ let
         exec "$@" 2> >(tee /dev/stderr)
       '')
     ];
+
+    # Re-enable switch-to-configuration
+    system.switch.enable = true;
   };
 
   # Setup for target nodes
@@ -180,6 +183,9 @@ let
       sshKeys.snakeOilPublicKey
     ];
     virtualisation.writableStore = true;
+
+    # Re-enable switch-to-configuration
+    system.switch.enable = true;
   };
 
   nodes = let

From 45ca75bcea32f712d3b6144ebf06ef861511d40e Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 10/15] integration-tests/flakes: Switch to direct flake
 evaluation (experimental)

---
 integration-tests/default.nix        | 14 +++++++++++--
 integration-tests/flakes/default.nix | 30 ++++++++++++++++++++++------
 integration-tests/flakes/flake.nix   |  4 +++-
 3 files changed, 39 insertions(+), 9 deletions(-)

diff --git a/integration-tests/default.nix b/integration-tests/default.nix
index 4fdaa39..a9453a2 100644
--- a/integration-tests/default.nix
+++ b/integration-tests/default.nix
@@ -8,8 +8,18 @@
   apply-local = import ./apply-local { inherit pkgs; };
   build-on-target = import ./build-on-target { inherit pkgs; };
   exec = import ./exec { inherit pkgs; };
-  flakes = import ./flakes { inherit pkgs; };
-  flakes-streaming = import ./flakes { inherit pkgs; evaluator = "streaming"; };
+
+  # FIXME: The old evaluation method doesn't work purely with Nix 2.21+
+  flakes = import ./flakes {
+    inherit pkgs;
+    extraApplyFlags = "--experimental-flake-eval";
+  };
+  flakes-impure = import ./flakes {
+    inherit pkgs;
+    pure = false;
+  };
+  #flakes-streaming = import ./flakes { inherit pkgs; evaluator = "streaming"; };
+
   parallel = import ./parallel { inherit pkgs; };
 
   allow-apply-all = import ./allow-apply-all { inherit pkgs; };
diff --git a/integration-tests/flakes/default.nix b/integration-tests/flakes/default.nix
index 2f3f3c7..8938310 100644
--- a/integration-tests/flakes/default.nix
+++ b/integration-tests/flakes/default.nix
@@ -1,13 +1,29 @@
 { pkgs
 , evaluator ? "chunked"
+, extraApplyFlags ? ""
+, pure ? true
 }:
 
 let
+  inherit (pkgs) lib;
+
   tools = pkgs.callPackage ../tools.nix {
     targets = [ "alpha" ];
   };
+
+  applyFlags = "--evaluator ${evaluator} ${extraApplyFlags}"
+    + lib.optionalString (!pure) "--impure";
+
+  # From integration-tests/nixpkgs.nix
+  colmenaFlakeInputs = pkgs._inputs;
 in tools.runTest {
-  name = "colmena-flakes-${evaluator}";
+  name = "colmena-flakes-${evaluator}"
+    + lib.optionalString (!pure) "-impure";
+
+  nodes.deployer = {
+    virtualisation.additionalPaths =
+      lib.mapAttrsToList (k: v: v.outPath) colmenaFlakeInputs;
+  };
 
   colmena.test = {
     bundle = ./.;
@@ -16,12 +32,13 @@ in tools.runTest {
       import re
 
       deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix")
+      deployer.succeed("sed -i 's @colmena@ path:${tools.colmena.src} g' /tmp/bundle/flake.nix")
 
       with subtest("Lock flake dependencies"):
           deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock")
 
       with subtest("Deploy with a plain flake without git"):
-          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
+          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}")
           alpha.succeed("grep FIRST /etc/deployment")
 
       with subtest("Deploy with a git flake"):
@@ -29,21 +46,22 @@ in tools.runTest {
 
           # don't put probe.nix in source control - should fail
           deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix")
-          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
+          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
           assert re.search(r"probe.nix.*No such file or directory", logs)
 
           # now it should succeed
           deployer.succeed("cd /tmp/bundle && git add probe.nix")
-          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
+          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}")
           alpha.succeed("grep SECOND /etc/deployment")
 
+    '' + lib.optionalString pure ''
       with subtest("Check that impure expressions are forbidden"):
           deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix")
-          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
+          logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
           assert re.search(r"access to absolute path.*forbidden in pure eval mode", logs)
 
       with subtest("Check that impure expressions can be allowed with --impure"):
-          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator} --impure")
+          deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags} --impure")
           alpha.succeed("grep deployer /etc/deployment")
     '';
   };
diff --git a/integration-tests/flakes/flake.nix b/integration-tests/flakes/flake.nix
index 4924867..3346774 100644
--- a/integration-tests/flakes/flake.nix
+++ b/integration-tests/flakes/flake.nix
@@ -3,13 +3,15 @@
 
   inputs = {
     nixpkgs.url = "@nixpkgs@";
+    colmena.url = "@colmena@";
   };
 
-  outputs = { self, nixpkgs }: let
+  outputs = { self, nixpkgs, colmena }: let
     pkgs = import nixpkgs {
       system = "x86_64-linux";
     };
   in {
     colmena = import ./hive.nix { inherit pkgs; };
+    colmenaHive = colmena.lib.makeHive self.outputs.colmena;
   };
 }

From 0ef98d060c265062083bab380793e3f92bdd6337 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 11/15] integration-tests/flakes: Fix error message matching

---
 integration-tests/flakes/default.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration-tests/flakes/default.nix b/integration-tests/flakes/default.nix
index 8938310..c8b6935 100644
--- a/integration-tests/flakes/default.nix
+++ b/integration-tests/flakes/default.nix
@@ -47,7 +47,7 @@ in tools.runTest {
           # don't put probe.nix in source control - should fail
           deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix")
           logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
-          assert re.search(r"probe.nix.*No such file or directory", logs)
+          assert re.search(r"probe.nix.*(No such file or directory|does not exist)", logs), "Expected error message not found in log"
 
           # now it should succeed
           deployer.succeed("cd /tmp/bundle && git add probe.nix")
@@ -58,7 +58,7 @@ in tools.runTest {
       with subtest("Check that impure expressions are forbidden"):
           deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix")
           logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
-          assert re.search(r"access to absolute path.*forbidden in pure eval mode", logs)
+          assert re.search(r"access to absolute path.*forbidden in pure (eval|evaluation) mode", logs), "Expected error message not found in log"
 
       with subtest("Check that impure expressions can be allowed with --impure"):
           deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags} --impure")

From e17c521c15746e326f800dc67e2fb3e211c9d3bb Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 12/15] Run integration tests on GitHub Actions

---
 .github/workflows/build.yml | 47 +++++++++++++++++++++++++++++++++++++
 flake.lock                  | 21 +++++++++++++++++
 flake.nix                   | 20 +++++++++++++++-
 3 files changed, 87 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3b5415f..477eeb9 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -49,3 +49,50 @@ jobs:
 
       - name: Build manual
         run: nix build .#manual -L
+
+  nix-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4.1.7
+
+      - uses: DeterminateSystems/nix-installer-action@v14
+        continue-on-error: true # Self-hosted runners already have Nix installed
+
+      - name: Enable Binary Cache
+        uses: cachix/cachix-action@v12
+        with:
+          name: colmena
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - id: set-matrix
+        name: Generate Nix Matrix
+        run: |
+          set -Eeu
+          matrix="$(nix eval --json '.#githubActions.matrix')"
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+
+  nix-matrix-job:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    needs:
+      - build
+      - nix-matrix
+    strategy:
+      matrix: ${{ fromJSON(needs.nix-matrix.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v4.1.7
+
+      - uses: DeterminateSystems/nix-installer-action@v14
+        continue-on-error: true # Self-hosted runners already have Nix installed
+
+      - name: Enable Binary Cache
+        uses: cachix/cachix-action@v12
+        with:
+          name: colmena
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
+      - name: Build ${{ matrix.attr }}
+        run: |
+          nix build --no-link --print-out-paths -L '.#${{ matrix.attr }}'
diff --git a/flake.lock b/flake.lock
index 365a9e4..b24e14b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -31,6 +31,26 @@
         "type": "github"
       }
     },
+    "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729742964,
+        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1730785428,
@@ -51,6 +71,7 @@
       "inputs": {
         "flake-compat": "flake-compat",
         "flake-utils": "flake-utils",
+        "nix-github-actions": "nix-github-actions",
         "nixpkgs": "nixpkgs",
         "stable": "stable"
       }
diff --git a/flake.nix b/flake.nix
index c9ba039..a00ae53 100644
--- a/flake.nix
+++ b/flake.nix
@@ -5,6 +5,11 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
     stable.url = "github:NixOS/nixpkgs/nixos-24.05";
 
+    nix-github-actions = {
+      url = "github:nix-community/nix-github-actions";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
     flake-utils.url = "github:numtide/flake-utils";
 
     flake-compat = {
@@ -13,7 +18,14 @@
     };
   };
 
-  outputs = { self, nixpkgs, stable, flake-utils, ... } @ inputs: let
+  outputs = {
+    self,
+    nixpkgs,
+    stable,
+    flake-utils,
+    nix-github-actions,
+    ...
+  } @ inputs: let
     supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
     colmenaOptions = import ./src/nix/hive/options.nix;
     colmenaModules = import ./src/nix/hive/modules.nix;
@@ -136,6 +148,12 @@
       inherit rawHive colmenaOptions colmenaModules;
       hermetic = true;
     };
+
+    githubActions = nix-github-actions.lib.mkGithubMatrix {
+      checks = {
+        inherit (self.checks) x86_64-linux;
+      };
+    };
   };
 
   nixConfig = {

From f593c24aa2d314a9d39830f021e3a04c33b8c684 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 13/15] .github/nix-matrix-job: Maximize build space

---
 .github/workflows/build.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 477eeb9..6af53b5 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -82,6 +82,16 @@ jobs:
     strategy:
       matrix: ${{ fromJSON(needs.nix-matrix.outputs.matrix) }}
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          remove-dotnet: 'true'
+          build-mount-path: /nix
+
+      - name: Set /nix permissions
+        run: |
+          sudo chown root:root /nix
+
       - uses: actions/checkout@v4.1.7
 
       - uses: DeterminateSystems/nix-installer-action@v14

From 33c41abd44fe5614a2fb28a22c034953eb17e17d Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:24:37 -0700
Subject: [PATCH 14/15] Remove garnix config

---
 garnix.yaml | 3 ---
 1 file changed, 3 deletions(-)
 delete mode 100644 garnix.yaml

diff --git a/garnix.yaml b/garnix.yaml
deleted file mode 100644
index 9da4f8e..0000000
--- a/garnix.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-builds:
-  include:
-    - 'checks.x86_64-linux.*'

From a4604f33719eeb4fee048b0313e909c8b9c8af71 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Thu, 7 Nov 2024 15:50:41 -0700
Subject: [PATCH 15/15] manual/flakes: Document direct flake evaluation

---
 manual/src/tutorial/flakes.md | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/manual/src/tutorial/flakes.md b/manual/src/tutorial/flakes.md
index 5995971..b87301c 100644
--- a/manual/src/tutorial/flakes.md
+++ b/manual/src/tutorial/flakes.md
@@ -90,6 +90,34 @@ To build and deploy to all nodes:
 colmena apply
 ```
 
+## Direct Flake Evaluation (Experimental)
+
+By default, Colmena uses `nix-instantiate` to evaluate your flake which does not work purely on Nix 2.21+, necessitating the use of `--impure`.
+There is experimental support for evaluating flakes directly with `nix eval`, enabled via `--experimental-flake-eval`.
+
+To use this new evaluation mode, your flake needs to depend on Colmena itself as an input and expose a new output called `colmenaHive`:
+
+```diff
+ {
+   inputs = {
++    # ADDED: Colmena input
++    colmena.url = "github:zhaofengli/colmena";
+
+     # ... Rest of configuration ...
+   };
+   outputs = { self, colmena, ... }: {
++    # ADDED: New colmenaHive output
++    colmenaHive = colmena.lib.makeHive self.outputs.colmena;
+
+     # Your existing colmena output
+     colmena = {
+       # ... Rest of configuration ...
+     };
+   };
+ }
+```
+
+
 ## Next Steps
 
 - Head to the [Features](../features/index.md) section to see what else Colmena can do.