Merge pull request #228 from zhaofengli/direct-flake-eval
Add direct flake evaluation support
This commit is contained in:
commit
2c95c1766a
16 changed files with 328 additions and 62 deletions
|
@ -10,11 +10,6 @@ insert_final_newline = true
|
||||||
trim_trailing_whitespace = true
|
trim_trailing_whitespace = true
|
||||||
charset = utf-8
|
charset = utf-8
|
||||||
|
|
||||||
# Rust
|
|
||||||
[*.rs]
|
|
||||||
indent_style = space
|
|
||||||
indent_size = 2
|
|
||||||
|
|
||||||
# Vendored
|
# Vendored
|
||||||
[manual/theme/highlight.js]
|
[manual/theme/highlight.js]
|
||||||
end_of_line = unset
|
end_of_line = unset
|
||||||
|
|
57
.github/workflows/build.yml
vendored
57
.github/workflows/build.yml
vendored
|
@ -49,3 +49,60 @@ jobs:
|
||||||
|
|
||||||
- name: Build manual
|
- name: Build manual
|
||||||
run: nix build .#manual -L
|
run: nix build .#manual -L
|
||||||
|
|
||||||
|
nix-matrix:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4.1.7
|
||||||
|
|
||||||
|
- uses: DeterminateSystems/nix-installer-action@v14
|
||||||
|
continue-on-error: true # Self-hosted runners already have Nix installed
|
||||||
|
|
||||||
|
- name: Enable Binary Cache
|
||||||
|
uses: cachix/cachix-action@v12
|
||||||
|
with:
|
||||||
|
name: colmena
|
||||||
|
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
|
||||||
|
|
||||||
|
- id: set-matrix
|
||||||
|
name: Generate Nix Matrix
|
||||||
|
run: |
|
||||||
|
set -Eeu
|
||||||
|
matrix="$(nix eval --json '.#githubActions.matrix')"
|
||||||
|
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
nix-matrix-job:
|
||||||
|
name: ${{ matrix.name }}
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
needs:
|
||||||
|
- build
|
||||||
|
- nix-matrix
|
||||||
|
strategy:
|
||||||
|
matrix: ${{ fromJSON(needs.nix-matrix.outputs.matrix) }}
|
||||||
|
steps:
|
||||||
|
- name: Maximize build space
|
||||||
|
uses: easimon/maximize-build-space@master
|
||||||
|
with:
|
||||||
|
remove-dotnet: 'true'
|
||||||
|
build-mount-path: /nix
|
||||||
|
|
||||||
|
- name: Set /nix permissions
|
||||||
|
run: |
|
||||||
|
sudo chown root:root /nix
|
||||||
|
|
||||||
|
- uses: actions/checkout@v4.1.7
|
||||||
|
|
||||||
|
- uses: DeterminateSystems/nix-installer-action@v14
|
||||||
|
continue-on-error: true # Self-hosted runners already have Nix installed
|
||||||
|
|
||||||
|
- name: Enable Binary Cache
|
||||||
|
uses: cachix/cachix-action@v12
|
||||||
|
with:
|
||||||
|
name: colmena
|
||||||
|
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
|
||||||
|
|
||||||
|
- name: Build ${{ matrix.attr }}
|
||||||
|
run: |
|
||||||
|
nix build --no-link --print-out-paths -L '.#${{ matrix.attr }}'
|
||||||
|
|
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -1,3 +1,4 @@
|
||||||
result*
|
result*
|
||||||
/target
|
/target
|
||||||
/.direnv
|
/.direnv
|
||||||
|
/.vscode
|
||||||
|
|
33
flake.lock
33
flake.lock
|
@ -31,13 +31,33 @@
|
||||||
"type": "github"
|
"type": "github"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"nix-github-actions": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixpkgs"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1729742964,
|
||||||
|
"narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
|
||||||
|
"owner": "nix-community",
|
||||||
|
"repo": "nix-github-actions",
|
||||||
|
"rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "nix-community",
|
||||||
|
"repo": "nix-github-actions",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"nixpkgs": {
|
"nixpkgs": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1725103162,
|
"lastModified": 1730785428,
|
||||||
"narHash": "sha256-Ym04C5+qovuQDYL/rKWSR+WESseQBbNAe5DsXNx5trY=",
|
"narHash": "sha256-Zwl8YgTVJTEum+L+0zVAWvXAGbWAuXHax3KzuejaDyo=",
|
||||||
"owner": "NixOS",
|
"owner": "NixOS",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"rev": "12228ff1752d7b7624a54e9c1af4b222b3c1073b",
|
"rev": "4aa36568d413aca0ea84a1684d2d46f55dbabad7",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
@ -51,17 +71,18 @@
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"flake-compat": "flake-compat",
|
"flake-compat": "flake-compat",
|
||||||
"flake-utils": "flake-utils",
|
"flake-utils": "flake-utils",
|
||||||
|
"nix-github-actions": "nix-github-actions",
|
||||||
"nixpkgs": "nixpkgs",
|
"nixpkgs": "nixpkgs",
|
||||||
"stable": "stable"
|
"stable": "stable"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"stable": {
|
"stable": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1724316499,
|
"lastModified": 1730883749,
|
||||||
"narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
|
"narHash": "sha256-mwrFF0vElHJP8X3pFCByJR365Q2463ATp2qGIrDUdlE=",
|
||||||
"owner": "NixOS",
|
"owner": "NixOS",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
|
"rev": "dba414932936fde69f0606b4f1d87c5bc0003ede",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
|
|
61
flake.nix
61
flake.nix
|
@ -5,6 +5,11 @@
|
||||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||||
stable.url = "github:NixOS/nixpkgs/nixos-24.05";
|
stable.url = "github:NixOS/nixpkgs/nixos-24.05";
|
||||||
|
|
||||||
|
nix-github-actions = {
|
||||||
|
url = "github:nix-community/nix-github-actions";
|
||||||
|
inputs.nixpkgs.follows = "nixpkgs";
|
||||||
|
};
|
||||||
|
|
||||||
flake-utils.url = "github:numtide/flake-utils";
|
flake-utils.url = "github:numtide/flake-utils";
|
||||||
|
|
||||||
flake-compat = {
|
flake-compat = {
|
||||||
|
@ -13,12 +18,41 @@
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
outputs = { self, nixpkgs, stable, flake-utils, ... } @ inputs: let
|
outputs = {
|
||||||
|
self,
|
||||||
|
nixpkgs,
|
||||||
|
stable,
|
||||||
|
flake-utils,
|
||||||
|
nix-github-actions,
|
||||||
|
...
|
||||||
|
} @ inputs: let
|
||||||
supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
|
supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
|
||||||
colmenaOptions = import ./src/nix/hive/options.nix;
|
colmenaOptions = import ./src/nix/hive/options.nix;
|
||||||
colmenaModules = import ./src/nix/hive/modules.nix;
|
colmenaModules = import ./src/nix/hive/modules.nix;
|
||||||
|
|
||||||
|
# Temporary fork of nix-eval-jobs with changes to be upstreamed
|
||||||
|
# Mostly for the integration test setup and not needed in most use cases
|
||||||
|
_evalJobsOverlay = final: prev: let
|
||||||
|
patched = prev.nix-eval-jobs.overrideAttrs (old: {
|
||||||
|
version = old.version + "-colmena";
|
||||||
|
patches = (old.patches or []) ++ [
|
||||||
|
# Allows NIX_PATH to be honored
|
||||||
|
(final.fetchpatch {
|
||||||
|
url = "https://github.com/zhaofengli/nix-eval-jobs/commit/6ff5972724230ac2b96eb1ec355cd25ca512ef57.patch";
|
||||||
|
hash = "sha256-2NiMYpw27N+X7Ixh2HkP3fcWvopDJWQDVjgRdhOL2QQ";
|
||||||
|
})
|
||||||
|
];
|
||||||
|
});
|
||||||
|
in {
|
||||||
|
nix-eval-jobs = patched;
|
||||||
|
};
|
||||||
in flake-utils.lib.eachSystem supportedSystems (system: let
|
in flake-utils.lib.eachSystem supportedSystems (system: let
|
||||||
pkgs = nixpkgs.legacyPackages.${system};
|
pkgs = import nixpkgs {
|
||||||
|
inherit system;
|
||||||
|
overlays = [
|
||||||
|
_evalJobsOverlay
|
||||||
|
];
|
||||||
|
};
|
||||||
in rec {
|
in rec {
|
||||||
# We still maintain the expression in a Nixpkgs-acceptable form
|
# We still maintain the expression in a Nixpkgs-acceptable form
|
||||||
defaultPackage = self.packages.${system}.colmena;
|
defaultPackage = self.packages.${system}.colmena;
|
||||||
|
@ -87,7 +121,7 @@
|
||||||
self.overlays.default
|
self.overlays.default
|
||||||
inputsOverlay
|
inputsOverlay
|
||||||
|
|
||||||
self._evalJobsOverlay
|
_evalJobsOverlay
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
pkgsStable = import stable {
|
pkgsStable = import stable {
|
||||||
|
@ -96,7 +130,7 @@
|
||||||
self.overlays.default
|
self.overlays.default
|
||||||
inputsOverlay
|
inputsOverlay
|
||||||
|
|
||||||
self._evalJobsOverlay
|
_evalJobsOverlay
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
} else {};
|
} else {};
|
||||||
|
@ -115,21 +149,10 @@
|
||||||
hermetic = true;
|
hermetic = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
# Temporary fork of nix-eval-jobs with changes to be upstreamed
|
githubActions = nix-github-actions.lib.mkGithubMatrix {
|
||||||
# Mostly for the integration test setup and not needed in most use cases
|
checks = {
|
||||||
_evalJobsOverlay = final: prev: let
|
inherit (self.checks) x86_64-linux;
|
||||||
patched = prev.nix-eval-jobs.overrideAttrs (old: {
|
};
|
||||||
version = old.version + "-colmena";
|
|
||||||
patches = (old.patches or []) ++ [
|
|
||||||
# Allows NIX_PATH to be honored
|
|
||||||
(final.fetchpatch {
|
|
||||||
url = "https://github.com/zhaofengli/nix-eval-jobs/commit/6ff5972724230ac2b96eb1ec355cd25ca512ef57.patch";
|
|
||||||
hash = "sha256-2NiMYpw27N+X7Ixh2HkP3fcWvopDJWQDVjgRdhOL2QQ";
|
|
||||||
})
|
|
||||||
];
|
|
||||||
});
|
|
||||||
in {
|
|
||||||
nix-eval-jobs = patched;
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -1,3 +0,0 @@
|
||||||
builds:
|
|
||||||
include:
|
|
||||||
- 'checks.x86_64-linux.*'
|
|
|
@ -8,8 +8,18 @@
|
||||||
apply-local = import ./apply-local { inherit pkgs; };
|
apply-local = import ./apply-local { inherit pkgs; };
|
||||||
build-on-target = import ./build-on-target { inherit pkgs; };
|
build-on-target = import ./build-on-target { inherit pkgs; };
|
||||||
exec = import ./exec { inherit pkgs; };
|
exec = import ./exec { inherit pkgs; };
|
||||||
flakes = import ./flakes { inherit pkgs; };
|
|
||||||
flakes-streaming = import ./flakes { inherit pkgs; evaluator = "streaming"; };
|
# FIXME: The old evaluation method doesn't work purely with Nix 2.21+
|
||||||
|
flakes = import ./flakes {
|
||||||
|
inherit pkgs;
|
||||||
|
extraApplyFlags = "--experimental-flake-eval";
|
||||||
|
};
|
||||||
|
flakes-impure = import ./flakes {
|
||||||
|
inherit pkgs;
|
||||||
|
pure = false;
|
||||||
|
};
|
||||||
|
#flakes-streaming = import ./flakes { inherit pkgs; evaluator = "streaming"; };
|
||||||
|
|
||||||
parallel = import ./parallel { inherit pkgs; };
|
parallel = import ./parallel { inherit pkgs; };
|
||||||
|
|
||||||
allow-apply-all = import ./allow-apply-all { inherit pkgs; };
|
allow-apply-all = import ./allow-apply-all { inherit pkgs; };
|
||||||
|
|
|
@ -1,13 +1,29 @@
|
||||||
{ pkgs
|
{ pkgs
|
||||||
, evaluator ? "chunked"
|
, evaluator ? "chunked"
|
||||||
|
, extraApplyFlags ? ""
|
||||||
|
, pure ? true
|
||||||
}:
|
}:
|
||||||
|
|
||||||
let
|
let
|
||||||
|
inherit (pkgs) lib;
|
||||||
|
|
||||||
tools = pkgs.callPackage ../tools.nix {
|
tools = pkgs.callPackage ../tools.nix {
|
||||||
targets = [ "alpha" ];
|
targets = [ "alpha" ];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
applyFlags = "--evaluator ${evaluator} ${extraApplyFlags}"
|
||||||
|
+ lib.optionalString (!pure) "--impure";
|
||||||
|
|
||||||
|
# From integration-tests/nixpkgs.nix
|
||||||
|
colmenaFlakeInputs = pkgs._inputs;
|
||||||
in tools.runTest {
|
in tools.runTest {
|
||||||
name = "colmena-flakes-${evaluator}";
|
name = "colmena-flakes-${evaluator}"
|
||||||
|
+ lib.optionalString (!pure) "-impure";
|
||||||
|
|
||||||
|
nodes.deployer = {
|
||||||
|
virtualisation.additionalPaths =
|
||||||
|
lib.mapAttrsToList (k: v: v.outPath) colmenaFlakeInputs;
|
||||||
|
};
|
||||||
|
|
||||||
colmena.test = {
|
colmena.test = {
|
||||||
bundle = ./.;
|
bundle = ./.;
|
||||||
|
@ -16,12 +32,13 @@ in tools.runTest {
|
||||||
import re
|
import re
|
||||||
|
|
||||||
deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix")
|
deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix")
|
||||||
|
deployer.succeed("sed -i 's @colmena@ path:${tools.colmena.src} g' /tmp/bundle/flake.nix")
|
||||||
|
|
||||||
with subtest("Lock flake dependencies"):
|
with subtest("Lock flake dependencies"):
|
||||||
deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock")
|
deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock")
|
||||||
|
|
||||||
with subtest("Deploy with a plain flake without git"):
|
with subtest("Deploy with a plain flake without git"):
|
||||||
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
|
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}")
|
||||||
alpha.succeed("grep FIRST /etc/deployment")
|
alpha.succeed("grep FIRST /etc/deployment")
|
||||||
|
|
||||||
with subtest("Deploy with a git flake"):
|
with subtest("Deploy with a git flake"):
|
||||||
|
@ -29,21 +46,22 @@ in tools.runTest {
|
||||||
|
|
||||||
# don't put probe.nix in source control - should fail
|
# don't put probe.nix in source control - should fail
|
||||||
deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix")
|
deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix")
|
||||||
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
|
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
|
||||||
assert re.search(r"probe.nix.*No such file or directory", logs)
|
assert re.search(r"probe.nix.*(No such file or directory|does not exist)", logs), "Expected error message not found in log"
|
||||||
|
|
||||||
# now it should succeed
|
# now it should succeed
|
||||||
deployer.succeed("cd /tmp/bundle && git add probe.nix")
|
deployer.succeed("cd /tmp/bundle && git add probe.nix")
|
||||||
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
|
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}")
|
||||||
alpha.succeed("grep SECOND /etc/deployment")
|
alpha.succeed("grep SECOND /etc/deployment")
|
||||||
|
|
||||||
|
'' + lib.optionalString pure ''
|
||||||
with subtest("Check that impure expressions are forbidden"):
|
with subtest("Check that impure expressions are forbidden"):
|
||||||
deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix")
|
deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix")
|
||||||
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
|
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
|
||||||
assert re.search(r"access to absolute path.*forbidden in pure eval mode", logs)
|
assert re.search(r"access to absolute path.*forbidden in pure (eval|evaluation) mode", logs), "Expected error message not found in log"
|
||||||
|
|
||||||
with subtest("Check that impure expressions can be allowed with --impure"):
|
with subtest("Check that impure expressions can be allowed with --impure"):
|
||||||
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator} --impure")
|
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags} --impure")
|
||||||
alpha.succeed("grep deployer /etc/deployment")
|
alpha.succeed("grep deployer /etc/deployment")
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
|
@ -3,13 +3,15 @@
|
||||||
|
|
||||||
inputs = {
|
inputs = {
|
||||||
nixpkgs.url = "@nixpkgs@";
|
nixpkgs.url = "@nixpkgs@";
|
||||||
|
colmena.url = "@colmena@";
|
||||||
};
|
};
|
||||||
|
|
||||||
outputs = { self, nixpkgs }: let
|
outputs = { self, nixpkgs, colmena }: let
|
||||||
pkgs = import nixpkgs {
|
pkgs = import nixpkgs {
|
||||||
system = "x86_64-linux";
|
system = "x86_64-linux";
|
||||||
};
|
};
|
||||||
in {
|
in {
|
||||||
colmena = import ./hive.nix { inherit pkgs; };
|
colmena = import ./hive.nix { inherit pkgs; };
|
||||||
|
colmenaHive = colmena.lib.makeHive self.outputs.colmena;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -129,9 +129,6 @@ let
|
||||||
extraDeployerConfig
|
extraDeployerConfig
|
||||||
];
|
];
|
||||||
|
|
||||||
# FIXME: Colmena flake support is broken with Nix 2.24
|
|
||||||
nix.package = pkgs.nixVersions.nix_2_18;
|
|
||||||
|
|
||||||
nix.registry = lib.mkIf (pkgs ? _inputs) {
|
nix.registry = lib.mkIf (pkgs ? _inputs) {
|
||||||
nixpkgs.flake = pkgs._inputs.nixpkgs;
|
nixpkgs.flake = pkgs._inputs.nixpkgs;
|
||||||
};
|
};
|
||||||
|
@ -168,6 +165,9 @@ let
|
||||||
exec "$@" 2> >(tee /dev/stderr)
|
exec "$@" 2> >(tee /dev/stderr)
|
||||||
'')
|
'')
|
||||||
];
|
];
|
||||||
|
|
||||||
|
# Re-enable switch-to-configuration
|
||||||
|
system.switch.enable = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
# Setup for target nodes
|
# Setup for target nodes
|
||||||
|
@ -183,6 +183,9 @@ let
|
||||||
sshKeys.snakeOilPublicKey
|
sshKeys.snakeOilPublicKey
|
||||||
];
|
];
|
||||||
virtualisation.writableStore = true;
|
virtualisation.writableStore = true;
|
||||||
|
|
||||||
|
# Re-enable switch-to-configuration
|
||||||
|
system.switch.enable = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
nodes = let
|
nodes = let
|
||||||
|
|
|
@ -90,6 +90,34 @@ To build and deploy to all nodes:
|
||||||
colmena apply
|
colmena apply
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Direct Flake Evaluation (Experimental)
|
||||||
|
|
||||||
|
By default, Colmena uses `nix-instantiate` to evaluate your flake which does not work purely on Nix 2.21+, necessitating the use of `--impure`.
|
||||||
|
There is experimental support for evaluating flakes directly with `nix eval`, enabled via `--experimental-flake-eval`.
|
||||||
|
|
||||||
|
To use this new evaluation mode, your flake needs to depend on Colmena itself as an input and expose a new output called `colmenaHive`:
|
||||||
|
|
||||||
|
```diff
|
||||||
|
{
|
||||||
|
inputs = {
|
||||||
|
+ # ADDED: Colmena input
|
||||||
|
+ colmena.url = "github:zhaofengli/colmena";
|
||||||
|
|
||||||
|
# ... Rest of configuration ...
|
||||||
|
};
|
||||||
|
outputs = { self, colmena, ... }: {
|
||||||
|
+ # ADDED: New colmenaHive output
|
||||||
|
+ colmenaHive = colmena.lib.makeHive self.outputs.colmena;
|
||||||
|
|
||||||
|
# Your existing colmena output
|
||||||
|
colmena = {
|
||||||
|
# ... Rest of configuration ...
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
## Next Steps
|
## Next Steps
|
||||||
|
|
||||||
- Head to the [Features](../features/index.md) section to see what else Colmena can do.
|
- Head to the [Features](../features/index.md) section to see what else Colmena can do.
|
||||||
|
|
|
@ -5,7 +5,12 @@ rustPlatform.buildRustPackage rec {
|
||||||
version = "0.5.0-pre";
|
version = "0.5.0-pre";
|
||||||
|
|
||||||
src = lib.cleanSourceWith {
|
src = lib.cleanSourceWith {
|
||||||
filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) [ "target" "manual" "integration-tests" ]);
|
filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) [
|
||||||
|
".github"
|
||||||
|
"target"
|
||||||
|
"manual"
|
||||||
|
"integration-tests"
|
||||||
|
]);
|
||||||
src = lib.cleanSource ./.;
|
src = lib.cleanSource ./.;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
22
src/cli.rs
22
src/cli.rs
|
@ -10,7 +10,7 @@ use env_logger::fmt::WriteStyle;
|
||||||
use crate::{
|
use crate::{
|
||||||
command::{self, apply::DeployOpts},
|
command::{self, apply::DeployOpts},
|
||||||
error::ColmenaResult,
|
error::ColmenaResult,
|
||||||
nix::{Hive, HivePath},
|
nix::{hive::EvaluationMethod, Hive, HivePath},
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Base URL of the manual, without the trailing slash.
|
/// Base URL of the manual, without the trailing slash.
|
||||||
|
@ -137,6 +137,21 @@ This only works when building locally.
|
||||||
value_names = ["NAME", "VALUE"],
|
value_names = ["NAME", "VALUE"],
|
||||||
)]
|
)]
|
||||||
nix_option: Vec<String>,
|
nix_option: Vec<String>,
|
||||||
|
#[arg(
|
||||||
|
long,
|
||||||
|
default_value_t,
|
||||||
|
help = "Use direct flake evaluation (experimental)",
|
||||||
|
long_help = r#"If enabled, flakes will be evaluated using `nix eval`. This requires the flake to depend on Colmena as an input and expose a compatible `colmenaHive` output:
|
||||||
|
|
||||||
|
outputs = { self, colmena, ... }: {
|
||||||
|
colmenaHive = colmena.lib.makeHive self.outputs.colmena;
|
||||||
|
colmena = ...;
|
||||||
|
};
|
||||||
|
|
||||||
|
This is an experimental feature."#,
|
||||||
|
global = true
|
||||||
|
)]
|
||||||
|
experimental_flake_eval: bool,
|
||||||
#[arg(
|
#[arg(
|
||||||
long,
|
long,
|
||||||
value_name = "WHEN",
|
value_name = "WHEN",
|
||||||
|
@ -262,6 +277,11 @@ async fn get_hive(opts: &Opts) -> ColmenaResult<Hive> {
|
||||||
hive.set_impure(true);
|
hive.set_impure(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if opts.experimental_flake_eval {
|
||||||
|
log::warn!("Using direct flake evaluation (experimental)");
|
||||||
|
hive.set_evaluation_method(EvaluationMethod::DirectFlakeEval);
|
||||||
|
}
|
||||||
|
|
||||||
for chunks in opts.nix_option.chunks_exact(2) {
|
for chunks in opts.nix_option.chunks_exact(2) {
|
||||||
let [name, value] = chunks else {
|
let [name, value] = chunks else {
|
||||||
unreachable!()
|
unreachable!()
|
||||||
|
|
|
@ -181,7 +181,7 @@ let
|
||||||
|
|
||||||
in rec {
|
in rec {
|
||||||
# Exported attributes
|
# Exported attributes
|
||||||
__schema = "v0";
|
__schema = "v0.20241006";
|
||||||
|
|
||||||
nodes = listToAttrs (map (name: { inherit name; value = evalNode name (configsFor name); }) nodeNames);
|
nodes = listToAttrs (map (name: { inherit name; value = evalNode name (configsFor name); }) nodeNames);
|
||||||
toplevel = lib.mapAttrs (_: v: v.config.system.build.toplevel) nodes;
|
toplevel = lib.mapAttrs (_: v: v.config.system.build.toplevel) nodes;
|
||||||
|
|
|
@ -7,7 +7,7 @@
|
||||||
|
|
||||||
outputs = { self, hive }: {
|
outputs = { self, hive }: {
|
||||||
processFlake = let
|
processFlake = let
|
||||||
compatibleSchema = "v0";
|
compatibleSchema = "v0.20241006";
|
||||||
|
|
||||||
# Evaluates a raw hive.
|
# Evaluates a raw hive.
|
||||||
#
|
#
|
||||||
|
|
|
@ -8,6 +8,7 @@ use std::convert::AsRef;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
|
|
||||||
|
use const_format::formatcp;
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
use tokio::sync::OnceCell;
|
use tokio::sync::OnceCell;
|
||||||
use validator::Validate;
|
use validator::Validate;
|
||||||
|
@ -22,6 +23,21 @@ use crate::job::JobHandle;
|
||||||
use crate::util::{CommandExecution, CommandExt};
|
use crate::util::{CommandExecution, CommandExt};
|
||||||
use assets::Assets;
|
use assets::Assets;
|
||||||
|
|
||||||
|
/// The version of the Hive schema we are compatible with.
|
||||||
|
///
|
||||||
|
/// Currently we are tied to one specific version.
|
||||||
|
const HIVE_SCHEMA: &str = "v0.20241006";
|
||||||
|
|
||||||
|
/// The snippet to be used for `nix eval --apply`.
|
||||||
|
const FLAKE_APPLY_SNIPPET: &str = formatcp!(
|
||||||
|
r#"with builtins; hive: assert (hive.__schema == "{}" || throw ''
|
||||||
|
The colmenaHive output (schema ${{hive.__schema}}) isn't compatible with this version of Colmena.
|
||||||
|
|
||||||
|
Hint: Use the same version of Colmena as in the Flake input.
|
||||||
|
''); "#,
|
||||||
|
HIVE_SCHEMA
|
||||||
|
);
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub enum HivePath {
|
pub enum HivePath {
|
||||||
/// A Nix Flake.
|
/// A Nix Flake.
|
||||||
|
@ -63,11 +79,33 @@ impl FromStr for HivePath {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||||
|
pub enum EvaluationMethod {
|
||||||
|
/// Use nix-instantiate and specify the entire Nix expression.
|
||||||
|
///
|
||||||
|
/// This is the default method.
|
||||||
|
///
|
||||||
|
/// For flakes, we use `builtins.getFlakes`. Pure evaluation no longer works
|
||||||
|
/// with this method in Nix 2.21+.
|
||||||
|
NixInstantiate,
|
||||||
|
|
||||||
|
/// Use `nix eval --apply` on top of a flake.
|
||||||
|
///
|
||||||
|
/// This can be activated with --experimental-flake-eval.
|
||||||
|
///
|
||||||
|
/// In this method, we can no longer pull in our bundled assets and
|
||||||
|
/// the flake must expose a compatible `colmenaHive` output.
|
||||||
|
DirectFlakeEval,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct Hive {
|
pub struct Hive {
|
||||||
/// Path to the hive.
|
/// Path to the hive.
|
||||||
path: HivePath,
|
path: HivePath,
|
||||||
|
|
||||||
|
/// Method to evaluate the hive with.
|
||||||
|
evaluation_method: EvaluationMethod,
|
||||||
|
|
||||||
/// Path to the context directory.
|
/// Path to the context directory.
|
||||||
///
|
///
|
||||||
/// Normally this is directory containing the "hive.nix"
|
/// Normally this is directory containing the "hive.nix"
|
||||||
|
@ -134,6 +172,7 @@ impl Hive {
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
path,
|
path,
|
||||||
|
evaluation_method: EvaluationMethod::NixInstantiate,
|
||||||
context_dir,
|
context_dir,
|
||||||
assets,
|
assets,
|
||||||
show_trace: false,
|
show_trace: false,
|
||||||
|
@ -158,6 +197,14 @@ impl Hive {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn set_evaluation_method(&mut self, method: EvaluationMethod) {
|
||||||
|
if !self.is_flake() && method == EvaluationMethod::DirectFlakeEval {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.evaluation_method = method;
|
||||||
|
}
|
||||||
|
|
||||||
pub fn set_show_trace(&mut self, value: bool) {
|
pub fn set_show_trace(&mut self, value: bool) {
|
||||||
self.show_trace = value;
|
self.show_trace = value;
|
||||||
}
|
}
|
||||||
|
@ -421,7 +468,10 @@ impl Hive {
|
||||||
|
|
||||||
/// Returns the base expression from which the evaluated Hive can be used.
|
/// Returns the base expression from which the evaluated Hive can be used.
|
||||||
fn get_base_expression(&self) -> String {
|
fn get_base_expression(&self) -> String {
|
||||||
self.assets.get_base_expression()
|
match self.evaluation_method {
|
||||||
|
EvaluationMethod::NixInstantiate => self.assets.get_base_expression(),
|
||||||
|
EvaluationMethod::DirectFlakeEval => FLAKE_APPLY_SNIPPET.to_string(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns whether this Hive is a flake.
|
/// Returns whether this Hive is a flake.
|
||||||
|
@ -444,6 +494,11 @@ impl<'hive> NixInstantiate<'hive> {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn instantiate(&self) -> Command {
|
fn instantiate(&self) -> Command {
|
||||||
|
// TODO: Better error handling
|
||||||
|
if self.hive.evaluation_method == EvaluationMethod::DirectFlakeEval {
|
||||||
|
panic!("Instantiation is not supported with DirectFlakeEval");
|
||||||
|
}
|
||||||
|
|
||||||
let mut command = Command::new("nix-instantiate");
|
let mut command = Command::new("nix-instantiate");
|
||||||
|
|
||||||
if self.hive.is_flake() {
|
if self.hive.is_flake() {
|
||||||
|
@ -462,17 +517,48 @@ impl<'hive> NixInstantiate<'hive> {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn eval(self) -> Command {
|
fn eval(self) -> Command {
|
||||||
let mut command = self.instantiate();
|
|
||||||
let flags = self.hive.nix_flags();
|
let flags = self.hive.nix_flags();
|
||||||
command
|
|
||||||
.arg("--eval")
|
match self.hive.evaluation_method {
|
||||||
.arg("--json")
|
EvaluationMethod::NixInstantiate => {
|
||||||
.arg("--strict")
|
let mut command = self.instantiate();
|
||||||
// Ensures the derivations are instantiated
|
|
||||||
// Required for system profile evaluation and IFD
|
command
|
||||||
.arg("--read-write-mode")
|
.arg("--eval")
|
||||||
.args(flags.to_args());
|
.arg("--json")
|
||||||
command
|
.arg("--strict")
|
||||||
|
// Ensures the derivations are instantiated
|
||||||
|
// Required for system profile evaluation and IFD
|
||||||
|
.arg("--read-write-mode")
|
||||||
|
.args(flags.to_args());
|
||||||
|
|
||||||
|
command
|
||||||
|
}
|
||||||
|
EvaluationMethod::DirectFlakeEval => {
|
||||||
|
let mut command = Command::new("nix");
|
||||||
|
let flake = if let HivePath::Flake(flake) = self.hive.path() {
|
||||||
|
flake
|
||||||
|
} else {
|
||||||
|
panic!("The DirectFlakeEval evaluation method only support flakes");
|
||||||
|
};
|
||||||
|
|
||||||
|
let hive_installable = format!("{}#colmenaHive", flake.uri());
|
||||||
|
|
||||||
|
let mut full_expression = self.hive.get_base_expression();
|
||||||
|
full_expression += &self.expression;
|
||||||
|
|
||||||
|
command
|
||||||
|
.arg("eval") // nix eval
|
||||||
|
.args(["--extra-experimental-features", "flakes nix-command"])
|
||||||
|
.arg(hive_installable)
|
||||||
|
.arg("--json")
|
||||||
|
.arg("--apply")
|
||||||
|
.arg(&full_expression)
|
||||||
|
.args(flags.to_args());
|
||||||
|
|
||||||
|
command
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn instantiate_with_builders(self) -> ColmenaResult<Command> {
|
async fn instantiate_with_builders(self) -> ColmenaResult<Command> {
|
||||||
|
|
Loading…
Reference in a new issue