Add integration tests

default.nix

@@ -16,7 +16,7 @@ in rustPlatform.buildRustPackage rec {
   apiVersion = builtins.concatStringsSep "." (lib.take 2 (lib.splitString "." version));
 
   src = lib.cleanSourceWith {
-    filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) [ "target" "manual" ]);
+    filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) [ "target" "manual" "integration-tests" ]);
     src = lib.cleanSource ./.;
   };
 

integration-tests/apply-local/default.nix

@@ -0,0 +1,12 @@
+let
+  tools = import ../tools.nix {};
+in tools.makeTest {
+  name = "colmena-exec";
+
+  bundle = ./.;
+
+  testScript = ''
+    deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply-local -v")
+    deployer.succeed("grep SUCCESS /etc/deployment")
+  '';
+}

integration-tests/apply-local/hive.nix

@@ -0,0 +1,23 @@
+let
+  tools = import ./tools.nix { insideVm = true; };
+in {
+  meta = {
+    nixpkgs = tools.pkgs;
+  };
+
+  deployer = { lib, ... }: {
+    imports = [
+      (tools.getStandaloneConfigFor "deployer")
+    ];
+
+    deployment = {
+      allowLocalDeployment = true;
+    };
+
+    environment.etc."deployment".text = "SUCCESS";
+  };
+
+  alpha = tools.getStandaloneConfigFor "alpha";
+  beta = tools.getStandaloneConfigFor "beta";
+  gamma = tools.getStandaloneConfigFor "gamma";
+}

integration-tests/apply/default.nix

@@ -0,0 +1,27 @@
+let
+  tools = import ../tools.nix {};
+in tools.makeTest {
+  name = "colmena-apply";
+
+  bundle = ./.;
+
+  testScript = ''
+    logs = deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply -v --on @target 2> >(tee /dev/stderr)")
+
+    with subtest("Check whether build messages are logged correctly"):
+        assert "must appear during build" in logs
+
+    with subtest("Check whether push messages are logged correctly"):
+        assert "copying path" in logs
+
+    with subtest("Check whether activation messages are logged correctly"):
+        assert "must appear during activation" in logs
+
+    alpha.succeed("grep SUCCESS /etc/deployment")
+    alpha.succeed("grep 'key content' /run/keys/example")
+
+    deployer.succeed("ssh alpha true")
+    deployer.succeed("ssh beta true")
+    deployer.succeed("ssh gamma true")
+  '';
+}

integration-tests/apply/hive.nix

@@ -0,0 +1,33 @@
+let
+  tools = import ./tools.nix { insideVm = true; };
+
+  testPkg = tools.pkgs.runCommand "test-package" {} ''
+    echo "must appear during build"
+    mkdir -p $out
+  '';
+in {
+  meta = {
+    nixpkgs = tools.pkgs;
+  };
+
+  alpha = { lib, ... }: {
+    imports = [
+      (tools.getStandaloneConfigFor "alpha")
+    ];
+
+    environment.systemPackages = [ testPkg ];
+    environment.etc."deployment".text = "SUCCESS";
+
+    system.activationScripts.colmena-test.text = ''
+      echo "must appear during activation"
+    '';
+
+    deployment.keys.example.text = ''
+      key content
+    '';
+  };
+
+  deployer = tools.getStandaloneConfigFor "deployer";
+  beta = tools.getStandaloneConfigFor "beta";
+  gamma = tools.getStandaloneConfigFor "gamma";
+}

integration-tests/exec/default.nix

@@ -0,0 +1,15 @@
+let
+  tools = import ../tools.nix {};
+in tools.makeTest {
+  name = "colmena-exec";
+
+  bundle = ./.;
+
+  testScript = ''
+    logs = deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} exec -v --on @target -- echo output from '$(hostname)' 2>&1")
+
+    assert "output from alpha" in logs
+    assert "output from beta" in logs
+    assert "output from gamma" in logs
+  '';
+}

integration-tests/exec/hive.nix

@@ -0,0 +1,12 @@
+let
+  tools = import ./tools.nix { insideVm = true; };
+in {
+  meta = {
+    nixpkgs = tools.pkgs;
+  };
+
+  deployer = tools.getStandaloneConfigFor "deployer";
+  alpha = tools.getStandaloneConfigFor "alpha";
+  beta = tools.getStandaloneConfigFor "beta";
+  gamma = tools.getStandaloneConfigFor "gamma";
+}

integration-tests/tools.nix

@@ -0,0 +1,137 @@
+# Adapted from the NixOps test in Nixpkgs.
+#
+# We have four nodes: deployer, alpha, beta, gamma.
+# deployer is where colmena will run.
+#
+# `nixos/lib/build-vms.nix` will generate NixOS configurations
+# for each node, and we need to include those configurations
+# in our Colmena setup as well.
+
+{ insideVm ? false }:
+
+let
+  lock = builtins.fromJSON (builtins.readFile ../flake.lock);
+  pinned = if insideVm then <nixpkgs> else fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${lock.nodes.nixpkgs.locked.rev}.tar.gz";
+    sha256 = lock.nodes.nixpkgs.locked.narHash;
+  };
+  pkgs = import pinned {};
+
+  colmena =
+    if !insideVm then import ../default.nix { inherit pkgs; }
+    else throw "Cannot be used inside the VM";
+  colmenaExec = "${colmena}/bin/colmena";
+
+  sshKeys = import (pkgs.path + "/nixos/tests/ssh-keys.nix") pkgs;
+  buildVms = import (pkgs.path + "/nixos/lib/build-vms.nix") {
+    inherit (pkgs) system pkgs lib;
+  };
+
+  # Common setup
+  nodes = let
+    deployer = { lib, config, ... }: {
+      nix.nixPath = [
+        "nixpkgs=${pkgs.path}"
+      ];
+
+      nix.binaryCaches = lib.mkForce [];
+
+      virtualisation = {
+        memorySize = 1024;
+        writableStore = true;
+        additionalPaths = [
+          "${pkgs.path}"
+          prebuiltNode
+          (inputClosureOf prebuiltNode)
+        ];
+      };
+    };
+    target = {
+      services.openssh.enable = true;
+      users.users.root.openssh.authorizedKeys.keys = [
+        sshKeys.snakeOilPublicKey
+      ];
+      virtualisation.writableStore = true;
+    };
+  in {
+    inherit deployer;
+    alpha = target;
+    beta = target;
+    gamma = target;
+  };
+
+  prebuiltNode = let
+    all = buildVms.buildVirtualNetwork nodes;
+  in all.alpha.config.system.build.toplevel;
+
+  # Utilities
+  getStandaloneConfigFor = node: let
+    configsWithIp = buildVms.assignIPAddresses nodes;
+  in { modulesPath, lib, config, ... }: {
+    imports = configsWithIp.${node} ++ [
+      (modulesPath + "/virtualisation/qemu-vm.nix")
+      (modulesPath + "/testing/test-instrumentation.nix")
+    ];
+
+    documentation.nixos.enable = false;
+    boot.loader.grub.enable = false;
+    system.nixos.revision = lib.mkForce "constant-nixos-revision";
+
+    # otherwise the evaluation is unnecessarily slow in VM
+    virtualisation.additionalPaths = lib.mkForce [];
+    nix.nixPath = lib.mkForce [ "nixpkgs=/nixpkgs" ];
+
+    deployment.tags = lib.optional (config.networking.hostName != "deployer") "target";
+  };
+
+  inputClosureOf = pkg: pkgs.runCommand "full-closure" {
+    refs = pkgs.writeReferencesToFile pkg.drvPath;
+  } ''
+    touch $out
+
+    while read ref; do
+      case $ref in
+        *.drv)
+          cat $ref >>$out
+          ;;
+      esac
+    done <$refs
+  '';
+
+  makeTest = test: let
+    fullScript = ''
+      start_all()
+
+      deployer.succeed("nix-store -qR ${prebuiltNode}")
+      deployer.succeed("nix-store -qR ${pkgs.path}")
+      deployer.succeed("ln -sf ${pkgs.path} /nixpkgs")
+      deployer.succeed("mkdir -p /root/.ssh && touch /root/.ssh/id_rsa && chmod 600 /root/.ssh/id_rsa && cat ${sshKeys.snakeOilPrivateKey} > /root/.ssh/id_rsa")
+
+      for node in [alpha, beta, gamma]:
+          node.wait_for_unit("sshd.service")
+      deployer.succeed("ssh -o StrictHostKeyChecking=accept-new alpha ls")
+
+      deployer.succeed("cp --no-preserve=mode -r ${bundle} /tmp/bundle && chmod u+w /tmp/bundle")
+    '' + test.testScript;
+
+    bundle = pkgs.stdenv.mkDerivation {
+      name = "${test.name}-bundle";
+      dontUnpack = true;
+      dontInstall = true;
+      buildPhase = ''
+        cp -r ${test.bundle} $out
+        chmod u+w $out
+        cp ${./tools.nix} $out/tools.nix
+      '';
+    };
+
+    combined = {
+      inherit nodes;
+    } // test // {
+      testScript = fullScript;
+    };
+  in pkgs.nixosTest combined;
+in {
+  inherit pkgs nodes colmena colmenaExec prebuiltNode
+    getStandaloneConfigFor inputClosureOf makeTest;
+}