diff --git a/flake.lock b/flake.lock
index 6208d49..bd74e06 100644
--- a/flake.lock
+++ b/flake.lock
@@ -16,6 +16,43 @@
         "type": "github"
       }
     },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1659877975,
+        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nix-eval-jobs": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1660713017,
+        "narHash": "sha256-w6KLNkT/2fI7UvPg3NoKj7gve4CgUHpyWOFq0BUwW7o=",
+        "owner": "zhaofengli",
+        "repo": "nix-eval-jobs",
+        "rev": "b4c078a4870c5b1a5b74420ca1dca41f4d75b7cc",
+        "type": "github"
+      },
+      "original": {
+        "owner": "zhaofengli",
+        "ref": "colmena",
+        "repo": "nix-eval-jobs",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1660485612,
@@ -35,6 +72,7 @@
     "root": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "nix-eval-jobs": "nix-eval-jobs",
         "nixpkgs": "nixpkgs",
         "stable": "stable",
         "utils": "utils"
diff --git a/flake.nix b/flake.nix
index 24faa54..30dedcd 100644
--- a/flake.nix
+++ b/flake.nix
@@ -7,19 +7,24 @@
 
     utils.url = "github:numtide/flake-utils";
 
+    # Temporary fork of nix-eval-job with changes to be upstreamed
+    nix-eval-jobs.url = "github:zhaofengli/nix-eval-jobs/colmena";
+    nix-eval-jobs.inputs.nixpkgs.follows = "nixpkgs";
+
     flake-compat = {
       url = "github:edolstra/flake-compat";
       flake = false;
     };
   };
 
-  outputs = { self, nixpkgs, utils, ... }: let
+  outputs = { self, nixpkgs, utils, nix-eval-jobs, ... }: let
     supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
     colmenaOptions = import ./src/nix/hive/options.nix;
     colmenaModules = import ./src/nix/hive/modules.nix;
   in utils.lib.eachSystem supportedSystems (system: let
     pkgs = import nixpkgs {
       inherit system;
+      overlays = [ self._evalJobsOverlay ];
     };
   in rec {
     # We still maintain the expression in a Nixpkgs-acceptable form
@@ -79,6 +84,15 @@
       ];
     };
   }) // {
+    # Temporary fork of nix-eval-job with changes to be upstreamed
+    _evalJobsOverlay = final: prev: let
+      patched = nix-eval-jobs.packages.${final.system}.nix-eval-jobs.overrideAttrs (old: {
+        version = "2.9.0-colmena";
+      });
+    in {
+      nix-eval-jobs = patched;
+    };
+
     overlay = final: prev: {
       colmena = final.callPackage ./package.nix { };
     };
diff --git a/integration-tests/nixpkgs-stable.nix b/integration-tests/nixpkgs-stable.nix
index da06033..a23282b 100644
--- a/integration-tests/nixpkgs-stable.nix
+++ b/integration-tests/nixpkgs-stable.nix
@@ -2,6 +2,7 @@ let
   flake = (import ../flake-compat.nix).defaultNix;
 in import flake.inputs.stable.outPath {
   overlays = [
+    flake._evalJobsOverlay
     flake.overlay
   ];
 }
diff --git a/integration-tests/nixpkgs.nix b/integration-tests/nixpkgs.nix
index fd4b9c0..92cce60 100644
--- a/integration-tests/nixpkgs.nix
+++ b/integration-tests/nixpkgs.nix
@@ -2,6 +2,7 @@ let
   flake = (import ../flake-compat.nix).defaultNix;
 in import flake.inputs.nixpkgs.outPath {
   overlays = [
+    flake._evalJobsOverlay
     flake.overlay
   ];
 }