refactor(3p/nixpkgs): Pin nixpkgs sources via niv

There's no need for us to reinvent the wheel here.

niv pins are in //third_party/sources, and niv commands need to be run
with `niv -s third_party/sources/sources.json` to operate on the
correct file.

Note that niv by default wants to put the loader file in
//nix/sources.nix. This file has been moved to
//third_party/sources/default.nix which makes niv throw a warning, but
everything still works as intended.

Change-Id: I2b2a6f8edf33c429a6d7be9d174ba1996d9a0193
Reviewed-on: https://cl.tvl.fyi/c/depot/+/5143
Reviewed-by: grfn <grfn@gws.fyi>
Reviewed-by: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI
Autosubmit: tazjin <tazjin@tvl.su>

third_party/nixpkgs/default.nix

@@ -1,6 +1,8 @@
 # This file imports the pinned nixpkgs sets and applies relevant
 # modifications, such as our overlays.
 #
+# The actual source pinning happens via niv in //third_party/sources
+#
 # Note that the attribute exposed by this (third_party.nixpkgs) is
 # "special" in that the fixpoint used as readTree's config parameter
 # in //default.nix passes this attribute as the `pkgs` argument to all
@@ -9,40 +11,16 @@
 { depot ? { }, externalArgs ? { }, depotOverlays ? true, ... }:
 
 let
-  # This provides the sources of nixpkgs. We track both
-  # nixos-unstable, and the current stable channel of the latest NixOS
-  # release.
-
-  # Tracking nixos-unstable as of 2022-01-27.
-  unstableHashes = {
-    commit = "945ec499041db73043f745fad3b2a3a01e826081";
-    sha256 = "1ixv310sjw0r5vda4yfwp3snyha2i9h7aqygd43cyvdk2qsjk8pq";
-  };
-
-  # Tracking nixos-21.11 as of 2022-01-26.
-  stableHashes = {
-    commit = "b3d86c56c786ad9530f1400adbd4dfac3c42877b";
-    sha256 = "09nslcjdgwwb6j9alxrsnq1wvhifq1nmzl2w02l305j0wsmgdial";
-  };
-
   # import the nixos-unstable package set, or optionally use the
   # source (e.g. a path) specified by the `nixpkgsBisectPath`
   # argument. This is intended for use-cases where the depot is
   # bisected against nixpkgs to find the root cause of an issue in a
   # channel bump.
-  nixpkgsSrc = externalArgs.nixpkgsBisectPath or (fetchTarball {
+  nixpkgsSrc = externalArgs.nixpkgsBisectPath or depot.third_party.sources.nixpkgs;
-    url = "https://github.com/NixOS/nixpkgs/archive/${unstableHashes.commit}.tar.gz";
-    sha256 = unstableHashes.sha256;
-  });
-
-  stableNixpkgsSrc = fetchTarball {
-    url = "https://github.com/NixOS/nixpkgs/archive/${stableHashes.commit}.tar.gz";
-    sha256 = stableHashes.sha256;
-  };
 
   # Stable package set is imported, but not exposed, to overlay
   # required packages into the unstable set.
-  stableNixpkgs = import stableNixpkgsSrc { };
+  stableNixpkgs = import depot.third_party.sources.nixpkgs-stable {};
 
   # Overlay for packages that should come from the stable channel
   # instead (e.g. because something is broken in unstable).
@@ -53,11 +31,10 @@ let
   # Overlay to expose the nixpkgs commits we are using to other Nix code.
   commitsOverlay = _: _: {
     nixpkgsCommits = {
-      unstable = unstableHashes.commit;
+      unstable = depot.third_party.sources.nixpkgs.rev;
-      stable = stableHashes.commit;
+      stable = depot.third_party.sources.nixpkgs-stable.rev;
     };
   };
-
 in
 import nixpkgsSrc {
   # allow users to inject their config into builds (e.g. to test CA derivations)

third_party/sources/default.nix

@@ -0,0 +1,147 @@
+# This file has been generated by Niv.
+_: let
+
+  #
+  # The fetchers. fetch_<type> fetches specs of type <type>.
+  #
+
+  fetch_file = pkgs: spec:
+    if spec.builtin or true then
+      builtins_fetchurl { inherit (spec) url sha256; }
+    else
+      pkgs.fetchurl { inherit (spec) url sha256; };
+
+  fetch_tarball = pkgs: name: spec:
+    let
+      ok = str: ! builtins.isNull (builtins.match "[a-zA-Z0-9+-._?=]" str);
+      # sanitize the name, though nix will still fail if name starts with period
+      name' = stringAsChars (x: if ! ok x then "-" else x) "${name}-src";
+    in
+      if spec.builtin or true then
+        builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
+      else
+        pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
+
+  fetch_git = spec:
+    builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; };
+
+  fetch_local = spec: spec.path;
+
+  fetch_builtin-tarball = name: throw
+    ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
+        $ niv modify ${name} -a type=tarball -a builtin=true'';
+
+  fetch_builtin-url = name: throw
+    ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
+        $ niv modify ${name} -a type=file -a builtin=true'';
+
+  #
+  # Various helpers
+  #
+
+  # The set of packages used when specs are fetched using non-builtins.
+  mkPkgs = sources:
+    let
+      sourcesNixpkgs =
+        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {};
+      hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
+      hasThisAsNixpkgsPath = <nixpkgs> == ./.;
+    in
+      if builtins.hasAttr "nixpkgs" sources
+      then sourcesNixpkgs
+      else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
+        import <nixpkgs> {}
+      else
+        abort
+          ''
+            Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
+            add a package called "nixpkgs" to your sources.json.
+          '';
+
+  # The actual fetching function.
+  fetch = pkgs: name: spec:
+
+    if ! builtins.hasAttr "type" spec then
+      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
+    else if spec.type == "file" then fetch_file pkgs spec
+    else if spec.type == "tarball" then fetch_tarball pkgs name spec
+    else if spec.type == "git" then fetch_git spec
+    else if spec.type == "local" then fetch_local spec
+    else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
+    else if spec.type == "builtin-url" then fetch_builtin-url name
+    else
+      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
+
+  # If the environment variable NIV_OVERRIDE_${name} is set, then use
+  # the path directly as opposed to the fetched source.
+  replace = name: drv:
+    let
+      saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
+      ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
+    in
+      if ersatz == "" then drv else ersatz;
+
+  # Ports of functions for older nix versions
+
+  # a Nix version of mapAttrs if the built-in doesn't exist
+  mapAttrs = builtins.mapAttrs or (
+    f: set: with builtins;
+    listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
+  );
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
+  range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
+  stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
+  stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
+  concatStrings = builtins.concatStringsSep "";
+
+  # fetchTarball version that is compatible between all the versions of Nix
+  builtins_fetchTarball = { url, name, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchTarball;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchTarball { inherit name url; }
+      else
+        fetchTarball attrs;
+
+  # fetchurl version that is compatible between all the versions of Nix
+  builtins_fetchurl = { url, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchurl;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchurl { inherit url; }
+      else
+        fetchurl attrs;
+
+  # Create the final "sources" from the config
+  mkSources = config:
+    mapAttrs (
+      name: spec:
+        if builtins.hasAttr "outPath" spec
+        then abort
+          "The values in sources.json should not have an 'outPath' attribute"
+        else
+          spec // { outPath = replace name (fetch config.pkgs name spec); }
+    ) config.sources;
+
+  # The "config" used by the fetchers
+  mkConfig =
+    { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
+    , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
+    , pkgs ? mkPkgs sources
+    }: rec {
+      # The sources, i.e. the attribute set of spec name to spec
+      inherit sources;
+
+      # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
+      inherit pkgs;
+    };
+
+in
+mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }

third_party/sources/sources.json

@@ -0,0 +1,26 @@
+{
+    "nixpkgs": {
+        "branch": "nixos-unstable",
+        "description": "Nix Packages collection",
+        "homepage": "",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "efeefb2af1469a5d1f0ae7ca8f0dfd9bb87d5cfb",
+        "sha256": "1b3sxslv5id61phq7zx3lybw72x29bx9595i8m708fax7iml07j2",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/efeefb2af1469a5d1f0ae7ca8f0dfd9bb87d5cfb.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgs-stable": {
+        "branch": "nixos-21.11",
+        "description": "Nix Packages collection",
+        "homepage": "",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "0f316e4d72daed659233817ffe52bf08e081b5de",
+        "sha256": "0vh0fk5is5s9l0lxpi16aabv2kk1fwklr7szy731kfcz9gdrr65l",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/0f316e4d72daed659233817ffe52bf08e081b5de.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    }
+}