From 6285cd8dbfb77c287fc5b30263bbfd5770b47413 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Tue, 13 Aug 2019 00:29:08 +0100 Subject: [PATCH] feat(build-image): Use new image layering algorithm for images Removes usage of the old layering algorithm and replaces it with the new one. Apart from the new layer layout this means that each layer is now built in a separate derivation, which hopefully leads to better cacheability. --- tools/nixery/build-image/build-image.nix | 85 +++++++++++++----------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/tools/nixery/build-image/build-image.nix b/tools/nixery/build-image/build-image.nix index e5b195a63..d68ed6d37 100644 --- a/tools/nixery/build-image/build-image.nix +++ b/tools/nixery/build-image/build-image.nix @@ -54,12 +54,12 @@ ... }: -let +with builtins; let # If a nixpkgs channel is requested, it is retrieved from Github (as # a tarball) and imported. fetchImportChannel = channel: let url = "https://github.com/NixOS/nixpkgs-channels/archive/${channel}.tar.gz"; - in import (builtins.fetchTarball url) {}; + in import (fetchTarball url) {}; # If a git repository is requested, it is retrieved via # builtins.fetchGit which defaults to the git configuration of the @@ -76,32 +76,31 @@ let # There are some additional caveats around whether the default # branch contains the specified revision, which need to be # explained to users. - spec = if (builtins.stringLength rev) == 40 then { + spec = if (stringLength rev) == 40 then { inherit url rev; } else { inherit url; ref = rev; }; - in import (builtins.fetchGit spec) {}; + in import (fetchGit spec) {}; - importPath = path: import (builtins.toPath path) {}; + importPath = path: import (toPath path) {}; - source = builtins.split "!" pkgSource; - sourceType = builtins.elemAt source 0; - pkgs = with builtins; + source = split "!" pkgSource; + sourceType = elemAt source 0; + pkgs = if sourceType == "nixpkgs" then fetchImportChannel (elemAt source 2) else if sourceType == "git" then fetchImportGit (elemAt source 2) (elemAt source 4) else if sourceType == "path" then importPath (elemAt source 2) - else builtins.throw("Invalid package set source specification: ${pkgSource}"); + else throw("Invalid package set source specification: ${pkgSource}"); in # Since this is essentially a re-wrapping of some of the functionality that is # implemented in the dockerTools, we need all of its components in our top-level # namespace. -with builtins; with pkgs; with dockerTools; @@ -168,6 +167,11 @@ let paths = allContents.contents; }; + popularity = builtins.fetchurl { + url = "https://storage.googleapis.com/nixery-layers/popularity/nixos-19.03-20190812.json"; + sha256 = "16sxd49vqqg2nrhwynm36ba6bc2yff5cd5hf83wi0hanw5sx3svk"; + }; + # Before actually creating any image layers, the store paths that need to be # included in the image must be sorted into the layers that they should go # into. @@ -181,31 +185,37 @@ let # To invoke the program, a graph of all runtime references is created via # Nix's exportReferencesGraph feature - the resulting layers are read back # into Nix using import-from-derivation. - groupedLayers = runCommand "grouped-layers.json" { - buildInputs = [ groupLayers ]; + groupedLayers = fromJSON (readFile (runCommand "grouped-layers.json" { + __structuredAttrs = true; + exportReferencesGraph.graph = allContents.contents; + PATH = "${groupLayers}/bin"; + builder = toFile "builder" '' + . .attrs.sh + group-layers --budget ${toString (maxLayers - 1)} --pop ${popularity} --out ''${outputs[out]} + ''; + } "")); + + # Given a list of store paths, create an image layer tarball with + # their contents. + pathsToLayer = paths: runCommand "layer.tar" { } '' - group-layers --fnorg + tar --no-recursion -rf "$out" \ + --mtime="@$SOURCE_DATE_EPOCH" \ + --owner=0 --group=0 /nix /nix/store + + tar -rpf "$out" --hard-dereference --sort=name \ + --mtime="@$SOURCE_DATE_EPOCH" \ + --owner=0 --group=0 ${lib.concatStringsSep " " paths} ''; - # The image build infrastructure expects to be outputting a slightly different - # format than the one we serve over the registry protocol. To work around its - # expectations we need to provide an empty JSON file that it can write some - # fun data into. - emptyJson = writeText "empty.json" "{}"; - - bulkLayers = mkManyPureLayers { - name = baseName; - configJson = emptyJson; - closure = writeText "closure" "${contentsEnv} ${emptyJson}"; - # One layer will be taken up by the customisationLayer, so - # take up one less. - maxLayers = maxLayers - 1; - }; + bulkLayers = writeText "bulk-layers" + (lib.concatStringsSep "\n" (map (layer: pathsToLayer layer.contents) + groupedLayers)); customisationLayer = mkCustomisationLayer { name = baseName; contents = contentsEnv; - baseJson = emptyJson; + baseJson = writeText "empty.json" "{}"; inherit uid gid extraCommands; }; @@ -214,23 +224,22 @@ let # # This computes both an MD5 and a SHA256 hash of each layer, which are used # for different purposes. See the registry server implementation for details. - # - # Some of this logic is copied straight from `buildLayeredImage`. allLayersJson = runCommand "fs-layer-list.json" { buildInputs = [ coreutils findutils jq openssl ]; } '' - find ${bulkLayers} -mindepth 1 -maxdepth 1 | sort -t/ -k5 -n > layer-list - echo ${customisationLayer} >> layer-list + cat ${bulkLayers} | sort -t/ -k5 -n > layer-list + echo -n layer-list: + cat layer-list + echo ${customisationLayer}/layer.tar >> layer-list for layer in $(cat layer-list); do - layerPath="$layer/layer.tar" - layerSha256=$(sha256sum $layerPath | cut -d ' ' -f1) + layerSha256=$(sha256sum $layer | cut -d ' ' -f1) # The server application compares binary MD5 hashes and expects base64 # encoding instead of hex. - layerMd5=$(openssl dgst -md5 -binary $layerPath | openssl enc -base64) - layerSize=$(wc -c $layerPath | cut -d ' ' -f1) + layerMd5=$(openssl dgst -md5 -binary $layer | openssl enc -base64) + layerSize=$(wc -c $layer | cut -d ' ' -f1) - jq -n -c --arg sha256 $layerSha256 --arg md5 $layerMd5 --arg size $layerSize --arg path $layerPath \ + jq -n -c --arg sha256 $layerSha256 --arg md5 $layerMd5 --arg size $layerSize --arg path $layer \ '{ size: ($size | tonumber), sha256: $sha256, md5: $md5, path: $path }' >> fs-layers done @@ -309,6 +318,6 @@ let pkgs = map (err: err.pkg) allContents.errors; }; in writeText "manifest-output.json" (if (length allContents.errors) == 0 - then toJSON groupedLayers # manifestOutput + then toJSON manifestOutput else toJSON errorOutput )