feat(build-image): Use new image layering algorithm for images

Removes usage of the old layering algorithm and replaces it with the
new one.

Apart from the new layer layout this means that each layer is now
built in a separate derivation, which hopefully leads to better
cacheability.
This commit is contained in:
Vincent Ambo 2019-08-13 00:29:08 +01:00 committed by Vincent Ambo
parent 7214d0aa4f
commit 6285cd8dbf

View file

@ -54,12 +54,12 @@
... ...
}: }:
let with builtins; let
# If a nixpkgs channel is requested, it is retrieved from Github (as # If a nixpkgs channel is requested, it is retrieved from Github (as
# a tarball) and imported. # a tarball) and imported.
fetchImportChannel = channel: fetchImportChannel = channel:
let url = "https://github.com/NixOS/nixpkgs-channels/archive/${channel}.tar.gz"; let url = "https://github.com/NixOS/nixpkgs-channels/archive/${channel}.tar.gz";
in import (builtins.fetchTarball url) {}; in import (fetchTarball url) {};
# If a git repository is requested, it is retrieved via # If a git repository is requested, it is retrieved via
# builtins.fetchGit which defaults to the git configuration of the # builtins.fetchGit which defaults to the git configuration of the
@ -76,32 +76,31 @@ let
# There are some additional caveats around whether the default # There are some additional caveats around whether the default
# branch contains the specified revision, which need to be # branch contains the specified revision, which need to be
# explained to users. # explained to users.
spec = if (builtins.stringLength rev) == 40 then { spec = if (stringLength rev) == 40 then {
inherit url rev; inherit url rev;
} else { } else {
inherit url; inherit url;
ref = rev; ref = rev;
}; };
in import (builtins.fetchGit spec) {}; in import (fetchGit spec) {};
importPath = path: import (builtins.toPath path) {}; importPath = path: import (toPath path) {};
source = builtins.split "!" pkgSource; source = split "!" pkgSource;
sourceType = builtins.elemAt source 0; sourceType = elemAt source 0;
pkgs = with builtins; pkgs =
if sourceType == "nixpkgs" if sourceType == "nixpkgs"
then fetchImportChannel (elemAt source 2) then fetchImportChannel (elemAt source 2)
else if sourceType == "git" else if sourceType == "git"
then fetchImportGit (elemAt source 2) (elemAt source 4) then fetchImportGit (elemAt source 2) (elemAt source 4)
else if sourceType == "path" else if sourceType == "path"
then importPath (elemAt source 2) then importPath (elemAt source 2)
else builtins.throw("Invalid package set source specification: ${pkgSource}"); else throw("Invalid package set source specification: ${pkgSource}");
in in
# Since this is essentially a re-wrapping of some of the functionality that is # Since this is essentially a re-wrapping of some of the functionality that is
# implemented in the dockerTools, we need all of its components in our top-level # implemented in the dockerTools, we need all of its components in our top-level
# namespace. # namespace.
with builtins;
with pkgs; with pkgs;
with dockerTools; with dockerTools;
@ -168,6 +167,11 @@ let
paths = allContents.contents; paths = allContents.contents;
}; };
popularity = builtins.fetchurl {
url = "https://storage.googleapis.com/nixery-layers/popularity/nixos-19.03-20190812.json";
sha256 = "16sxd49vqqg2nrhwynm36ba6bc2yff5cd5hf83wi0hanw5sx3svk";
};
# Before actually creating any image layers, the store paths that need to be # Before actually creating any image layers, the store paths that need to be
# included in the image must be sorted into the layers that they should go # included in the image must be sorted into the layers that they should go
# into. # into.
@ -181,31 +185,37 @@ let
# To invoke the program, a graph of all runtime references is created via # To invoke the program, a graph of all runtime references is created via
# Nix's exportReferencesGraph feature - the resulting layers are read back # Nix's exportReferencesGraph feature - the resulting layers are read back
# into Nix using import-from-derivation. # into Nix using import-from-derivation.
groupedLayers = runCommand "grouped-layers.json" { groupedLayers = fromJSON (readFile (runCommand "grouped-layers.json" {
buildInputs = [ groupLayers ]; __structuredAttrs = true;
exportReferencesGraph.graph = allContents.contents;
PATH = "${groupLayers}/bin";
builder = toFile "builder" ''
. .attrs.sh
group-layers --budget ${toString (maxLayers - 1)} --pop ${popularity} --out ''${outputs[out]}
'';
} ""));
# Given a list of store paths, create an image layer tarball with
# their contents.
pathsToLayer = paths: runCommand "layer.tar" {
} '' } ''
group-layers --fnorg tar --no-recursion -rf "$out" \
--mtime="@$SOURCE_DATE_EPOCH" \
--owner=0 --group=0 /nix /nix/store
tar -rpf "$out" --hard-dereference --sort=name \
--mtime="@$SOURCE_DATE_EPOCH" \
--owner=0 --group=0 ${lib.concatStringsSep " " paths}
''; '';
# The image build infrastructure expects to be outputting a slightly different bulkLayers = writeText "bulk-layers"
# format than the one we serve over the registry protocol. To work around its (lib.concatStringsSep "\n" (map (layer: pathsToLayer layer.contents)
# expectations we need to provide an empty JSON file that it can write some groupedLayers));
# fun data into.
emptyJson = writeText "empty.json" "{}";
bulkLayers = mkManyPureLayers {
name = baseName;
configJson = emptyJson;
closure = writeText "closure" "${contentsEnv} ${emptyJson}";
# One layer will be taken up by the customisationLayer, so
# take up one less.
maxLayers = maxLayers - 1;
};
customisationLayer = mkCustomisationLayer { customisationLayer = mkCustomisationLayer {
name = baseName; name = baseName;
contents = contentsEnv; contents = contentsEnv;
baseJson = emptyJson; baseJson = writeText "empty.json" "{}";
inherit uid gid extraCommands; inherit uid gid extraCommands;
}; };
@ -214,23 +224,22 @@ let
# #
# This computes both an MD5 and a SHA256 hash of each layer, which are used # This computes both an MD5 and a SHA256 hash of each layer, which are used
# for different purposes. See the registry server implementation for details. # for different purposes. See the registry server implementation for details.
#
# Some of this logic is copied straight from `buildLayeredImage`.
allLayersJson = runCommand "fs-layer-list.json" { allLayersJson = runCommand "fs-layer-list.json" {
buildInputs = [ coreutils findutils jq openssl ]; buildInputs = [ coreutils findutils jq openssl ];
} '' } ''
find ${bulkLayers} -mindepth 1 -maxdepth 1 | sort -t/ -k5 -n > layer-list cat ${bulkLayers} | sort -t/ -k5 -n > layer-list
echo ${customisationLayer} >> layer-list echo -n layer-list:
cat layer-list
echo ${customisationLayer}/layer.tar >> layer-list
for layer in $(cat layer-list); do for layer in $(cat layer-list); do
layerPath="$layer/layer.tar" layerSha256=$(sha256sum $layer | cut -d ' ' -f1)
layerSha256=$(sha256sum $layerPath | cut -d ' ' -f1)
# The server application compares binary MD5 hashes and expects base64 # The server application compares binary MD5 hashes and expects base64
# encoding instead of hex. # encoding instead of hex.
layerMd5=$(openssl dgst -md5 -binary $layerPath | openssl enc -base64) layerMd5=$(openssl dgst -md5 -binary $layer | openssl enc -base64)
layerSize=$(wc -c $layerPath | cut -d ' ' -f1) layerSize=$(wc -c $layer | cut -d ' ' -f1)
jq -n -c --arg sha256 $layerSha256 --arg md5 $layerMd5 --arg size $layerSize --arg path $layerPath \ jq -n -c --arg sha256 $layerSha256 --arg md5 $layerMd5 --arg size $layerSize --arg path $layer \
'{ size: ($size | tonumber), sha256: $sha256, md5: $md5, path: $path }' >> fs-layers '{ size: ($size | tonumber), sha256: $sha256, md5: $md5, path: $path }' >> fs-layers
done done
@ -309,6 +318,6 @@ let
pkgs = map (err: err.pkg) allContents.errors; pkgs = map (err: err.pkg) allContents.errors;
}; };
in writeText "manifest-output.json" (if (length allContents.errors) == 0 in writeText "manifest-output.json" (if (length allContents.errors) == 0
then toJSON groupedLayers # manifestOutput then toJSON manifestOutput
else toJSON errorOutput else toJSON errorOutput
) )