feat(nix/buildkite): Add support for extra steps

This introduces a new feature to our CI system in which targets can
declare extra steps in `meta.ci.extraSteps`.

See the comment in //nix/buildkite/default.nix for an explanation of
how these extra steps are defined.

Change-Id: Icce2890c743286dd37f43024cd390dcebac8cdba
Reviewed-on: https://cl.tvl.fyi/c/depot/+/5008
Tested-by: BuildkiteCI
Reviewed-by: ezemtsov <eugene.zemtsov@gmail.com>
This commit is contained in:
Vincent Ambo 2022-01-19 21:18:52 +03:00 committed by tazjin
parent 16d72933fc
commit 4e82a9dc86

View file

@ -17,6 +17,9 @@ let
foldl' foldl'
getEnv getEnv
hasAttr hasAttr
hashString
isNull
isString
length length
listToAttrs listToAttrs
mapAttrs mapAttrs
@ -54,6 +57,20 @@ in rec {
then "Target has not changed." then "Target has not changed."
else false; else false;
# Create build command for a derivation target.
mkBuildCommand = target: drvPath: concatStringsSep " " [
# First try to realise the drvPath of the target so we don't evaluate twice.
# Nix has no concept of depending on a derivation file without depending on
# at least one of its `outPath`s, so we need to discard the string context
# if we don't want to build everything during pipeline construction.
"nix-store --realise '${drvPath}' --add-root result --indirect"
# Since we don't gcroot the derivation files, they may be deleted by the
# garbage collector. In that case we can reevaluate and build the attribute
# using nix-build.
"|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr target}' --show-trace)"
];
# Create a pipeline step from a single target. # Create a pipeline step from a single target.
mkStep = headBranch: parentTargetMap: target: mkStep = headBranch: parentTargetMap: target:
let let
@ -62,19 +79,10 @@ in rec {
shouldSkip' = shouldSkip parentTargetMap; shouldSkip' = shouldSkip parentTargetMap;
in { in {
label = ":nix: " + label; label = ":nix: " + label;
key = hashString "sha1" label;
skip = shouldSkip' label drvPath; skip = shouldSkip' label drvPath;
command = mkBuildCommand target drvPath;
command = concatStringsSep " " [ env.READTREE_TARGET = label;
# First try to realise the drvPath of the target so we don't evaluate twice.
# Nix has no concept of depending on a derivation file without depending on
# at least one of its `outPath`s, so we need to discard the string context
# if we don't want to build everything during pipeline construction.
"nix-store --realise '${drvPath}'"
# Since we don't gcroot the derivation files, they may be deleted by the
# garbage collector. In that case we can reevaluate and build the attribute
# using nix-build.
"|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr target}' --show-trace)"
];
# Add a dependency on the initial static pipeline step which # Add a dependency on the initial static pipeline step which
# always runs. This allows build steps uploaded in batches to # always runs. This allows build steps uploaded in batches to
@ -141,10 +149,17 @@ in rec {
# Can be used for status reporting steps and the like. # Can be used for status reporting steps and the like.
postBuildSteps ? [] postBuildSteps ? []
}: let }: let
mkStep' = mkStep headBranch parentTargetMap; targetToSteps = target: let
steps = step = mkStep headBranch parentTargetMap target;
# Add build steps for each derivation target. extraSteps = (attrValues (mapAttrs (mkExtraStep step) (target.meta.ci.extraSteps or {})));
(map mkStep' drvTargets) in [ step ] ++ extraSteps;
steps = map targetToSteps drvTargets;
allSteps =
# Add build steps for each derivation target and their extra
# steps.
(lib.concatLists steps)
# Add additional steps (if set). # Add additional steps (if set).
++ additionalSteps ++ additionalSteps
@ -157,7 +172,8 @@ in rec {
# Run post-build steps for status reporting and co. # Run post-build steps for status reporting and co.
++ postBuildSteps; ++ postBuildSteps;
chunks = pipelineChunks steps;
chunks = pipelineChunks allSteps;
in runCommandNoCC "buildkite-pipeline" {} '' in runCommandNoCC "buildkite-pipeline" {} ''
mkdir $out mkdir $out
echo "Generated ${toString (length chunks)} pipeline chunks" echo "Generated ${toString (length chunks)} pipeline chunks"
@ -182,4 +198,56 @@ in rec {
]; ];
}; };
}) drvTargets))); }) drvTargets)));
# Implementation of extra step logic.
#
# Each target extra step is an attribute specified in
# `meta.ci.extraSteps`. Its attribute name will be used as the step
# name on Buildkite.
#
# command (required): A command that will be run in the depot
# checkout when this step is executed. Should be a derivation
# resulting in a single executable file, e.g. through
# pkgs.writeShellScript.
#
# label (optional): Human-readable label for this step to display
# in the Buildkite UI instead of the attribute name.
#
# needsOutput (optional): If set to true, the parent derivation
# will be built in the working directory before running the
# command. Output will be available as 'result'.
# TODO: Figure out multiple-output derivations.
#
# condition (optional): Any other Buildkite condition, such as
# specific branch requirements, for this step.
# See https://buildkite.com/docs/pipelines/conditionals
#
# alwaysRun (optional): If set to true, this step will always run,
# even if its parent has not been rebuilt.
# Create the Buildkite configuration for an extra step.
mkExtraStep = parent: key: {
command,
label ? key,
needsOutput ? false,
condition ? null,
alwaysRun ? false
}@cfg: let
parentLabel = parent.env.READTREE_TARGET;
step = {
label = ":gear: ${label} (from ${parentLabel})";
skip = if alwaysRun then false else parent.skip or false;
"if" = condition;
depends_on = lib.optional (!alwaysRun && !needsOutput) parent.key;
command = pkgs.writeShellScript "${key}-script" ''
set -ueo pipefail
${lib.optionalString needsOutput "echo '~~~ Preparing build output of ${parentLabel}'"}
${lib.optionalString needsOutput parent.command}
echo '+++ Running extra step command'
exec ${command}
'';
};
in step;
} }