0e6ac814c6
--show-trace should make it easier to debug tricky evaluation errors without running nix-build -A ops.pipelines.depot locally again. Change-Id: Ice540562c3b389fc2a49ec1fc0adacb17db2a528 Reviewed-on: https://cl.tvl.fyi/c/depot/+/2947 Tested-by: BuildkiteCI Reviewed-by: tazjin <mail@tazj.in>
105 lines
3.4 KiB
Nix
105 lines
3.4 KiB
Nix
# This file configures the primary build pipeline used for the
|
|
# top-level list of depot targets.
|
|
#
|
|
# It outputs a "YAML" (actually JSON) file which is evaluated and
|
|
# submitted to Buildkite at the start of each build. This means we can
|
|
# dynamically configure the pipeline execution here.
|
|
{ depot, lib, pkgs, ... }:
|
|
|
|
let
|
|
inherit (builtins) concatStringsSep foldl' map toJSON;
|
|
inherit (pkgs) symlinkJoin writeText;
|
|
|
|
# Create an expression that builds the target at the specified
|
|
# location.
|
|
mkBuildExpr = target:
|
|
let
|
|
descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})";
|
|
targetExpr = foldl' descend "import ./. {}" target.__readTree;
|
|
subtargetExpr = descend targetExpr target.__subtarget;
|
|
in if target ? __subtarget then subtargetExpr else targetExpr;
|
|
|
|
# Create a pipeline label from the targets tree location.
|
|
mkLabel = target:
|
|
let label = concatStringsSep "/" target.__readTree;
|
|
in if target ? __subtarget
|
|
then "${label}:${target.__subtarget}"
|
|
else label;
|
|
|
|
# Create a pipeline step from a single target.
|
|
#
|
|
# If the build fails, Buildkite metadata is updated to mark the
|
|
# pipeline as failed. Buildkite has a concept of a failed pipeline
|
|
# regardless, but this data is not accessible.
|
|
mkStep = target: {
|
|
command = ''
|
|
nix-build -E '${mkBuildExpr target}' --show-trace || (buildkite-agent meta-data set "failure" "1"; exit 1)
|
|
'';
|
|
label = ":nix: ${mkLabel target}";
|
|
};
|
|
|
|
# Protobuf check step which validates that changes to .proto files
|
|
# between revisions don't cause backwards-incompatible or otherwise
|
|
# flawed changes.
|
|
protoCheck = {
|
|
command = "${depot.nix.bufCheck}/bin/ci-buf-check";
|
|
label = ":water_buffalo:";
|
|
};
|
|
|
|
# This defines the build pipeline, using the pipeline format
|
|
# documented on https://buildkite.com/docs/pipelines/defining-steps
|
|
#
|
|
# Pipeline steps need to stay in order.
|
|
pipeline.steps =
|
|
# Zero the failure status
|
|
[
|
|
{
|
|
command = "buildkite-agent meta-data set 'failure' '0'";
|
|
label = ":buildkite:";
|
|
}
|
|
{ wait = null; }
|
|
]
|
|
|
|
# Create build steps for each CI target
|
|
++ (map mkStep depot.ci.targets)
|
|
|
|
++ [
|
|
# Simultaneously run protobuf checks
|
|
protoCheck
|
|
|
|
# Wait for all previous checks to complete
|
|
({
|
|
wait = null;
|
|
continue_on_failure = true;
|
|
})
|
|
|
|
# Wait for all steps to complete, then exit with success or
|
|
# failure depending on whether any failure status was written.
|
|
# This step must be :duck:! (yes, really!)
|
|
({
|
|
command = "exit $(buildkite-agent meta-data get 'failure')";
|
|
label = ":duck:";
|
|
key = ":duck:";
|
|
})
|
|
|
|
# After duck, on success, create a gcroot if the build branch is
|
|
# canon.
|
|
#
|
|
# We care that this anchors *most* of the depot, in practice
|
|
# it's unimportant if there is a build race and we get +-1 of
|
|
# the targets.
|
|
#
|
|
# Unfortunately this requires a third evaluation of the graph,
|
|
# but since it happens after :duck: it should not affect the
|
|
# timing of status reporting back to Gerrit.
|
|
({
|
|
command = "nix-instantiate -A ci.gcroot --add-root /nix/var/nix/gcroots/depot/canon";
|
|
label = ":anchor:";
|
|
"if" = ''build.branch == "refs/heads/canon"'';
|
|
depends_on = [{
|
|
step = ":duck:";
|
|
allow_failure = false;
|
|
}];
|
|
})
|
|
];
|
|
in (writeText "depot.yaml" (toJSON pipeline))
|