feat(ops/pipelines): Create drvmap structure for each commit

Always create a structure that maps all targets to derivations, and
persist it as a JSON file.

This relates to some of the ideas expressed in:

https://docs.google.com/document/d/16A0a5oUxH1VoiSM8hyFyLW0WiUYpNo2e2D6FTW4BlH8/edit

The file is always uploaded to Buildkite as an artifact. This allows
for retrieving it based on the commit ID in a Buildkite GraphQL query.

By default, Buildkite stores artefacts for 6 months. Storage location
can be overridden (with custom retention) through some environment
variables, but for now at TVL the Buildkite-managed storage is fine.
See also: https://buildkite.com/docs/pipelines/artifacts

In the subsequent filtering implementation, when diffing commits
across a time-range that exceeds artefact retention time, we should
simply default to building everything.

Change-Id: I6d808461cd1c1fdd6983ba8c8ef075736d42caa7
Reviewed-on: https://cl.tvl.fyi/c/depot/+/3662
Tested-by: BuildkiteCI
Reviewed-by: sterni <sternenseemann@systemli.org>
This commit is contained in:
Vincent Ambo 2021-10-01 13:43:53 +03:00 committed by tazjin
parent 546251678a
commit 0a21da2bb4
3 changed files with 40 additions and 10 deletions

View file

@ -15,9 +15,13 @@ let
concatStringsSep
filter
foldl'
getEnv
length
listToAttrs
mapAttrs
toJSON;
pathExists
toJSON
unsafeDiscardStringContext;
inherit (pkgs) lib runCommandNoCC writeText;
in rec {
@ -44,7 +48,7 @@ in rec {
# Skip build steps if their out path has already been built.
skip = headBranch: target: let
shouldSkip = with builtins;
shouldSkip =
# Only skip in real Buildkite builds
(getEnv "BUILDKITE_BUILD_ID" != "") &&
# Always build everything for the canon branch.
@ -60,7 +64,7 @@ in rec {
skip = if skipIfBuilt then skip headBranch target else false;
command = let
drvPath = builtins.unsafeDiscardStringContext target.drvPath;
drvPath = unsafeDiscardStringContext target.drvPath;
in concatStringsSep " " [
# First try to realise the drvPath of the target so we don't evaluate twice.
# Nix has no concept of depending on a derivation file without depending on
@ -162,4 +166,20 @@ in rec {
(chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks
}
'';
# Create a drvmap structure for the given targets, containing the
# mapping of all target paths to their derivations. The mapping can
# be persisted for future use.
mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map (target: {
name = mkLabel target;
value = {
drvPath = unsafeDiscardStringContext target.drvPath;
# Include the attrPath in the output to reconstruct the drv
# without parsing the human-readable label.
attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [
target.__subtarget
];
};
}) drvTargets)));
}

View file

@ -1,6 +1,6 @@
# This file configures the primary build pipeline used for the
# top-level list of depot targets.
{ depot, ... }:
{ depot, pkgs, ... }:
let
# Protobuf check step which validates that changes to .proto files
@ -17,9 +17,16 @@ let
command = "${depot.tools.depotfmt.check}";
label = ":evergreen_tree: (tools/depotfmt)";
};
in depot.nix.buildkite.mkPipeline {
pipeline = depot.nix.buildkite.mkPipeline {
headBranch = "refs/heads/canon";
drvTargets = depot.ci.targets;
skipIfBuilt = true;
additionalSteps = [ depotfmtCheck protoCheck ];
}
};
drvmap = depot.nix.buildkite.mkDrvmap depot.ci.targets;
in pkgs.runCommandNoCC "depot-pipeline" {} ''
mkdir $out
cp -r ${pipeline}/* $out
cp ${drvmap} $out/drvmap.json
''

View file

@ -6,6 +6,7 @@
---
steps:
- label: ":llama:"
key: "pipeline-gen"
command: |
set -ue
@ -22,6 +23,8 @@ steps:
buildkite-agent pipeline upload $$chunk
done
buildkite-agent artifact upload pipeline/drvmap.json
# Wait for all previous steps to complete.
- wait: null
continue_on_failure: true