feat(ops/pipelines): Create drvmap structure for each commit
Always create a structure that maps all targets to derivations, and persist it as a JSON file. This relates to some of the ideas expressed in: https://docs.google.com/document/d/16A0a5oUxH1VoiSM8hyFyLW0WiUYpNo2e2D6FTW4BlH8/edit The file is always uploaded to Buildkite as an artifact. This allows for retrieving it based on the commit ID in a Buildkite GraphQL query. By default, Buildkite stores artefacts for 6 months. Storage location can be overridden (with custom retention) through some environment variables, but for now at TVL the Buildkite-managed storage is fine. See also: https://buildkite.com/docs/pipelines/artifacts In the subsequent filtering implementation, when diffing commits across a time-range that exceeds artefact retention time, we should simply default to building everything. Change-Id: I6d808461cd1c1fdd6983ba8c8ef075736d42caa7 Reviewed-on: https://cl.tvl.fyi/c/depot/+/3662 Tested-by: BuildkiteCI Reviewed-by: sterni <sternenseemann@systemli.org>
This commit is contained in:
parent
546251678a
commit
0a21da2bb4
3 changed files with 40 additions and 10 deletions
|
@ -15,9 +15,13 @@ let
|
||||||
concatStringsSep
|
concatStringsSep
|
||||||
filter
|
filter
|
||||||
foldl'
|
foldl'
|
||||||
|
getEnv
|
||||||
length
|
length
|
||||||
|
listToAttrs
|
||||||
mapAttrs
|
mapAttrs
|
||||||
toJSON;
|
pathExists
|
||||||
|
toJSON
|
||||||
|
unsafeDiscardStringContext;
|
||||||
|
|
||||||
inherit (pkgs) lib runCommandNoCC writeText;
|
inherit (pkgs) lib runCommandNoCC writeText;
|
||||||
in rec {
|
in rec {
|
||||||
|
@ -44,7 +48,7 @@ in rec {
|
||||||
|
|
||||||
# Skip build steps if their out path has already been built.
|
# Skip build steps if their out path has already been built.
|
||||||
skip = headBranch: target: let
|
skip = headBranch: target: let
|
||||||
shouldSkip = with builtins;
|
shouldSkip =
|
||||||
# Only skip in real Buildkite builds
|
# Only skip in real Buildkite builds
|
||||||
(getEnv "BUILDKITE_BUILD_ID" != "") &&
|
(getEnv "BUILDKITE_BUILD_ID" != "") &&
|
||||||
# Always build everything for the canon branch.
|
# Always build everything for the canon branch.
|
||||||
|
@ -60,7 +64,7 @@ in rec {
|
||||||
skip = if skipIfBuilt then skip headBranch target else false;
|
skip = if skipIfBuilt then skip headBranch target else false;
|
||||||
|
|
||||||
command = let
|
command = let
|
||||||
drvPath = builtins.unsafeDiscardStringContext target.drvPath;
|
drvPath = unsafeDiscardStringContext target.drvPath;
|
||||||
in concatStringsSep " " [
|
in concatStringsSep " " [
|
||||||
# First try to realise the drvPath of the target so we don't evaluate twice.
|
# First try to realise the drvPath of the target so we don't evaluate twice.
|
||||||
# Nix has no concept of depending on a derivation file without depending on
|
# Nix has no concept of depending on a derivation file without depending on
|
||||||
|
@ -162,4 +166,20 @@ in rec {
|
||||||
(chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks
|
(chunk: "cp ${chunk.path} $out/${chunk.filename}") chunks
|
||||||
}
|
}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
|
# Create a drvmap structure for the given targets, containing the
|
||||||
|
# mapping of all target paths to their derivations. The mapping can
|
||||||
|
# be persisted for future use.
|
||||||
|
mkDrvmap = drvTargets: writeText "drvmap.json" (toJSON (listToAttrs (map (target: {
|
||||||
|
name = mkLabel target;
|
||||||
|
value = {
|
||||||
|
drvPath = unsafeDiscardStringContext target.drvPath;
|
||||||
|
|
||||||
|
# Include the attrPath in the output to reconstruct the drv
|
||||||
|
# without parsing the human-readable label.
|
||||||
|
attrPath = target.__readTree ++ lib.optionals (target ? __subtarget) [
|
||||||
|
target.__subtarget
|
||||||
|
];
|
||||||
|
};
|
||||||
|
}) drvTargets)));
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
# This file configures the primary build pipeline used for the
|
# This file configures the primary build pipeline used for the
|
||||||
# top-level list of depot targets.
|
# top-level list of depot targets.
|
||||||
{ depot, ... }:
|
{ depot, pkgs, ... }:
|
||||||
|
|
||||||
let
|
let
|
||||||
# Protobuf check step which validates that changes to .proto files
|
# Protobuf check step which validates that changes to .proto files
|
||||||
|
@ -17,9 +17,16 @@ let
|
||||||
command = "${depot.tools.depotfmt.check}";
|
command = "${depot.tools.depotfmt.check}";
|
||||||
label = ":evergreen_tree: (tools/depotfmt)";
|
label = ":evergreen_tree: (tools/depotfmt)";
|
||||||
};
|
};
|
||||||
in depot.nix.buildkite.mkPipeline {
|
pipeline = depot.nix.buildkite.mkPipeline {
|
||||||
headBranch = "refs/heads/canon";
|
headBranch = "refs/heads/canon";
|
||||||
drvTargets = depot.ci.targets;
|
drvTargets = depot.ci.targets;
|
||||||
skipIfBuilt = true;
|
skipIfBuilt = true;
|
||||||
additionalSteps = [ depotfmtCheck protoCheck ];
|
additionalSteps = [ depotfmtCheck protoCheck ];
|
||||||
}
|
};
|
||||||
|
|
||||||
|
drvmap = depot.nix.buildkite.mkDrvmap depot.ci.targets;
|
||||||
|
in pkgs.runCommandNoCC "depot-pipeline" {} ''
|
||||||
|
mkdir $out
|
||||||
|
cp -r ${pipeline}/* $out
|
||||||
|
cp ${drvmap} $out/drvmap.json
|
||||||
|
''
|
||||||
|
|
|
@ -6,6 +6,7 @@
|
||||||
---
|
---
|
||||||
steps:
|
steps:
|
||||||
- label: ":llama:"
|
- label: ":llama:"
|
||||||
|
key: "pipeline-gen"
|
||||||
command: |
|
command: |
|
||||||
set -ue
|
set -ue
|
||||||
|
|
||||||
|
@ -22,6 +23,8 @@ steps:
|
||||||
buildkite-agent pipeline upload $$chunk
|
buildkite-agent pipeline upload $$chunk
|
||||||
done
|
done
|
||||||
|
|
||||||
|
buildkite-agent artifact upload pipeline/drvmap.json
|
||||||
|
|
||||||
# Wait for all previous steps to complete.
|
# Wait for all previous steps to complete.
|
||||||
- wait: null
|
- wait: null
|
||||||
continue_on_failure: true
|
continue_on_failure: true
|
||||||
|
|
Loading…
Reference in a new issue