mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(ops/pipelines): Split build/post steps into separate chunks

Browse source 
This will create `build-chunk-$n.json` files for steps that should run
_before_ duck, and `post-chunk-$n.json` files for steps that should
run after duck.

The post steps are not yet uploaded to Buildkite, but we also don't
have any right now.

Change-Id: I7e1b59cf55a8bf1d97266f6e988aa496959077bf
Reviewed-on: https://cl.tvl.fyi/c/depot/+/5047
Tested-by: BuildkiteCI
Reviewed-by: ezemtsov <eugene.zemtsov@gmail.com>
Autosubmit: tazjin <tazjin@tvl.su>
This commit is contained in:
Vincent Ambo 2022-01-22 14:32:19 +03:00 committed by clbot
parent f12ceaa622
commit 5a88e47b71
2 changed files with 10 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
nix/buildkite/default.nix
View file

@ -104,8 +104,8 @@ in rec {
# Define a build pipeline chunk as a JSON file, using the pipeline # Define a build pipeline chunk as a JSON file, using the pipeline
# format documented on # format documented on
# https://buildkite.com/docs/pipelines/defining-steps. # https://buildkite.com/docs/pipelines/defining-steps.
makePipelineChunk = chunkId: chunk: rec { makePipelineChunk = name: chunkId: chunk: rec {
filename = "chunk-${chunkId}.json"; filename = "${name}-chunk-${chunkId}.json";
path = writeText filename (toJSON { path = writeText filename (toJSON {
steps = chunk; steps = chunk;
}); });
@ -115,8 +115,8 @@ in rec {
# are uploaded sequentially. This is because of a limitation in the # are uploaded sequentially. This is because of a limitation in the
# Buildkite backend which struggles to process more than a specific # Buildkite backend which struggles to process more than a specific
# number of chunks at once. # number of chunks at once.
pipelineChunks = steps: pipelineChunks = name: steps:
attrValues (mapAttrs makePipelineChunk (chunksOf 256 steps)); attrValues (mapAttrs (makePipelineChunk name) (chunksOf 256 steps));
# Create a pipeline structure for the given targets. # Create a pipeline structure for the given targets.
mkPipeline = { mkPipeline = {
@ -156,24 +156,17 @@ in rec {
steps = map targetToSteps drvTargets; steps = map targetToSteps drvTargets;
allSteps = buildSteps =
# Add build steps for each derivation target and their extra # Add build steps for each derivation target and their extra
# steps. # steps.
(lib.concatLists steps) (lib.concatLists steps)
# Add additional steps (if set). # Add additional steps (if set).
++ additionalSteps ++ additionalSteps;
# Wait for all previous checks to complete buildChunks = pipelineChunks "build" buildSteps;
++ [({ postBuildChunks = pipelineChunks "post" postBuildSteps;
wait = null; chunks = buildChunks ++ postBuildChunks;
continue_on_failure = true;
})]
# Run post-build steps for status reporting and co.
++ postBuildSteps;
chunks = pipelineChunks allSteps;
in runCommandNoCC "buildkite-pipeline" {} '' in runCommandNoCC "buildkite-pipeline" {} ''
mkdir $out mkdir $out
echo "Generated ${toString (length chunks)} pipeline chunks" echo "Generated ${toString (length chunks)} pipeline chunks"

2
ops/pipelines/static-pipeline.yaml
View file

@ -54,7 +54,7 @@ steps:
# Steps need to be uploaded in reverse order because pipeline # Steps need to be uploaded in reverse order because pipeline
# upload prepends instead of appending. # upload prepends instead of appending.
ls pipeline/chunk-*.json | tac | while read chunk; do ls pipeline/build-chunk-*.json | tac | while read chunk; do
buildkite-agent pipeline upload $$chunk buildkite-agent pipeline upload $$chunk
done done