refactor(ops/pipelines): Split build/post steps into separate chunks

This will create `build-chunk-$n.json` files for steps that should run
_before_ duck, and `post-chunk-$n.json` files for steps that should
run after duck.

The post steps are not yet uploaded to Buildkite, but we also don't
have any right now.

Change-Id: I7e1b59cf55a8bf1d97266f6e988aa496959077bf
Reviewed-on: https://cl.tvl.fyi/c/depot/+/5047
Tested-by: BuildkiteCI
Reviewed-by: ezemtsov <eugene.zemtsov@gmail.com>
Autosubmit: tazjin <tazjin@tvl.su>
This commit is contained in:
Vincent Ambo 2022-01-22 14:32:19 +03:00 committed by clbot
parent f12ceaa622
commit 5a88e47b71
2 changed files with 10 additions and 17 deletions

View file

@ -104,8 +104,8 @@ in rec {
# Define a build pipeline chunk as a JSON file, using the pipeline
# format documented on
# https://buildkite.com/docs/pipelines/defining-steps.
makePipelineChunk = chunkId: chunk: rec {
filename = "chunk-${chunkId}.json";
makePipelineChunk = name: chunkId: chunk: rec {
filename = "${name}-chunk-${chunkId}.json";
path = writeText filename (toJSON {
steps = chunk;
});
@ -115,8 +115,8 @@ in rec {
# are uploaded sequentially. This is because of a limitation in the
# Buildkite backend which struggles to process more than a specific
# number of chunks at once.
pipelineChunks = steps:
attrValues (mapAttrs makePipelineChunk (chunksOf 256 steps));
pipelineChunks = name: steps:
attrValues (mapAttrs (makePipelineChunk name) (chunksOf 256 steps));
# Create a pipeline structure for the given targets.
mkPipeline = {
@ -156,24 +156,17 @@ in rec {
steps = map targetToSteps drvTargets;
allSteps =
buildSteps =
# Add build steps for each derivation target and their extra
# steps.
(lib.concatLists steps)
# Add additional steps (if set).
++ additionalSteps
++ additionalSteps;
# Wait for all previous checks to complete
++ [({
wait = null;
continue_on_failure = true;
})]
# Run post-build steps for status reporting and co.
++ postBuildSteps;
chunks = pipelineChunks allSteps;
buildChunks = pipelineChunks "build" buildSteps;
postBuildChunks = pipelineChunks "post" postBuildSteps;
chunks = buildChunks ++ postBuildChunks;
in runCommandNoCC "buildkite-pipeline" {} ''
mkdir $out
echo "Generated ${toString (length chunks)} pipeline chunks"

View file

@ -54,7 +54,7 @@ steps:
# Steps need to be uploaded in reverse order because pipeline
# upload prepends instead of appending.
ls pipeline/chunk-*.json | tac | while read chunk; do
ls pipeline/build-chunk-*.json | tac | while read chunk; do
buildkite-agent pipeline upload $$chunk
done