refactor(ops/pipelines): Split build/post steps into separate chunks
This will create `build-chunk-$n.json` files for steps that should run _before_ duck, and `post-chunk-$n.json` files for steps that should run after duck. The post steps are not yet uploaded to Buildkite, but we also don't have any right now. Change-Id: I7e1b59cf55a8bf1d97266f6e988aa496959077bf Reviewed-on: https://cl.tvl.fyi/c/depot/+/5047 Tested-by: BuildkiteCI Reviewed-by: ezemtsov <eugene.zemtsov@gmail.com> Autosubmit: tazjin <tazjin@tvl.su>
This commit is contained in:
parent
f12ceaa622
commit
5a88e47b71
2 changed files with 10 additions and 17 deletions
|
@ -104,8 +104,8 @@ in rec {
|
|||
# Define a build pipeline chunk as a JSON file, using the pipeline
|
||||
# format documented on
|
||||
# https://buildkite.com/docs/pipelines/defining-steps.
|
||||
makePipelineChunk = chunkId: chunk: rec {
|
||||
filename = "chunk-${chunkId}.json";
|
||||
makePipelineChunk = name: chunkId: chunk: rec {
|
||||
filename = "${name}-chunk-${chunkId}.json";
|
||||
path = writeText filename (toJSON {
|
||||
steps = chunk;
|
||||
});
|
||||
|
@ -115,8 +115,8 @@ in rec {
|
|||
# are uploaded sequentially. This is because of a limitation in the
|
||||
# Buildkite backend which struggles to process more than a specific
|
||||
# number of chunks at once.
|
||||
pipelineChunks = steps:
|
||||
attrValues (mapAttrs makePipelineChunk (chunksOf 256 steps));
|
||||
pipelineChunks = name: steps:
|
||||
attrValues (mapAttrs (makePipelineChunk name) (chunksOf 256 steps));
|
||||
|
||||
# Create a pipeline structure for the given targets.
|
||||
mkPipeline = {
|
||||
|
@ -156,24 +156,17 @@ in rec {
|
|||
|
||||
steps = map targetToSteps drvTargets;
|
||||
|
||||
allSteps =
|
||||
buildSteps =
|
||||
# Add build steps for each derivation target and their extra
|
||||
# steps.
|
||||
(lib.concatLists steps)
|
||||
|
||||
# Add additional steps (if set).
|
||||
++ additionalSteps
|
||||
++ additionalSteps;
|
||||
|
||||
# Wait for all previous checks to complete
|
||||
++ [({
|
||||
wait = null;
|
||||
continue_on_failure = true;
|
||||
})]
|
||||
|
||||
# Run post-build steps for status reporting and co.
|
||||
++ postBuildSteps;
|
||||
|
||||
chunks = pipelineChunks allSteps;
|
||||
buildChunks = pipelineChunks "build" buildSteps;
|
||||
postBuildChunks = pipelineChunks "post" postBuildSteps;
|
||||
chunks = buildChunks ++ postBuildChunks;
|
||||
in runCommandNoCC "buildkite-pipeline" {} ''
|
||||
mkdir $out
|
||||
echo "Generated ${toString (length chunks)} pipeline chunks"
|
||||
|
|
|
@ -54,7 +54,7 @@ steps:
|
|||
|
||||
# Steps need to be uploaded in reverse order because pipeline
|
||||
# upload prepends instead of appending.
|
||||
ls pipeline/chunk-*.json | tac | while read chunk; do
|
||||
ls pipeline/build-chunk-*.json | tac | while read chunk; do
|
||||
buildkite-agent pipeline upload $$chunk
|
||||
done
|
||||
|
||||
|
|
Loading…
Reference in a new issue