feat(nix/buildkite): allow custom phases

This change automatically extends the list of known phases as soon as
they are added to active phase list.

This is great when a user wants to design pipelines with multiple
groups of dynamic steps.

For example in Resoptima we want to design deployment pipeline where
first only staging k8s namespaces are updated/tested and only after,
we update production.

Change-Id: Iab0f2dc3eadda281e483055e26f00a95442e15b9
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6923
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
This commit is contained in:
Evgeny Zemtsov 2022-10-10 13:20:56 +02:00 committed by ezemtsov
parent 0b06d94606
commit 2ca153141d

View file

@ -143,7 +143,21 @@ rec {
# #
# Can be used for status reporting steps and the like. # Can be used for status reporting steps and the like.
postBuildSteps ? [ ] postBuildSteps ? [ ]
, # Build phases that are active for this invocation (i.e. their # The list of phases known by the current Buildkite
# pipeline. Dynamic pipeline chunks for each phase are uploaded
# to Buildkite on execution of static part of the
# pipeline. Phases selection is hard-coded in the static
# pipeline.
#
# Pipeline generation will fail when an extra step with
# unregistered phase is added.
#
# Common scenarios for different phase:
# - "build" - main phase for building all Nix targets
# - "release" - pushing artifacts to external repositories
# - "deploy" - updating external deployment configurations
, phases ? [ "build" "release" ]
# Build phases that are active for this invocation (i.e. their
# steps should be generated). # steps should be generated).
# #
# This can be used to disable outputting parts of a pipeline if, # This can be used to disable outputting parts of a pipeline if,
@ -151,7 +165,7 @@ rec {
# eval contexts. # eval contexts.
# #
# TODO(tazjin): Fail/warn if unknown phase is requested. # TODO(tazjin): Fail/warn if unknown phase is requested.
activePhases ? [ "build" "release" ] , activePhases ? phases
# Setting this attribute to true cancels dynamic pipeline steps # Setting this attribute to true cancels dynamic pipeline steps
# as soon as the build is marked as failing. # as soon as the build is marked as failing.
# #
@ -160,20 +174,13 @@ rec {
, cancelOnBuildFailing ? false , cancelOnBuildFailing ? false
}: }:
let let
# Currently the only known phases are 'build' (Nix builds and
# extra steps that are not post-build steps) and 'release' (all
# post-build steps).
#
# TODO(tazjin): Fully configurable set of phases?
knownPhases = [ "build" "release" ];
# List of phases to include. # List of phases to include.
phases = lib.intersectLists activePhases knownPhases; enabledPhases = lib.intersectLists activePhases phases;
# Is the 'build' phase included? This phase is treated specially # Is the 'build' phase included? This phase is treated specially
# because it always contains the plain Nix builds, and some # because it always contains the plain Nix builds, and some
# logic/optimisation depends on knowing whether is executing. # logic/optimisation depends on knowing whether is executing.
buildEnabled = elem "build" phases; buildEnabled = elem "build" enabledPhases;
# Convert a target into all of its steps, separated by build # Convert a target into all of its steps, separated by build
# phase (as phases end up in different chunks). # phase (as phases end up in different chunks).
@ -190,7 +197,7 @@ rec {
# Split extra steps by phase. # Split extra steps by phase.
splitExtraSteps = lib.groupBy ({ phase, ... }: phase) splitExtraSteps = lib.groupBy ({ phase, ... }: phase)
(attrValues (mapAttrs (normaliseExtraStep knownPhases overridable) (attrValues (mapAttrs (normaliseExtraStep enabledPhases overridable)
(target.meta.ci.extraSteps or { }))); (target.meta.ci.extraSteps or { })));
extraSteps = mapAttrs extraSteps = mapAttrs
@ -211,7 +218,7 @@ rec {
release = postBuildSteps; release = postBuildSteps;
}; };
phasesWithSteps = lib.zipAttrsWithNames phases (_: concatLists) phasesWithSteps = lib.zipAttrsWithNames enabledPhases (_: concatLists)
((map targetToSteps drvTargets) ++ [ globalSteps ]); ((map targetToSteps drvTargets) ++ [ globalSteps ]);
# Generate pipeline chunks for each phase. # Generate pipeline chunks for each phase.
@ -222,7 +229,7 @@ rec {
then acc then acc
else acc ++ (pipelineChunks phase phaseSteps)) else acc ++ (pipelineChunks phase phaseSteps))
[ ] [ ]
phases; enabledPhases;
in in
runCommand "buildkite-pipeline" { } '' runCommand "buildkite-pipeline" { } ''
@ -316,7 +323,7 @@ rec {
# Validate and normalise extra step configuration before actually # Validate and normalise extra step configuration before actually
# generating build steps, in order to use user-provided metadata # generating build steps, in order to use user-provided metadata
# during the pipeline generation. # during the pipeline generation.
normaliseExtraStep = knownPhases: overridableParent: key: normaliseExtraStep = phases: overridableParent: key:
{ command { command
, label ? key , label ? key
, needsOutput ? false , needsOutput ? false
@ -337,12 +344,12 @@ rec {
parent = overridableParent parentOverride; parent = overridableParent parentOverride;
parentLabel = parent.env.READTREE_TARGET; parentLabel = parent.env.READTREE_TARGET;
validPhase = lib.throwIfNot (elem phase knownPhases) '' validPhase = lib.throwIfNot (elem phase phases) ''
In step '${label}' (from ${parentLabel}): In step '${label}' (from ${parentLabel}):
Phase '${phase}' is not valid. Phase '${phase}' is not valid.
Known phases: ${concatStringsSep ", " knownPhases} Known phases: ${concatStringsSep ", " phases}
'' ''
phase; phase;
in in