feat(nix/buildkite): allow custom phases
This change automatically extends the list of known phases as soon as they are added to active phase list. This is great when a user wants to design pipelines with multiple groups of dynamic steps. For example in Resoptima we want to design deployment pipeline where first only staging k8s namespaces are updated/tested and only after, we update production. Change-Id: Iab0f2dc3eadda281e483055e26f00a95442e15b9 Reviewed-on: https://cl.tvl.fyi/c/depot/+/6923 Tested-by: BuildkiteCI Reviewed-by: tazjin <tazjin@tvl.su>
This commit is contained in:
parent
0b06d94606
commit
2ca153141d
1 changed files with 24 additions and 17 deletions
|
@ -143,7 +143,21 @@ rec {
|
||||||
#
|
#
|
||||||
# Can be used for status reporting steps and the like.
|
# Can be used for status reporting steps and the like.
|
||||||
postBuildSteps ? [ ]
|
postBuildSteps ? [ ]
|
||||||
, # Build phases that are active for this invocation (i.e. their
|
# The list of phases known by the current Buildkite
|
||||||
|
# pipeline. Dynamic pipeline chunks for each phase are uploaded
|
||||||
|
# to Buildkite on execution of static part of the
|
||||||
|
# pipeline. Phases selection is hard-coded in the static
|
||||||
|
# pipeline.
|
||||||
|
#
|
||||||
|
# Pipeline generation will fail when an extra step with
|
||||||
|
# unregistered phase is added.
|
||||||
|
#
|
||||||
|
# Common scenarios for different phase:
|
||||||
|
# - "build" - main phase for building all Nix targets
|
||||||
|
# - "release" - pushing artifacts to external repositories
|
||||||
|
# - "deploy" - updating external deployment configurations
|
||||||
|
, phases ? [ "build" "release" ]
|
||||||
|
# Build phases that are active for this invocation (i.e. their
|
||||||
# steps should be generated).
|
# steps should be generated).
|
||||||
#
|
#
|
||||||
# This can be used to disable outputting parts of a pipeline if,
|
# This can be used to disable outputting parts of a pipeline if,
|
||||||
|
@ -151,7 +165,7 @@ rec {
|
||||||
# eval contexts.
|
# eval contexts.
|
||||||
#
|
#
|
||||||
# TODO(tazjin): Fail/warn if unknown phase is requested.
|
# TODO(tazjin): Fail/warn if unknown phase is requested.
|
||||||
activePhases ? [ "build" "release" ]
|
, activePhases ? phases
|
||||||
# Setting this attribute to true cancels dynamic pipeline steps
|
# Setting this attribute to true cancels dynamic pipeline steps
|
||||||
# as soon as the build is marked as failing.
|
# as soon as the build is marked as failing.
|
||||||
#
|
#
|
||||||
|
@ -160,20 +174,13 @@ rec {
|
||||||
, cancelOnBuildFailing ? false
|
, cancelOnBuildFailing ? false
|
||||||
}:
|
}:
|
||||||
let
|
let
|
||||||
# Currently the only known phases are 'build' (Nix builds and
|
|
||||||
# extra steps that are not post-build steps) and 'release' (all
|
|
||||||
# post-build steps).
|
|
||||||
#
|
|
||||||
# TODO(tazjin): Fully configurable set of phases?
|
|
||||||
knownPhases = [ "build" "release" ];
|
|
||||||
|
|
||||||
# List of phases to include.
|
# List of phases to include.
|
||||||
phases = lib.intersectLists activePhases knownPhases;
|
enabledPhases = lib.intersectLists activePhases phases;
|
||||||
|
|
||||||
# Is the 'build' phase included? This phase is treated specially
|
# Is the 'build' phase included? This phase is treated specially
|
||||||
# because it always contains the plain Nix builds, and some
|
# because it always contains the plain Nix builds, and some
|
||||||
# logic/optimisation depends on knowing whether is executing.
|
# logic/optimisation depends on knowing whether is executing.
|
||||||
buildEnabled = elem "build" phases;
|
buildEnabled = elem "build" enabledPhases;
|
||||||
|
|
||||||
# Convert a target into all of its steps, separated by build
|
# Convert a target into all of its steps, separated by build
|
||||||
# phase (as phases end up in different chunks).
|
# phase (as phases end up in different chunks).
|
||||||
|
@ -190,7 +197,7 @@ rec {
|
||||||
|
|
||||||
# Split extra steps by phase.
|
# Split extra steps by phase.
|
||||||
splitExtraSteps = lib.groupBy ({ phase, ... }: phase)
|
splitExtraSteps = lib.groupBy ({ phase, ... }: phase)
|
||||||
(attrValues (mapAttrs (normaliseExtraStep knownPhases overridable)
|
(attrValues (mapAttrs (normaliseExtraStep enabledPhases overridable)
|
||||||
(target.meta.ci.extraSteps or { })));
|
(target.meta.ci.extraSteps or { })));
|
||||||
|
|
||||||
extraSteps = mapAttrs
|
extraSteps = mapAttrs
|
||||||
|
@ -211,7 +218,7 @@ rec {
|
||||||
release = postBuildSteps;
|
release = postBuildSteps;
|
||||||
};
|
};
|
||||||
|
|
||||||
phasesWithSteps = lib.zipAttrsWithNames phases (_: concatLists)
|
phasesWithSteps = lib.zipAttrsWithNames enabledPhases (_: concatLists)
|
||||||
((map targetToSteps drvTargets) ++ [ globalSteps ]);
|
((map targetToSteps drvTargets) ++ [ globalSteps ]);
|
||||||
|
|
||||||
# Generate pipeline chunks for each phase.
|
# Generate pipeline chunks for each phase.
|
||||||
|
@ -222,7 +229,7 @@ rec {
|
||||||
then acc
|
then acc
|
||||||
else acc ++ (pipelineChunks phase phaseSteps))
|
else acc ++ (pipelineChunks phase phaseSteps))
|
||||||
[ ]
|
[ ]
|
||||||
phases;
|
enabledPhases;
|
||||||
|
|
||||||
in
|
in
|
||||||
runCommand "buildkite-pipeline" { } ''
|
runCommand "buildkite-pipeline" { } ''
|
||||||
|
@ -316,7 +323,7 @@ rec {
|
||||||
# Validate and normalise extra step configuration before actually
|
# Validate and normalise extra step configuration before actually
|
||||||
# generating build steps, in order to use user-provided metadata
|
# generating build steps, in order to use user-provided metadata
|
||||||
# during the pipeline generation.
|
# during the pipeline generation.
|
||||||
normaliseExtraStep = knownPhases: overridableParent: key:
|
normaliseExtraStep = phases: overridableParent: key:
|
||||||
{ command
|
{ command
|
||||||
, label ? key
|
, label ? key
|
||||||
, needsOutput ? false
|
, needsOutput ? false
|
||||||
|
@ -337,12 +344,12 @@ rec {
|
||||||
parent = overridableParent parentOverride;
|
parent = overridableParent parentOverride;
|
||||||
parentLabel = parent.env.READTREE_TARGET;
|
parentLabel = parent.env.READTREE_TARGET;
|
||||||
|
|
||||||
validPhase = lib.throwIfNot (elem phase knownPhases) ''
|
validPhase = lib.throwIfNot (elem phase phases) ''
|
||||||
In step '${label}' (from ${parentLabel}):
|
In step '${label}' (from ${parentLabel}):
|
||||||
|
|
||||||
Phase '${phase}' is not valid.
|
Phase '${phase}' is not valid.
|
||||||
|
|
||||||
Known phases: ${concatStringsSep ", " knownPhases}
|
Known phases: ${concatStringsSep ", " phases}
|
||||||
''
|
''
|
||||||
phase;
|
phase;
|
||||||
in
|
in
|
||||||
|
|
Loading…
Reference in a new issue