From d5ddfb7b96835828b4d2c38006e58f9303ef672e Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Thu, 26 Aug 2021 18:02:52 +0300 Subject: [PATCH] feat(pipelines/depot): Skip build steps if their out paths exist Skip build steps if they have already been built, reducing pipelines to the things that actually changed between builds. On canon all targets are always built (we require this for anchoring). Note that this is not perfect, garbage collection and competing pipelines may affect each other. Also note that we have some impure targets that change on every commit. Change-Id: Ic6bae3b6c8e1e7fd2116ec252f5089f471854ab6 Reviewed-on: https://cl.tvl.fyi/c/depot/+/3427 Tested-by: BuildkiteCI Reviewed-by: sterni Reviewed-by: grfn --- ops/pipelines/depot.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix index 02a2c114b..961d8e6ec 100644 --- a/ops/pipelines/depot.nix +++ b/ops/pipelines/depot.nix @@ -48,6 +48,18 @@ let "|| (buildkite-agent meta-data set 'failure' '1'; exit 1)" ]; label = ":nix: ${mkLabel target}"; + + # Skip build steps if their out path has already been built. + skip = let + shouldSkip = with builtins; + # Only skip in real Buildkite builds + (getEnv "BUILDKITE_BUILD_ID" != "") && + # Always build everything for the canon branch. + (getEnv "BUILDKITE_BRANCH" != "canon") && + # Discard string context to avoid realising the store path during + # pipeline construction. + (pathExists (unsafeDiscardStringContext target.outPath)); + in if shouldSkip then "Target was already built." else false; }; # Protobuf check step which validates that changes to .proto files