mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(nix/buildkite): fetch drvmap from *any* default branch builds

Browse source 
With this change, the query we make to Buildkite changes from
explicitly querying for a specific number of *ancestor* builds, to
any latest builds of the default branch that have a drvmap.

This is not really supported by Buildkite, it seems, and the query to
do it feels very wonky and requires a lot of fiddling with `jq` to get
the output into the right shape.

We lose the information about which build we downloaded this from in
the output. Adding that information back would make the `jq` query
much more complex.

Change-Id: I9e7cecdffa9ac09f9e0339eb24d98c0e8dd82292
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9168
Tested-by: BuildkiteCI
Reviewed-by: ezemtsov <eugene.zemtsov@gmail.com>
This commit is contained in:
Vincent Ambo 2023-08-30 15:28:01 +03:00 committed by tazjin
parent 23c811a2a0
commit ba9a62ca9f
1 changed files with 36 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
nix/buildkite/fetch-parent-targets.sh
View file

@ -2,43 +2,54 @@
set -ueo pipefail set -ueo pipefail
# Each Buildkite build stores the derivation target map as a pipeline # Each Buildkite build stores the derivation target map as a pipeline
# artifact. This script determines the most appropriate commit (the # artifact. To reduce the amount of work done by CI, each CI build is
# fork point of the current chain from HEAD) and fetches the artifact. # diffed against the latest such derivation map found for the
# repository.
# #
# New builds can be based on HEAD before the pipeline for the last # Note that this does not take into account when the currently
# commit has finished, in which case it is possible that the fork # processing CL was forked off from the canonical branch, meaning that
# point has no derivation map. To account for this, up to 3 commits # things like nixpkgs updates in between will cause mass rebuilds in
# prior to HEAD are also queried to find a map. # any case.
# #
# If no map is found, the failure mode is not critical: We simply # If no map is found, the failure mode is not critical: We simply
# build all targets. # build all targets.
readonly REPO_ROOT=$(git rev-parse --show-toplevel)
: ${DRVMAP_PATH:=pipeline/drvmap.json} : ${DRVMAP_PATH:=pipeline/drvmap.json}
: ${BUILDKITE_TOKEN_PATH:=~/buildkite-token} : ${BUILDKITE_TOKEN_PATH:=~/buildkite-token}
git fetch -v origin "${BUILDKITE_PIPELINE_DEFAULT_BRANCH}" # Runs a fairly complex Buildkite GraphQL query that attempts to fetch all
# pipeline-gen steps from the default branch, as long as one appears within the
FIRST=$(git merge-base FETCH_HEAD "${BUILDKITE_COMMIT}") # last 50 builds or so. The query restricts build states to running or passed
SECOND=$(git rev-parse "$FIRST~1") # builds, which means that it *should* be unlikely that nothing is found.
THIRD=$(git rev-parse "$FIRST~2") #
# There is no way to filter this more loosely (e.g. by saying "any recent build
function most_relevant_builds { # matching these conditions").
#
# The returned data structure is complex, and disassembled by a JQ script that
# first filters out all builds with no matching jobs (e.g. builds that are still
# in progress), and then filters those down to builds with artifacts, and then
# to drvmap artifacts specifically.
#
# If a recent drvmap was found, this returns its download URL. Otherwise, it
# returns the string "null".
function latest_drvmap_url {
set -u set -u
curl 'https://graphql.buildkite.com/v1' \ curl 'https://graphql.buildkite.com/v1' \
--silent \ --silent \
-H "Authorization: Bearer $(cat ${BUILDKITE_TOKEN_PATH})" \ -H "Authorization: Bearer $(cat ${BUILDKITE_TOKEN_PATH})" \
-d "{\"query\": \"query { pipeline(slug: \\\"$BUILDKITE_ORGANIZATION_SLUG/$BUILDKITE_PIPELINE_SLUG\\\") { builds(commit: [\\\"$FIRST\\\",\\\"$SECOND\\\",\\\"$THIRD\\\"]) { edges { node { uuid }}}}}\"}" | \ -H "Content-Type: application/json" \
jq -r '.data.pipeline.builds.edges[] | .node.uuid' -d "{\"query\": \"{ pipeline(slug: \\\"$BUILDKITE_ORGANIZATION_SLUG/$BUILDKITE_PIPELINE_SLUG\\\") { builds(first: 50, branch: [\\\"%default\\\"], state: [RUNNING, PASSED]) { edges { node { jobs(passed: true, first: 1, type: [COMMAND], step: {key: [\\\"pipeline-gen\\\"]}) { edges { node { ... on JobTypeCommand { url artifacts { edges { node { downloadURL path }}}}}}}}}}}}\"}" | tee out.json | \
jq -r '[.data.pipeline.builds.edges[] | select((.node.jobs.edges | length) > 0) | .node.jobs.edges[] | .node.artifacts[][] | select(.node.path == "pipeline/drvmap.json")][0].node.downloadURL'
} }
mkdir -p tmp readonly DOWNLOAD_URL=$(latest_drvmap_url)
for build in $(most_relevant_builds); do
echo "Checking artifacts for build $build"
buildkite-agent artifact download --build "${build}" "${DRVMAP_PATH}" 'tmp/' || true
if [[ -f "tmp/${DRVMAP_PATH}" ]]; then if [[ ${DOWNLOAD_URL} != "null" ]]; then
echo "Fetched target map from build ${build}" mkdir -p tmp
mv "tmp/${DRVMAP_PATH}" tmp/parent-target-map.json curl -o tmp/parent-target-map.json ${DOWNLOAD_URL} && echo "downloaded parent derivation map" \
break || echo "failed to download derivation map!"
else
echo "no derivation map found!"
fi fi
done