feat: Implement automatic CI target detection for the depot
Automatically walk the entire depot tree and pick out things that are "buildable", then include them in the attribute `ci.targets` (which is now also the target for CI builds). A long time ago, in a land far away, we (well, I, at the time) had a prototype of this which ran into constant issues with infinite recursions while trying to walk the tree. In fact, this is why readTree originally gained the `__readTree`-attribute which marks things that were imported automatically. Based on some code edef whipped up earlier (with the breakthrough being that we also add the attribute to top-level folders, which suddenly resolves a whole bunch of problems), I've now implemented this actually working version. At the moment all builds still happen as one big bag of builds, but at some point we will granularise this. Change-Id: I86f12ce7f63dae98e7e5c6646a4e9d220de783f2 Reviewed-on: https://cl.tvl.fyi/c/depot/+/1854 Tested-by: BuildkiteCI Reviewed-by: kanepyork <rikingcoding@gmail.com> Reviewed-by: glittershark <grfn@gws.fyi>
This commit is contained in:
parent
88317aea0d
commit
4ff9d5dee8
3 changed files with 22 additions and 122 deletions
119
ci-builds.nix
119
ci-builds.nix
|
@ -1,119 +0,0 @@
|
|||
# This file defines the derivations that should be built by CI.
|
||||
#
|
||||
# The "categories" (i.e. attributes) below exist because we run out of
|
||||
# space on Sourcehut otherwise.
|
||||
{ depot, lib, ... }:
|
||||
|
||||
let
|
||||
inherit (builtins) attrNames filter foldl' getAttr substring;
|
||||
|
||||
# attach a nix expression to a drv so we can “build” it
|
||||
# TODO(Profpatsch): instead of failing evaluation if a test fails,
|
||||
# we can put the expression of the test failure into $out
|
||||
# and continue with the other CI derivations.
|
||||
drvify = name: exp: depot.nix.emptyDerivation {
|
||||
inherit name;
|
||||
owo = lib.generators.toPretty {} exp;
|
||||
};
|
||||
|
||||
systemFor = configuration: (depot.third_party.nixos {
|
||||
inherit configuration;
|
||||
}).system;
|
||||
|
||||
in lib.fix (self: {
|
||||
__apprehendEvaluators = throw ''
|
||||
Do not evaluate this attribute set directly. It exists only to group builds
|
||||
for CI runs of different "project groups".
|
||||
|
||||
To use the depot, always start from the top-level attribute tree instead.
|
||||
'';
|
||||
|
||||
# Names of all evaluatable attributes in here. This list will be
|
||||
# used to trigger builds for each key.
|
||||
__evaluatable = filter (key: (substring 0 2 key) != "__") (attrNames self);
|
||||
|
||||
# Combined list of all the targets, used for building everything locally.
|
||||
__allTargets =
|
||||
(with depot.nix.yants; list drv)
|
||||
(foldl' (x: y: x ++ y) [] (map (k: getAttr k self) self.__evaluatable));
|
||||
|
||||
fun = with depot.fun; [
|
||||
amsterdump
|
||||
clbot
|
||||
gemma
|
||||
paroxysm
|
||||
paroxysm.docker
|
||||
quinistry
|
||||
watchblob
|
||||
wcl
|
||||
];
|
||||
|
||||
ops = with depot.ops; [
|
||||
depot.ops."posix_mq.rs"
|
||||
besadii
|
||||
journaldriver
|
||||
kontemplate
|
||||
mq_cli
|
||||
(systemFor nixos.whitby)
|
||||
];
|
||||
|
||||
third_party = with depot.third_party; [
|
||||
apereo-cas
|
||||
bufbuild
|
||||
cgit
|
||||
git
|
||||
grpc
|
||||
loxy
|
||||
nix
|
||||
nix.test-vm
|
||||
openldap
|
||||
rapidcheck
|
||||
];
|
||||
|
||||
gerrit = with depot.third_party.gerrit_plugins; [
|
||||
depot.third_party.gerrit
|
||||
checks
|
||||
owners
|
||||
];
|
||||
|
||||
lisp = with depot.lisp; [
|
||||
dns
|
||||
klatre
|
||||
];
|
||||
|
||||
various = with depot; [
|
||||
nix.buildLisp.example
|
||||
nix.yants.tests
|
||||
tools.cheddar
|
||||
tools.nsfv-setup
|
||||
web.cgit-taz
|
||||
web.todolist
|
||||
web.tvl
|
||||
web.panettone
|
||||
(drvify "getBins-tests" nix.getBins.tests)
|
||||
]
|
||||
++ nix.runExecline.tests
|
||||
;
|
||||
|
||||
# Haskell packages we've patched locally
|
||||
haskellPackages = with depot.third_party.haskellPackages; [
|
||||
generic-arbitrary
|
||||
hgeometry
|
||||
hgeometry-combinatorial
|
||||
vinyl
|
||||
comonad-extras
|
||||
|
||||
# TODO(grfn): Disabled because of build errors with recent nixpkgs
|
||||
# depot.third_party.haskell-language-server.ghc883
|
||||
];
|
||||
|
||||
# User-specific build targets
|
||||
tazjin = with depot.users.tazjin; [
|
||||
blog.rendered
|
||||
emacs
|
||||
finito
|
||||
homepage
|
||||
(systemFor nixos.camden)
|
||||
(systemFor nixos.frog)
|
||||
];
|
||||
})
|
23
default.nix
23
default.nix
|
@ -39,6 +39,22 @@ let
|
|||
users = readTree ./users;
|
||||
web = readTree ./web;
|
||||
};
|
||||
|
||||
# To determine build targets, we walk through the depot tree and
|
||||
# fetch attributes that were imported by readTree and are buildable.
|
||||
#
|
||||
# Any build target that contains `meta.ci = false` will be skipped.
|
||||
|
||||
# Is this tree node eligible for build inclusion?
|
||||
eligible = node: (node ? outPath) && (node.meta.ci or true);
|
||||
|
||||
# Walk the tree starting with 'node', recursively extending the list
|
||||
# of build targets with anything that looks buildable.
|
||||
gather = node:
|
||||
if node ? __readTree then
|
||||
(if eligible node then [node] else []) ++
|
||||
concatMap gather (attrValues node)
|
||||
else [];
|
||||
in fix(self: {
|
||||
config = config self;
|
||||
|
||||
|
@ -55,8 +71,11 @@ in fix(self: {
|
|||
# (e.g. NixOS module inclusions)
|
||||
depotPath = ./.;
|
||||
|
||||
# Load CI builds in a way that can be injected into programs like besadii.
|
||||
ciBuilds = import ./ci-builds.nix self.config;
|
||||
# List of all buildable targets, for CI purposes.
|
||||
#
|
||||
# Note: This *must* be a nested attribute, otherwise we will get
|
||||
# infinite recursion and everything blows up.
|
||||
ci.targets = gather self;
|
||||
}
|
||||
|
||||
# Add local packages as structured by readTree
|
||||
|
|
|
@ -14,7 +14,7 @@ let
|
|||
# documented on https://buildkite.com/docs/pipelines/defining-steps
|
||||
pipeline.steps = [
|
||||
{
|
||||
command = "nix-build -A ciBuilds.__allTargets --show-trace";
|
||||
command = "nix-build -A ci.targets --show-trace";
|
||||
label = ":duck:";
|
||||
}
|
||||
{
|
||||
|
|
Loading…
Reference in a new issue