feat(besadii): Trigger separate builds for build target sets
In CL/570 we split up the build targets into different buckets, with the idea that this should help us avoid the disk space issues on Sourcehut. This commit changes Besadii to read the list of target sets from a file and trigger a separate build for each one of them. Change-Id: If280fda5f40cd130c534c40911072e47c2d8f2be Reviewed-on: https://cl.tvl.fyi/c/depot/+/608 Reviewed-by: lukegb <lukegb@tvl.fyi>
This commit is contained in:
parent
ba94a9f341
commit
6edf69da4b
2 changed files with 60 additions and 18 deletions
|
@ -1,8 +1,14 @@
|
|||
# This program is used as a git post-update hook to trigger builds on
|
||||
# sourcehut.
|
||||
{ depot, ... }:
|
||||
{ ciBuilds, depot, ... }:
|
||||
|
||||
depot.nix.buildTypedGo.program {
|
||||
let
|
||||
inherit (builtins) toFile toJSON;
|
||||
in depot.nix.buildTypedGo.program {
|
||||
name = "besadii";
|
||||
srcs = [ ./main.go2 ];
|
||||
|
||||
x_defs = {
|
||||
"main.TargetList" = toFile "ci-targets.json" (toJSON ciBuilds.__evaluatable);
|
||||
};
|
||||
}
|
||||
|
|
|
@ -23,6 +23,11 @@ import (
|
|||
|
||||
var branchPrefix = "refs/heads/"
|
||||
|
||||
// TargetList is a path to the file containing the list of build
|
||||
// targets in the depot, as a JSON array of strings. This is populated
|
||||
// by the Nix build.
|
||||
var TargetList string
|
||||
|
||||
// Represents an updated branch, as passed to besadii by Gerrit.
|
||||
//
|
||||
// https://gerrit.googlesource.com/plugins/hooks/+/HEAD/src/main/resources/Documentation/hooks.md#ref_updated
|
||||
|
@ -58,27 +63,27 @@ type Manifest struct {
|
|||
Triggers []Trigger `json:"triggers"`
|
||||
}
|
||||
|
||||
func prepareManifest(commit string) string {
|
||||
func prepareManifest(commit, target string) string {
|
||||
m := Manifest{
|
||||
Image: "nixos/latest",
|
||||
Sources: []string{"https://code.tvl.fyi/"},
|
||||
Sources: []string{"https://cl.tvl.fyi/depot"},
|
||||
|
||||
// secret for cachix/tazjin
|
||||
// ID of the secret for cachix/tazjin
|
||||
Secrets: []string{"f7f02546-4d95-44f7-a98e-d61fdded8b5b"},
|
||||
|
||||
Tasks: [](map[string]string){
|
||||
{"setup": `# sourcehut does not censor secrets in builds, hence this hack:
|
||||
echo -n 'export CACHIX_SIGNING_KEY=' >> ~/.buildenv
|
||||
cat ~/.cachix-tazjin >> ~/.buildenv
|
||||
nix-env -iA third_party.cachix -f code.tvl.fyi
|
||||
nix-env -iA third_party.cachix -f depot
|
||||
cachix use tazjin
|
||||
cd code.tvl.fyi
|
||||
cd depot
|
||||
git checkout ` + commit},
|
||||
|
||||
{"build": `cd code.tvl.fyi
|
||||
nix-build ci-builds.nix > built-paths`},
|
||||
{"build": fmt.Sprintf(`cd depot
|
||||
nix-build -A ciBuilds.%s > built-paths`, target)},
|
||||
|
||||
{"cache": `cd code.tvl.fyi
|
||||
{"cache": `cd depot
|
||||
cat built-paths | cachix push tazjin`},
|
||||
},
|
||||
|
||||
|
@ -92,14 +97,16 @@ cat built-paths | cachix push tazjin`},
|
|||
}
|
||||
|
||||
// Trigger a build of a given branch & commit on builds.sr.ht
|
||||
func triggerBuild(log *syslog.Writer, token string, update *branchUpdate) error {
|
||||
func triggerBuild(log *syslog.Writer, token string, update *branchUpdate, target string) error {
|
||||
build := Build{
|
||||
Manifest: prepareManifest(update.commit),
|
||||
Note: fmt.Sprintf("build of %q at %q, submitted by %q", update.branch, update.commit, update.submitter),
|
||||
Manifest: prepareManifest(update.commit, target),
|
||||
Note: fmt.Sprintf("build of target set %q on %q at %q, submitted by %q", target, update.branch, update.commit, update.submitter),
|
||||
Tags: []string{
|
||||
// my branch names tend to contain slashes, which are not valid
|
||||
"depot",
|
||||
// our branch names tend to contain slashes, which are not valid
|
||||
// identifiers in sourcehut.
|
||||
"depot", strings.ReplaceAll(update.branch, "/", "_"),
|
||||
strings.ReplaceAll(update.branch, "/", "_"),
|
||||
target,
|
||||
},
|
||||
}
|
||||
|
||||
|
@ -190,6 +197,26 @@ func branchUpdateFromFlags() (*branchUpdate, error) {
|
|||
return &update, nil
|
||||
}
|
||||
|
||||
func loadBuildTargets() ([]string, error) {
|
||||
if TargetList == "" {
|
||||
return nil, fmt.Errorf("target list file was not set!")
|
||||
}
|
||||
|
||||
var targets []string
|
||||
raw, err := ioutil.ReadFile(TargetList)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read build targets: %w", err)
|
||||
}
|
||||
|
||||
err = json.Unmarshal(raw, &targets)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal build targets: %w", err)
|
||||
}
|
||||
|
||||
return targets, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
log, err := syslog.New(syslog.LOG_INFO|syslog.LOG_USER, "besadii")
|
||||
if err != nil {
|
||||
|
@ -197,6 +224,12 @@ func main() {
|
|||
os.Exit(1)
|
||||
}
|
||||
|
||||
targets, err := loadBuildTargets()
|
||||
if err != nil {
|
||||
log.Alert(fmt.Sprintf("failed to load build targets: %s", err))
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
srhtToken, err := ioutil.ReadFile("/etc/secrets/srht-token")
|
||||
if err != nil {
|
||||
log.Alert("sourcehut token could not be read")
|
||||
|
@ -219,13 +252,16 @@ func main() {
|
|||
os.Exit(0)
|
||||
}
|
||||
|
||||
err = triggerBuild(log, string(srhtToken), update)
|
||||
if err != nil {
|
||||
log.Err(fmt.Sprintf("failed to trigger sr.ht build: %s", err))
|
||||
for _, target := range targets {
|
||||
err = triggerBuild(log, string(srhtToken), update, target)
|
||||
if err != nil {
|
||||
log.Err(fmt.Sprintf("failed to trigger sr.ht build: %s", err))
|
||||
}
|
||||
}
|
||||
|
||||
err = triggerIndexUpdate(string(sourcegraphToken))
|
||||
if err != nil {
|
||||
log.Err(fmt.Sprintf("failed to trigger sourcegraph index update: %s", err))
|
||||
}
|
||||
log.Info("triggered sourcegraph index update")
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue