merge(feat/besadii): Integrate besadii at //ops/besadii
This tool supersedes the previous //ops/sync-gcsr, which no longer exists. Besadii runs as a git post-receive hook and triggers builds for the depot on sourcehut. Builds are only triggered for branches, not for tags. The sourcehut token is read from /etc/secrets where it has been manually placed on camden. Besadii was a hutt kajidic, and also part of the Hutt Clan.
This commit is contained in:
Vincent Ambo
commit
892418db8f
6 changed files with 210 additions and 240 deletions
|
|
@ -14,13 +14,13 @@ with (import ./default.nix {}); [
|
||||||
nix.buildLisp.example
|
nix.buildLisp.example
|
||||||
nix.yants.tests
|
nix.yants.tests
|
||||||
ops."posix_mq.rs"
|
ops."posix_mq.rs"
|
||||||
|
ops.besadii
|
||||||
ops.journaldriver
|
ops.journaldriver
|
||||||
ops.kms_pass
|
ops.kms_pass
|
||||||
ops.kontemplate
|
ops.kontemplate
|
||||||
ops.mq_cli
|
ops.mq_cli
|
||||||
ops.nixos.camdenSystem
|
ops.nixos.camdenSystem
|
||||||
ops.nixos.nuggetSystem
|
ops.nixos.nuggetSystem
|
||||||
ops.sync-gcsr
|
|
||||||
third_party.cgit
|
third_party.cgit
|
||||||
third_party.git
|
third_party.git
|
||||||
third_party.guile
|
third_party.guile
|
||||||
|
|
|
||||||
12
ops/besadii/default.nix
Normal file
12
ops/besadii/default.nix
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
# This program is used as a git post-update hook to trigger builds on
|
||||||
|
# sourcehut.
|
||||||
|
{ depot, ... }:
|
||||||
|
|
||||||
|
depot.buildGo.program {
|
||||||
|
name = "besadii";
|
||||||
|
srcs = [ ./main.go ];
|
||||||
|
|
||||||
|
x_defs = {
|
||||||
|
"main.gitBin" = "${depot.third_party.git}/bin/git";
|
||||||
|
};
|
||||||
|
}
|
||||||
197
ops/besadii/main.go
Normal file
197
ops/besadii/main.go
Normal file
|
|
@ -0,0 +1,197 @@
|
||||||
|
// Copyright 2019 Google LLC.
|
||||||
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
|
//
|
||||||
|
// besadii is a small CLI tool that triggers depot builds on
|
||||||
|
// builds.sr.ht
|
||||||
|
//
|
||||||
|
// It is designed to run as a post-update git hook on the server
|
||||||
|
// hosting the depot.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"log/syslog"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
var gitBin = "git"
|
||||||
|
|
||||||
|
// Represents an updated reference as passed to besadii by git
|
||||||
|
//
|
||||||
|
// https://git-scm.com/docs/githooks#pre-receive
|
||||||
|
type refUpdate struct {
|
||||||
|
name string
|
||||||
|
old string
|
||||||
|
new string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Represents a builds.sr.ht build object as described on
|
||||||
|
// https://man.sr.ht/builds.sr.ht/api.md
|
||||||
|
type Build struct {
|
||||||
|
Manifest string `json:"manifest"`
|
||||||
|
Note string `json:"note"`
|
||||||
|
Tags []string `json:"tags"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Represents a build trigger object as described on <the docs for
|
||||||
|
// this are currently down>
|
||||||
|
type Trigger struct {
|
||||||
|
Action string `json:"action"`
|
||||||
|
Condition string `json:"condition"`
|
||||||
|
To string `json:"to"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Represents a build manifest for sourcehut.
|
||||||
|
type Manifest struct {
|
||||||
|
Image string `json:"image"`
|
||||||
|
Sources []string `json:"sources"`
|
||||||
|
Secrets []string `json:"secrets"`
|
||||||
|
Tasks [](map[string]string) `json:"tasks"`
|
||||||
|
Triggers []Trigger `json:"triggers"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func prepareManifest(commit string) string {
|
||||||
|
m := Manifest{
|
||||||
|
Image: "nixos/latest",
|
||||||
|
Sources: []string{"https://git.camden.tazj.in/"},
|
||||||
|
|
||||||
|
// secret for cachix/tazjin
|
||||||
|
Secrets: []string{"f7f02546-4d95-44f7-a98e-d61fdded8b5b"},
|
||||||
|
|
||||||
|
Tasks: [](map[string]string){
|
||||||
|
{"setup": `# sourcehut does not censor secrets in builds, hence this hack:
|
||||||
|
echo -n 'export CACHIX_SIGNING_KEY=' >> ~/.buildenv
|
||||||
|
cat ~/.cachix-tazjin >> ~/.buildenv
|
||||||
|
nix-env -iA third_party.cachix -f git.tazj.in
|
||||||
|
cachix use tazjin
|
||||||
|
cd git.tazj.in
|
||||||
|
git checkout ` + commit},
|
||||||
|
|
||||||
|
{"build": `cd git.tazj.in
|
||||||
|
nix-build ci-builds.nix > built-paths`},
|
||||||
|
|
||||||
|
{"cache": `cd git.tazj.in
|
||||||
|
cat built-paths | cachix push tazjin`},
|
||||||
|
},
|
||||||
|
|
||||||
|
Triggers: []Trigger{
|
||||||
|
Trigger{Action: "email", Condition: "failure", To: "mail@tazj.in"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
j, _ := json.Marshal(m)
|
||||||
|
return string(j)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trigger a build of a given branch & commit on builds.sr.ht
|
||||||
|
func triggerBuild(log *syslog.Writer, token, branch, commit string) {
|
||||||
|
build := Build{
|
||||||
|
Manifest: prepareManifest(commit),
|
||||||
|
Note: fmt.Sprintf("Build of 'master' at '%s'", commit),
|
||||||
|
Tags: []string{
|
||||||
|
// my branch names tend to contain slashes, which are not valid
|
||||||
|
// identifiers in sourcehut.
|
||||||
|
"depot", strings.ReplaceAll(branch, "/", "_"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
body, _ := json.Marshal(build)
|
||||||
|
reader := ioutil.NopCloser(bytes.NewReader(body))
|
||||||
|
|
||||||
|
req, err := http.NewRequest("POST", "https://builds.sr.ht/api/jobs", reader)
|
||||||
|
if err != nil {
|
||||||
|
log.Err(fmt.Sprintf("failed to create an HTTP request: %s", err))
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Add("Authorization", "token "+token)
|
||||||
|
req.Header.Add("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
// This might indicate a temporary error on the sourcehut side, do
|
||||||
|
// not fail the whole program.
|
||||||
|
log.Err(fmt.Sprintf("failed to send builds.sr.ht request:", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != 200 {
|
||||||
|
respBody, _ := ioutil.ReadAll(resp.Body)
|
||||||
|
log.Err(fmt.Sprintf("received non-success response from builds.sr.ht: %s (%v)", respBody, resp.Status))
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(log, "triggered builds.sr.ht job for branch '%s' at commit '%s'", branch, commit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRefUpdates() ([]refUpdate, error) {
|
||||||
|
var updates []refUpdate
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(os.Stdin)
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
fragments := strings.Split(line, " ")
|
||||||
|
|
||||||
|
if len(fragments) != 3 {
|
||||||
|
return nil, fmt.Errorf("invalid ref update: '%s'", line)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.HasPrefix(fragments[2], "refs/heads/") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
updates = append(updates, refUpdate{
|
||||||
|
old: fragments[0],
|
||||||
|
new: fragments[1],
|
||||||
|
name: strings.TrimPrefix(fragments[2], "refs/heads/"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return updates, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
log, err := syslog.New(syslog.LOG_INFO|syslog.LOG_USER, "besadii")
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("failed to open syslog: %s\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Before triggering builds, it is important that git
|
||||||
|
// update-server-info is run so that cgit correctly serves the
|
||||||
|
// repository.
|
||||||
|
err = exec.Command(gitBin, "update-server-info").Run()
|
||||||
|
if err != nil {
|
||||||
|
log.Alert("failed to run 'git update-server-info' for depot!")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
token, err := ioutil.ReadFile("/etc/secrets/srht-token")
|
||||||
|
if err != nil {
|
||||||
|
log.Alert("sourcehot token could not be read")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
updates, err := parseRefUpdates()
|
||||||
|
if err != nil {
|
||||||
|
log.Err(fmt.Sprintf("could not parse updated refs:", err))
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(log, "triggering builds for %v refs", len(updates))
|
||||||
|
|
||||||
|
for _, update := range updates {
|
||||||
|
triggerBuild(log, string(token), update.name, update.new)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
{ depot, ... }:
|
|
||||||
|
|
||||||
depot.nix.buildGo.program {
|
|
||||||
name = "sync-gcsr";
|
|
||||||
srcs = [ ./main.go ];
|
|
||||||
|
|
||||||
deps = with depot.third_party; map (p: p.gopkg) [
|
|
||||||
gopkgs."gopkg.in".src-d.go-git
|
|
||||||
];
|
|
||||||
|
|
||||||
x_defs = {
|
|
||||||
"main.BuildManifest" = "${./manifest.yaml}";
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -1,202 +0,0 @@
|
||||||
// Copyright 2019 Google LLC.
|
|
||||||
// SPDX-License-Identifier: Apache-2.0
|
|
||||||
//
|
|
||||||
// sync-gcsr implements a small utility that periodically mirrors a
|
|
||||||
// remote Google Cloud Source Repository to a local file path.
|
|
||||||
//
|
|
||||||
// This utility is also responsible for triggering depot builds on
|
|
||||||
// builds.sr.ht if a change is detected on the master branch.
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"time"
|
|
||||||
"bytes"
|
|
||||||
|
|
||||||
git "gopkg.in/src-d/go-git.v4"
|
|
||||||
"gopkg.in/src-d/go-git.v4/plumbing"
|
|
||||||
githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Path to the build manifest, added by Nix at compile time.
|
|
||||||
var BuildManifest string
|
|
||||||
|
|
||||||
// Represents a builds.sr.ht build object as described on
|
|
||||||
// https://man.sr.ht/builds.sr.ht/api.md
|
|
||||||
type Build struct {
|
|
||||||
Manifest string `json:"manifest"`
|
|
||||||
Note string `json:"note"`
|
|
||||||
Tags []string `json:"tags"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func EnvOr(key, def string) string {
|
|
||||||
v := os.Getenv(key)
|
|
||||||
if v == "" {
|
|
||||||
return def
|
|
||||||
}
|
|
||||||
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
|
|
||||||
// Trigger a build of master on builds.sr.ht
|
|
||||||
func triggerBuild(commit string) {
|
|
||||||
manifest, err := ioutil.ReadFile(BuildManifest)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalln("[ERROR] failed to read sr.ht build manifest:", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
build := Build{
|
|
||||||
Manifest: string(manifest),
|
|
||||||
Note: fmt.Sprintf("Build of 'master' at '%s'", commit),
|
|
||||||
Tags: []string{
|
|
||||||
"depot", "master",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
body, _ := json.Marshal(build)
|
|
||||||
reader := ioutil.NopCloser(bytes.NewReader(body))
|
|
||||||
|
|
||||||
req, err := http.NewRequest("POST", "https://builds.sr.ht/api/jobs", reader)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalln("[ERROR] failed to create an HTTP request:", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Add("Authorization", fmt.Sprintf("token %s", os.Getenv("SRHT_TOKEN")))
|
|
||||||
req.Header.Add("Content-Type", "application/json")
|
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
// This might indicate a temporary error on the SourceHut side, do
|
|
||||||
// not fail the whole program.
|
|
||||||
log.Println("failed to send builds.sr.ht request:", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
if resp.StatusCode != 200 {
|
|
||||||
respBody, err := ioutil.ReadAll(resp.Body)
|
|
||||||
log.Printf("received non-success response from builds.sr.ht: %s (%v)[%s]", respBody, resp.Status, err)
|
|
||||||
} else {
|
|
||||||
log.Println("triggered builds.sr.ht job for commit", commit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensure that all remote branches exist locally & are up to date.
|
|
||||||
func updateBranches(auth *githttp.BasicAuth, repo *git.Repository) error {
|
|
||||||
origin, err := repo.Remote("origin")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
refs, err := origin.List(&git.ListOptions{
|
|
||||||
Auth: auth,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, ref := range refs {
|
|
||||||
if !ref.Name().IsBranch() || ref.Type() != plumbing.HashReference {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
name := plumbing.NewBranchReferenceName(ref.Name().Short())
|
|
||||||
|
|
||||||
if current, err := repo.Storer.Reference(name); err == nil {
|
|
||||||
// Determine whether the reference has changed to skip
|
|
||||||
// unnecessary modifications.
|
|
||||||
if current.Hash() == ref.Hash() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
branch := plumbing.NewHashReference(name, ref.Hash())
|
|
||||||
|
|
||||||
err := repo.Storer.SetReference(branch)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if ref.Name().Short() == "master" {
|
|
||||||
go triggerBuild(ref.Hash().String())
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Println("Updated branch", ref.Name().String())
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func updateRepo(auth *githttp.BasicAuth, repo *git.Repository, opts *git.FetchOptions) error {
|
|
||||||
err := repo.Fetch(opts)
|
|
||||||
|
|
||||||
if err == git.NoErrAlreadyUpToDate {
|
|
||||||
// nothing to do ...
|
|
||||||
return nil
|
|
||||||
} else if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Println("Fetched updates from remote, updating local branches")
|
|
||||||
return updateBranches(auth, repo)
|
|
||||||
}
|
|
||||||
|
|
||||||
func cloneRepo(dest, project, repo string, auth *githttp.BasicAuth) (*git.Repository, error) {
|
|
||||||
var cloneOpts = git.CloneOptions{
|
|
||||||
Auth: auth,
|
|
||||||
URL: fmt.Sprintf("https://source.developers.google.com/p/%s/r/%s", project, repo),
|
|
||||||
}
|
|
||||||
|
|
||||||
handle, err := git.PlainClone(dest, true, &cloneOpts)
|
|
||||||
|
|
||||||
if err == git.ErrRepositoryAlreadyExists {
|
|
||||||
handle, err = git.PlainOpen(dest)
|
|
||||||
}
|
|
||||||
|
|
||||||
return handle, updateBranches(auth, handle)
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
dest := EnvOr("SYNC_DEST", "/git/depot")
|
|
||||||
project := EnvOr("SYNC_PROJECT", "tazjins-infrastructure")
|
|
||||||
repo := EnvOr("SYNC_REPO", "depot")
|
|
||||||
user := os.Getenv("SYNC_USER")
|
|
||||||
pass := os.Getenv("SYNC_PASS")
|
|
||||||
|
|
||||||
log.Printf("Syncing repository '%s/%s' to destination '%s'", project, repo, dest)
|
|
||||||
|
|
||||||
var auth *githttp.BasicAuth
|
|
||||||
if user != "" && pass != "" {
|
|
||||||
auth = &githttp.BasicAuth{
|
|
||||||
Username: user,
|
|
||||||
Password: pass,
|
|
||||||
}
|
|
||||||
log.Println("Enabling basic authentication as user", user)
|
|
||||||
}
|
|
||||||
|
|
||||||
handle, err := cloneRepo(dest, project, repo, auth)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to clone repository: %s", err)
|
|
||||||
} else {
|
|
||||||
log.Println("Initiating update loop")
|
|
||||||
}
|
|
||||||
|
|
||||||
fetchOpts := git.FetchOptions{
|
|
||||||
Auth: auth,
|
|
||||||
Force: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
|
||||||
if err = updateRepo(auth, handle, &fetchOpts); err != nil {
|
|
||||||
log.Fatalf("Failed to pull updated repository: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(10 * time.Second)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,23 +0,0 @@
|
||||||
image: nixos/latest
|
|
||||||
sources:
|
|
||||||
- https://git.tazj.in/
|
|
||||||
secrets:
|
|
||||||
# cachix/tazjin
|
|
||||||
- f7f02546-4d95-44f7-a98e-d61fdded8b5b
|
|
||||||
tasks:
|
|
||||||
- setup: |
|
|
||||||
# sourcehut does not censor secrets in builds, hence this hack:
|
|
||||||
echo -n 'export CACHIX_SIGNING_KEY=' > cachix-preamble
|
|
||||||
cat cachix-preamble ~/.cachix-tazjin >> ~/.buildenv
|
|
||||||
nix-env -iA third_party.cachix -f git.tazj.in
|
|
||||||
cachix use tazjin
|
|
||||||
- build: |
|
|
||||||
cd git.tazj.in
|
|
||||||
nix-build ci-builds.nix > built-paths
|
|
||||||
- cache: |
|
|
||||||
cd git.tazj.in
|
|
||||||
cat built-paths | cachix push tazjin
|
|
||||||
triggers:
|
|
||||||
- action: email
|
|
||||||
condition: failure
|
|
||||||
to: mail@tazj.in
|
|
||||||
Loading…
Reference in a new issue