diff --git a/ci-builds.nix b/ci-builds.nix index 624f4893b..e9abcdb61 100644 --- a/ci-builds.nix +++ b/ci-builds.nix @@ -14,13 +14,13 @@ with (import ./default.nix {}); [ nix.buildLisp.example nix.yants.tests ops."posix_mq.rs" + ops.besadii ops.journaldriver ops.kms_pass ops.kontemplate ops.mq_cli ops.nixos.camdenSystem ops.nixos.nuggetSystem - ops.sync-gcsr third_party.cgit third_party.git third_party.guile diff --git a/ops/besadii/default.nix b/ops/besadii/default.nix new file mode 100644 index 000000000..31f2705d7 --- /dev/null +++ b/ops/besadii/default.nix @@ -0,0 +1,12 @@ +# This program is used as a git post-update hook to trigger builds on +# sourcehut. +{ depot, ... }: + +depot.buildGo.program { + name = "besadii"; + srcs = [ ./main.go ]; + + x_defs = { + "main.gitBin" = "${depot.third_party.git}/bin/git"; + }; +} diff --git a/ops/besadii/main.go b/ops/besadii/main.go new file mode 100644 index 000000000..460eddf26 --- /dev/null +++ b/ops/besadii/main.go @@ -0,0 +1,197 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 +// +// besadii is a small CLI tool that triggers depot builds on +// builds.sr.ht +// +// It is designed to run as a post-update git hook on the server +// hosting the depot. +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "log/syslog" + "net/http" + "os" + "os/exec" + "strings" +) + +var gitBin = "git" + +// Represents an updated reference as passed to besadii by git +// +// https://git-scm.com/docs/githooks#pre-receive +type refUpdate struct { + name string + old string + new string +} + +// Represents a builds.sr.ht build object as described on +// https://man.sr.ht/builds.sr.ht/api.md +type Build struct { + Manifest string `json:"manifest"` + Note string `json:"note"` + Tags []string `json:"tags"` +} + +// Represents a build trigger object as described on +type Trigger struct { + Action string `json:"action"` + Condition string `json:"condition"` + To string `json:"to"` +} + +// Represents a build manifest for sourcehut. +type Manifest struct { + Image string `json:"image"` + Sources []string `json:"sources"` + Secrets []string `json:"secrets"` + Tasks [](map[string]string) `json:"tasks"` + Triggers []Trigger `json:"triggers"` +} + +func prepareManifest(commit string) string { + m := Manifest{ + Image: "nixos/latest", + Sources: []string{"https://git.camden.tazj.in/"}, + + // secret for cachix/tazjin + Secrets: []string{"f7f02546-4d95-44f7-a98e-d61fdded8b5b"}, + + Tasks: [](map[string]string){ + {"setup": `# sourcehut does not censor secrets in builds, hence this hack: +echo -n 'export CACHIX_SIGNING_KEY=' >> ~/.buildenv +cat ~/.cachix-tazjin >> ~/.buildenv +nix-env -iA third_party.cachix -f git.tazj.in +cachix use tazjin +cd git.tazj.in +git checkout ` + commit}, + + {"build": `cd git.tazj.in +nix-build ci-builds.nix > built-paths`}, + + {"cache": `cd git.tazj.in +cat built-paths | cachix push tazjin`}, + }, + + Triggers: []Trigger{ + Trigger{Action: "email", Condition: "failure", To: "mail@tazj.in"}, + }, + } + + j, _ := json.Marshal(m) + return string(j) +} + +// Trigger a build of a given branch & commit on builds.sr.ht +func triggerBuild(log *syslog.Writer, token, branch, commit string) { + build := Build{ + Manifest: prepareManifest(commit), + Note: fmt.Sprintf("Build of 'master' at '%s'", commit), + Tags: []string{ + // my branch names tend to contain slashes, which are not valid + // identifiers in sourcehut. + "depot", strings.ReplaceAll(branch, "/", "_"), + }, + } + + body, _ := json.Marshal(build) + reader := ioutil.NopCloser(bytes.NewReader(body)) + + req, err := http.NewRequest("POST", "https://builds.sr.ht/api/jobs", reader) + if err != nil { + log.Err(fmt.Sprintf("failed to create an HTTP request: %s", err)) + os.Exit(1) + } + + req.Header.Add("Authorization", "token "+token) + req.Header.Add("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + // This might indicate a temporary error on the sourcehut side, do + // not fail the whole program. + log.Err(fmt.Sprintf("failed to send builds.sr.ht request:", err)) + return + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + respBody, _ := ioutil.ReadAll(resp.Body) + log.Err(fmt.Sprintf("received non-success response from builds.sr.ht: %s (%v)", respBody, resp.Status)) + } else { + fmt.Fprintf(log, "triggered builds.sr.ht job for branch '%s' at commit '%s'", branch, commit) + } +} + +func parseRefUpdates() ([]refUpdate, error) { + var updates []refUpdate + + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + line := scanner.Text() + fragments := strings.Split(line, " ") + + if len(fragments) != 3 { + return nil, fmt.Errorf("invalid ref update: '%s'", line) + } + + if !strings.HasPrefix(fragments[2], "refs/heads/") { + continue + } + + updates = append(updates, refUpdate{ + old: fragments[0], + new: fragments[1], + name: strings.TrimPrefix(fragments[2], "refs/heads/"), + }) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return updates, nil +} + +func main() { + log, err := syslog.New(syslog.LOG_INFO|syslog.LOG_USER, "besadii") + if err != nil { + fmt.Printf("failed to open syslog: %s\n", err) + os.Exit(1) + } + + // Before triggering builds, it is important that git + // update-server-info is run so that cgit correctly serves the + // repository. + err = exec.Command(gitBin, "update-server-info").Run() + if err != nil { + log.Alert("failed to run 'git update-server-info' for depot!") + os.Exit(1) + } + + token, err := ioutil.ReadFile("/etc/secrets/srht-token") + if err != nil { + log.Alert("sourcehot token could not be read") + os.Exit(1) + } + + updates, err := parseRefUpdates() + if err != nil { + log.Err(fmt.Sprintf("could not parse updated refs:", err)) + os.Exit(1) + } + + fmt.Fprintf(log, "triggering builds for %v refs", len(updates)) + + for _, update := range updates { + triggerBuild(log, string(token), update.name, update.new) + } +} diff --git a/ops/sync-gcsr/default.nix b/ops/sync-gcsr/default.nix deleted file mode 100644 index ea5d0594a..000000000 --- a/ops/sync-gcsr/default.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ depot, ... }: - -depot.nix.buildGo.program { - name = "sync-gcsr"; - srcs = [ ./main.go ]; - - deps = with depot.third_party; map (p: p.gopkg) [ - gopkgs."gopkg.in".src-d.go-git - ]; - - x_defs = { - "main.BuildManifest" = "${./manifest.yaml}"; - }; -} diff --git a/ops/sync-gcsr/main.go b/ops/sync-gcsr/main.go deleted file mode 100644 index 62c24a92c..000000000 --- a/ops/sync-gcsr/main.go +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2019 Google LLC. -// SPDX-License-Identifier: Apache-2.0 -// -// sync-gcsr implements a small utility that periodically mirrors a -// remote Google Cloud Source Repository to a local file path. -// -// This utility is also responsible for triggering depot builds on -// builds.sr.ht if a change is detected on the master branch. -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "time" - "bytes" - - git "gopkg.in/src-d/go-git.v4" - "gopkg.in/src-d/go-git.v4/plumbing" - githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" -) - -// Path to the build manifest, added by Nix at compile time. -var BuildManifest string - -// Represents a builds.sr.ht build object as described on -// https://man.sr.ht/builds.sr.ht/api.md -type Build struct { - Manifest string `json:"manifest"` - Note string `json:"note"` - Tags []string `json:"tags"` -} - -func EnvOr(key, def string) string { - v := os.Getenv(key) - if v == "" { - return def - } - - return v -} - -// Trigger a build of master on builds.sr.ht -func triggerBuild(commit string) { - manifest, err := ioutil.ReadFile(BuildManifest) - if err != nil { - log.Fatalln("[ERROR] failed to read sr.ht build manifest:", err) - } - - build := Build{ - Manifest: string(manifest), - Note: fmt.Sprintf("Build of 'master' at '%s'", commit), - Tags: []string{ - "depot", "master", - }, - } - - body, _ := json.Marshal(build) - reader := ioutil.NopCloser(bytes.NewReader(body)) - - req, err := http.NewRequest("POST", "https://builds.sr.ht/api/jobs", reader) - if err != nil { - log.Fatalln("[ERROR] failed to create an HTTP request:", err) - } - - req.Header.Add("Authorization", fmt.Sprintf("token %s", os.Getenv("SRHT_TOKEN"))) - req.Header.Add("Content-Type", "application/json") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - // This might indicate a temporary error on the SourceHut side, do - // not fail the whole program. - log.Println("failed to send builds.sr.ht request:", err) - return - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - respBody, err := ioutil.ReadAll(resp.Body) - log.Printf("received non-success response from builds.sr.ht: %s (%v)[%s]", respBody, resp.Status, err) - } else { - log.Println("triggered builds.sr.ht job for commit", commit) - } -} - -// ensure that all remote branches exist locally & are up to date. -func updateBranches(auth *githttp.BasicAuth, repo *git.Repository) error { - origin, err := repo.Remote("origin") - if err != nil { - return err - } - - refs, err := origin.List(&git.ListOptions{ - Auth: auth, - }) - if err != nil { - return err - } - - for _, ref := range refs { - if !ref.Name().IsBranch() || ref.Type() != plumbing.HashReference { - continue - } - - name := plumbing.NewBranchReferenceName(ref.Name().Short()) - - if current, err := repo.Storer.Reference(name); err == nil { - // Determine whether the reference has changed to skip - // unnecessary modifications. - if current.Hash() == ref.Hash() { - continue - } - } - - branch := plumbing.NewHashReference(name, ref.Hash()) - - err := repo.Storer.SetReference(branch) - if err != nil { - return err - } - - if ref.Name().Short() == "master" { - go triggerBuild(ref.Hash().String()) - } - - log.Println("Updated branch", ref.Name().String()) - } - - return nil -} - -func updateRepo(auth *githttp.BasicAuth, repo *git.Repository, opts *git.FetchOptions) error { - err := repo.Fetch(opts) - - if err == git.NoErrAlreadyUpToDate { - // nothing to do ... - return nil - } else if err != nil { - return err - } - - log.Println("Fetched updates from remote, updating local branches") - return updateBranches(auth, repo) -} - -func cloneRepo(dest, project, repo string, auth *githttp.BasicAuth) (*git.Repository, error) { - var cloneOpts = git.CloneOptions{ - Auth: auth, - URL: fmt.Sprintf("https://source.developers.google.com/p/%s/r/%s", project, repo), - } - - handle, err := git.PlainClone(dest, true, &cloneOpts) - - if err == git.ErrRepositoryAlreadyExists { - handle, err = git.PlainOpen(dest) - } - - return handle, updateBranches(auth, handle) -} - -func main() { - dest := EnvOr("SYNC_DEST", "/git/depot") - project := EnvOr("SYNC_PROJECT", "tazjins-infrastructure") - repo := EnvOr("SYNC_REPO", "depot") - user := os.Getenv("SYNC_USER") - pass := os.Getenv("SYNC_PASS") - - log.Printf("Syncing repository '%s/%s' to destination '%s'", project, repo, dest) - - var auth *githttp.BasicAuth - if user != "" && pass != "" { - auth = &githttp.BasicAuth{ - Username: user, - Password: pass, - } - log.Println("Enabling basic authentication as user", user) - } - - handle, err := cloneRepo(dest, project, repo, auth) - - if err != nil { - log.Fatalf("Failed to clone repository: %s", err) - } else { - log.Println("Initiating update loop") - } - - fetchOpts := git.FetchOptions{ - Auth: auth, - Force: true, - } - - for { - if err = updateRepo(auth, handle, &fetchOpts); err != nil { - log.Fatalf("Failed to pull updated repository: %s", err) - } - - time.Sleep(10 * time.Second) - } -} diff --git a/ops/sync-gcsr/manifest.yaml b/ops/sync-gcsr/manifest.yaml deleted file mode 100644 index 3016c2ca5..000000000 --- a/ops/sync-gcsr/manifest.yaml +++ /dev/null @@ -1,23 +0,0 @@ -image: nixos/latest -sources: - - https://git.tazj.in/ -secrets: - # cachix/tazjin - - f7f02546-4d95-44f7-a98e-d61fdded8b5b -tasks: - - setup: | - # sourcehut does not censor secrets in builds, hence this hack: - echo -n 'export CACHIX_SIGNING_KEY=' > cachix-preamble - cat cachix-preamble ~/.cachix-tazjin >> ~/.buildenv - nix-env -iA third_party.cachix -f git.tazj.in - cachix use tazjin - - build: | - cd git.tazj.in - nix-build ci-builds.nix > built-paths - - cache: | - cd git.tazj.in - cat built-paths | cachix push tazjin -triggers: - - action: email - condition: failure - to: mail@tazj.in