6e4b0f3cef
Some companies do not know the 'cl' term. They do know of 'change' and would maybe not like to introduce one more synonym. This cl introduce an optional entry 'gerritChangeName' in besadii.json. The string has to match `^[a-z0-9]+$` for readability. Change-Id: Id70fcb1e45158869f88bf37669be49b8b8a3b295 Reviewed-on: https://cl.tvl.fyi/c/depot/+/4825 Tested-by: BuildkiteCI Reviewed-by: tazjin <tazjin@tvl.su> Autosubmit: asmundo <asmundo@gmail.com>
558 lines
17 KiB
Go
558 lines
17 KiB
Go
// Copyright 2019-2020 Google LLC.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
//
|
|
// besadii is a small CLI tool that is invoked as a hook by various
|
|
// programs to cause CI-related actions.
|
|
//
|
|
// It supports the following modes & operations:
|
|
//
|
|
// Gerrit (ref-updated) hook:
|
|
// - Trigger Buildkite CI builds
|
|
// - Trigger SourceGraph repository index updates
|
|
//
|
|
// Buildkite (post-command) hook:
|
|
// - Submit CL verification status back to Gerrit
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log/syslog"
|
|
"net/http"
|
|
"net/mail"
|
|
"os"
|
|
"os/user"
|
|
"path"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Regular expression to extract change ID out of a URL
|
|
var changeIdRegexp = regexp.MustCompile(`^.*/(\d+)$`)
|
|
|
|
// Regular expression to check if gerritChangeName valid. The
|
|
// limitation could be what is allowed for a git branch name. For now
|
|
// we want to have a stricter limitation for readability and ease of
|
|
// use.
|
|
var gerritChangeNameRegexp = `^[a-z0-9]+$`
|
|
var gerritChangeNameCheck = regexp.MustCompile(gerritChangeNameRegexp)
|
|
|
|
// besadii configuration file structure
|
|
type config struct {
|
|
// Required configuration for Buildkite<>Gerrit monorepo
|
|
// integration.
|
|
Repository string `json:"repository"`
|
|
Branch string `json:"branch"`
|
|
GerritUrl string `json:"gerritUrl"`
|
|
GerritUser string `json:"gerritUser"`
|
|
GerritPassword string `json:"gerritPassword"`
|
|
GerritLabel string `json:"gerritLabel"`
|
|
BuildkiteOrg string `json:"buildkiteOrg"`
|
|
BuildkiteProject string `json:"buildkiteProject"`
|
|
BuildkiteToken string `json:"buildkiteToken"`
|
|
GerritChangeName string `json:"gerritChangeName"`
|
|
|
|
// Optional configuration for Sourcegraph trigger updates.
|
|
SourcegraphUrl string `json:"sourcegraphUrl"`
|
|
SourcegraphToken string `json:"sourcegraphToken"`
|
|
}
|
|
|
|
// buildTrigger represents the information passed to besadii when it
|
|
// is invoked as a Gerrit hook.
|
|
//
|
|
// https://gerrit.googlesource.com/plugins/hooks/+/HEAD/src/main/resources/Documentation/hooks.md
|
|
type buildTrigger struct {
|
|
project string
|
|
ref string
|
|
commit string
|
|
author string
|
|
email string
|
|
|
|
changeId string
|
|
patchset string
|
|
}
|
|
|
|
type Author struct {
|
|
Name string `json:"name"`
|
|
Email string `json:"email"`
|
|
}
|
|
|
|
// Build is the representation of a Buildkite build as described on
|
|
// https://buildkite.com/docs/apis/rest-api/builds#create-a-build
|
|
type Build struct {
|
|
Commit string `json:"commit"`
|
|
Branch string `json:"branch"`
|
|
Author Author `json:"author"`
|
|
Env map[string]string `json:"env"`
|
|
}
|
|
|
|
// BuildResponse is the representation of Buildkite's success response
|
|
// after triggering a build. This has many fields, but we only need
|
|
// one of them.
|
|
type buildResponse struct {
|
|
WebUrl string `json:"web_url"`
|
|
}
|
|
|
|
// reviewInput is a struct representing the data submitted to Gerrit
|
|
// to post a review on a CL.
|
|
//
|
|
// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
|
|
type reviewInput struct {
|
|
Message string `json:"message"`
|
|
Labels map[string]int `json:"labels,omitempty"`
|
|
OmitDuplicateComments bool `json:"omit_duplicate_comments"`
|
|
IgnoreDefaultAttentionSetRules bool `json:"ignore_default_attention_set_rules"`
|
|
Tag string `json:"tag"`
|
|
Notify string `json:"notify,omitempty"`
|
|
}
|
|
|
|
func defaultConfigLocation() (string, error) {
|
|
usr, err := user.Current()
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to get current user: %w", err)
|
|
}
|
|
|
|
return path.Join(usr.HomeDir, "besadii.json"), nil
|
|
}
|
|
|
|
func loadConfig() (*config, error) {
|
|
configPath := os.Getenv("BESADII_CONFIG")
|
|
|
|
if configPath == "" {
|
|
var err error
|
|
configPath, err = defaultConfigLocation()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get config location: %w", err)
|
|
}
|
|
}
|
|
|
|
configJson, err := ioutil.ReadFile(configPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to load besadii config: %w", err)
|
|
}
|
|
|
|
var cfg config
|
|
err = json.Unmarshal(configJson, &cfg)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to unmarshal besadii config: %w", err)
|
|
}
|
|
|
|
// The default Gerrit label to set is 'Verified', unless specified otherwise.
|
|
if cfg.GerritLabel == "" {
|
|
cfg.GerritLabel = "Verified"
|
|
}
|
|
|
|
// The default text referring to a Gerrit Change in BuildKite.
|
|
if cfg.GerritChangeName == "" {
|
|
cfg.GerritChangeName = "cl"
|
|
}
|
|
if !gerritChangeNameCheck.MatchString(cfg.GerritChangeName) {
|
|
return nil, fmt.Errorf("invalid 'gerritChangeName': %s", cfg.GerritChangeName)
|
|
}
|
|
|
|
// Rudimentary config validation logic
|
|
if cfg.SourcegraphUrl != "" && cfg.SourcegraphToken == "" {
|
|
return nil, fmt.Errorf("'SourcegraphToken' must be set if 'SourcegraphUrl' is set")
|
|
}
|
|
|
|
if cfg.Repository == "" || cfg.Branch == "" {
|
|
return nil, fmt.Errorf("missing repository configuration (required: repository, branch)")
|
|
}
|
|
|
|
if cfg.GerritUrl == "" || cfg.GerritUser == "" || cfg.GerritPassword == "" {
|
|
return nil, fmt.Errorf("missing Gerrit configuration (required: gerritUrl, gerritUser, gerritPassword)")
|
|
}
|
|
|
|
if cfg.BuildkiteOrg == "" || cfg.BuildkiteProject == "" || cfg.BuildkiteToken == "" {
|
|
return nil, fmt.Errorf("mising Buildkite configuration (required: buildkiteOrg, buildkiteProject, buildkiteToken)")
|
|
}
|
|
|
|
return &cfg, nil
|
|
}
|
|
|
|
// linkToChange creates the full link to a change's patchset in Gerrit
|
|
func linkToChange(cfg *config, changeId, patchset string) string {
|
|
return fmt.Sprintf("%s/c/%s/+/%s/%s", cfg.GerritUrl, cfg.Repository, changeId, patchset)
|
|
}
|
|
|
|
// updateGerrit posts a comment on a Gerrit CL to indicate the current build status.
|
|
func updateGerrit(cfg *config, review reviewInput, changeId, patchset string) {
|
|
body, _ := json.Marshal(review)
|
|
reader := ioutil.NopCloser(bytes.NewReader(body))
|
|
|
|
url := fmt.Sprintf("%s/a/changes/%s/revisions/%s/review", cfg.GerritUrl, changeId, patchset)
|
|
req, err := http.NewRequest("POST", url, reader)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "failed to create an HTTP request: %w", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
req.SetBasicAuth(cfg.GerritUser, cfg.GerritPassword)
|
|
req.Header.Add("Content-Type", "application/json")
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
fmt.Errorf("failed to update %s on %s: %w", cfg.GerritChangeName, cfg.GerritUrl, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
respBody, _ := ioutil.ReadAll(resp.Body)
|
|
fmt.Fprintf(os.Stderr, "received non-success response from Gerrit: %s (%v)", respBody, resp.Status)
|
|
} else {
|
|
fmt.Printf("Added CI status comment on %s", linkToChange(cfg, changeId, patchset))
|
|
}
|
|
}
|
|
|
|
// Trigger a build of a given branch & commit on Buildkite
|
|
func triggerBuild(cfg *config, log *syslog.Writer, trigger *buildTrigger) error {
|
|
env := make(map[string]string)
|
|
branch := trigger.ref
|
|
|
|
// Pass information about the originating Gerrit change to the
|
|
// build, if it is for a patchset.
|
|
//
|
|
// This information is later used by besadii when invoked by Gerrit
|
|
// to communicate the build status back to Gerrit.
|
|
headBuild := true
|
|
if trigger.changeId != "" && trigger.patchset != "" {
|
|
env["GERRIT_CHANGE_URL"] = linkToChange(cfg, trigger.changeId, trigger.patchset)
|
|
env["GERRIT_CHANGE_ID"] = trigger.changeId
|
|
env["GERRIT_PATCHSET"] = trigger.patchset
|
|
headBuild = false
|
|
|
|
// The branch doesn't have to be a real ref (it's just used to
|
|
// group builds), so make it the identifier for the CL.
|
|
branch = fmt.Sprintf("%s/%v", cfg.GerritChangeName, strings.Split(trigger.ref, "/")[3])
|
|
}
|
|
|
|
build := Build{
|
|
Commit: trigger.commit,
|
|
Branch: branch,
|
|
Env: env,
|
|
Author: Author{
|
|
Name: trigger.author,
|
|
Email: trigger.email,
|
|
},
|
|
}
|
|
|
|
body, _ := json.Marshal(build)
|
|
reader := ioutil.NopCloser(bytes.NewReader(body))
|
|
|
|
bkUrl := fmt.Sprintf("https://api.buildkite.com/v2/organizations/%s/pipelines/%s/builds", cfg.BuildkiteOrg, cfg.BuildkiteProject)
|
|
req, err := http.NewRequest("POST", bkUrl, reader)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create an HTTP request: %w", err)
|
|
}
|
|
|
|
req.Header.Add("Authorization", "Bearer "+cfg.BuildkiteToken)
|
|
req.Header.Add("Content-Type", "application/json")
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
// This might indicate a temporary error on the Buildkite side.
|
|
return fmt.Errorf("failed to send Buildkite request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
respBody, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read Buildkite response body: %w", err)
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusCreated {
|
|
return fmt.Errorf("received non-success response from Buildkite: %s (%v)", respBody, resp.Status)
|
|
}
|
|
|
|
var buildResp buildResponse
|
|
err = json.Unmarshal(respBody, &buildResp)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to unmarshal build response: %w", err)
|
|
}
|
|
|
|
fmt.Fprintf(log, "triggered build for ref %q at commit %q: %s", trigger.ref, trigger.commit, buildResp.WebUrl)
|
|
|
|
// For builds of the HEAD branch there is nothing else to do
|
|
if headBuild {
|
|
return nil
|
|
}
|
|
|
|
// Report the status back to the Gerrit CL so that users can click
|
|
// through to the running build.
|
|
msg := fmt.Sprintf("Started build for patchset #%s on: %s", trigger.patchset, buildResp.WebUrl)
|
|
review := reviewInput{
|
|
Message: msg,
|
|
OmitDuplicateComments: true,
|
|
Tag: "autogenerated:buildkite~trigger",
|
|
|
|
// Do not update the attention set for this comment.
|
|
IgnoreDefaultAttentionSetRules: true,
|
|
|
|
Notify: "NONE",
|
|
}
|
|
updateGerrit(cfg, review, trigger.changeId, trigger.patchset)
|
|
|
|
return nil
|
|
}
|
|
|
|
// Trigger a Sourcegraph repository index update.
|
|
//
|
|
// https://docs.sourcegraph.com/admin/repo/webhooks
|
|
func triggerIndexUpdate(cfg *config, log *syslog.Writer) error {
|
|
req, err := http.NewRequest("POST", cfg.SourcegraphUrl, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
req.Header.Add("Authorization", "token "+cfg.SourcegraphToken)
|
|
|
|
_, err = http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to trigger Sourcegraph index update: %w", err)
|
|
}
|
|
|
|
log.Info("triggered sourcegraph index update")
|
|
return nil
|
|
}
|
|
|
|
// Gerrit passes more flags than we want, but Rob Pike decided[0] in
|
|
// 2013 that the Go art project will not allow users to ignore flags
|
|
// because he "doesn't like it". This function allows users to ignore
|
|
// flags.
|
|
//
|
|
// [0]: https://github.com/golang/go/issues/6112#issuecomment-66083768
|
|
func ignoreFlags(ignore []string) {
|
|
for _, f := range ignore {
|
|
flag.String(f, "", "flag to ignore")
|
|
}
|
|
}
|
|
|
|
// Extract the username & email from Gerrit's uploader flag and set it
|
|
// on the trigger struct, for display in Buildkite.
|
|
func extractChangeUploader(uploader string, trigger *buildTrigger) error {
|
|
// Gerrit passes the uploader in another extra layer of quotes.
|
|
uploader, err := strconv.Unquote(uploader)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to unquote email - forgot quotes on manual invocation?: %w", err)
|
|
}
|
|
|
|
// Extract the uploader username & email from the input passed by
|
|
// Gerrit (in RFC 5322 format).
|
|
addr, err := mail.ParseAddress(uploader)
|
|
if err != nil {
|
|
return fmt.Errorf("invalid change uploader (%s): %w", uploader, err)
|
|
}
|
|
|
|
trigger.author = addr.Name
|
|
trigger.email = addr.Address
|
|
|
|
return nil
|
|
}
|
|
|
|
// Extract the buildtrigger struct out of the flags passed to besadii
|
|
// when invoked as Gerrit's 'patchset-created' hook. This hook is used
|
|
// for triggering CI on in-progress CLs.
|
|
func buildTriggerFromPatchsetCreated(cfg *config) (*buildTrigger, error) {
|
|
// Information that needs to be returned
|
|
var trigger buildTrigger
|
|
|
|
// Information that is only needed for parsing
|
|
var targetBranch, changeUrl, uploader, kind string
|
|
|
|
flag.StringVar(&trigger.project, "project", "", "Gerrit project")
|
|
flag.StringVar(&trigger.commit, "commit", "", "commit hash")
|
|
flag.StringVar(&trigger.patchset, "patchset", "", "patchset ID")
|
|
|
|
flag.StringVar(&targetBranch, "branch", "", "CL target branch")
|
|
flag.StringVar(&changeUrl, "change-url", "", "HTTPS URL of change")
|
|
flag.StringVar(&uploader, "uploader", "", "Change uploader name & email")
|
|
flag.StringVar(&kind, "kind", "", "Kind of patchset")
|
|
|
|
// patchset-created also passes various flags which we don't need.
|
|
ignoreFlags([]string{"topic", "change", "uploader-username", "change-owner", "change-owner-username"})
|
|
|
|
flag.Parse()
|
|
|
|
// Ignore patchsets which do not contain code changes
|
|
if kind == "NO_CODE_CHANGE" || kind == "NO_CHANGE" {
|
|
return nil, nil
|
|
}
|
|
|
|
// Parse username & email
|
|
err := extractChangeUploader(uploader, &trigger)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// If the patchset is not for the HEAD branch of the monorepo, then
|
|
// we can ignore it. It might be some other kind of change
|
|
// (refs/meta/config or Gerrit-internal), but it is not an error.
|
|
if trigger.project != cfg.Repository || targetBranch != cfg.Branch {
|
|
return nil, nil
|
|
}
|
|
|
|
// Change ID is not directly passed in the numeric format, so we
|
|
// need to extract it out of the URL
|
|
matches := changeIdRegexp.FindStringSubmatch(changeUrl)
|
|
trigger.changeId = matches[1]
|
|
|
|
// Construct the CL ref from which the build should happen.
|
|
changeId, _ := strconv.Atoi(trigger.changeId)
|
|
trigger.ref = fmt.Sprintf(
|
|
"refs/changes/%02d/%s/%s",
|
|
changeId%100, trigger.changeId, trigger.patchset,
|
|
)
|
|
|
|
return &trigger, nil
|
|
}
|
|
|
|
// Extract the buildtrigger struct out of the flags passed to besadii
|
|
// when invoked as Gerrit's 'change-merged' hook. This hook is used
|
|
// for triggering HEAD builds after change submission.
|
|
func buildTriggerFromChangeMerged(cfg *config) (*buildTrigger, error) {
|
|
// Information that needs to be returned
|
|
var trigger buildTrigger
|
|
|
|
// Information that is only needed for parsing
|
|
var targetBranch, submitter string
|
|
|
|
flag.StringVar(&trigger.project, "project", "", "Gerrit project")
|
|
flag.StringVar(&trigger.commit, "commit", "", "Commit hash")
|
|
flag.StringVar(&submitter, "submitter", "", "Submitter email & username")
|
|
flag.StringVar(&targetBranch, "branch", "", "CL target branch")
|
|
|
|
// Ignore extra flags passed by change-merged
|
|
ignoreFlags([]string{"change", "topic", "change-url", "submitter-username", "newrev", "change-owner", "change-owner-username"})
|
|
|
|
flag.Parse()
|
|
|
|
// Parse username & email
|
|
err := extractChangeUploader(submitter, &trigger)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// If the patchset is not for the HEAD branch of the monorepo, then
|
|
// we can ignore it.
|
|
if trigger.project != cfg.Repository || targetBranch != cfg.Branch {
|
|
return nil, nil
|
|
}
|
|
|
|
trigger.ref = "refs/heads/" + targetBranch
|
|
|
|
return &trigger, nil
|
|
}
|
|
|
|
func gerritHookMain(cfg *config, log *syslog.Writer, trigger *buildTrigger) {
|
|
if trigger == nil {
|
|
// The hook was not for something we care about.
|
|
os.Exit(0)
|
|
}
|
|
|
|
err := triggerBuild(cfg, log, trigger)
|
|
|
|
if err != nil {
|
|
log.Err(fmt.Sprintf("failed to trigger Buildkite build: %s", err))
|
|
}
|
|
|
|
if cfg.SourcegraphUrl != "" && trigger.ref == "refs/heads/canon" {
|
|
err = triggerIndexUpdate(cfg, log)
|
|
if err != nil {
|
|
log.Err(fmt.Sprintf("failed to trigger sourcegraph index update: %s", err))
|
|
}
|
|
}
|
|
}
|
|
|
|
func postCommandMain(cfg *config) {
|
|
changeId := os.Getenv("GERRIT_CHANGE_ID")
|
|
patchset := os.Getenv("GERRIT_PATCHSET")
|
|
|
|
if changeId == "" || patchset == "" {
|
|
// If these variables are unset, but the hook was invoked, the
|
|
// build was most likely for a branch and not for a CL - no status
|
|
// needs to be reported back to Gerrit!
|
|
fmt.Println("This isn't a %s build, nothing to do. Have a nice day!", cfg.GerritChangeName)
|
|
return
|
|
}
|
|
|
|
if os.Getenv("BUILDKITE_LABEL") != ":duck:" {
|
|
// this is not the build stage, don't do anything.
|
|
return
|
|
}
|
|
|
|
var vote int
|
|
var verb string
|
|
var notify string
|
|
|
|
if os.Getenv("BUILDKITE_COMMAND_EXIT_STATUS") == "0" {
|
|
vote = 1 // automation passed: +1 in Gerrit
|
|
verb = "passed"
|
|
notify = "NONE"
|
|
} else {
|
|
vote = -1
|
|
verb = "failed"
|
|
notify = "OWNER"
|
|
}
|
|
|
|
msg := fmt.Sprintf("Build of patchset %s %s: %s", patchset, verb, os.Getenv("BUILDKITE_BUILD_URL"))
|
|
review := reviewInput{
|
|
Message: msg,
|
|
OmitDuplicateComments: true,
|
|
Labels: map[string]int{
|
|
cfg.GerritLabel: vote,
|
|
},
|
|
|
|
// Update the attention set if we are failing this patchset.
|
|
IgnoreDefaultAttentionSetRules: vote == 1,
|
|
|
|
Tag: "autogenerated:buildkite~result",
|
|
|
|
Notify: notify,
|
|
}
|
|
updateGerrit(cfg, review, changeId, patchset)
|
|
}
|
|
|
|
func main() {
|
|
// Logging happens in syslog because it's almost impossible to get
|
|
// output out of Gerrit hooks otherwise.
|
|
log, err := syslog.New(syslog.LOG_INFO|syslog.LOG_USER, "besadii")
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "failed to open syslog: %s\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
log.Info(fmt.Sprintf("besadii called with arguments: %v", os.Args))
|
|
|
|
bin := path.Base(os.Args[0])
|
|
cfg, err := loadConfig()
|
|
|
|
if err != nil {
|
|
log.Crit(fmt.Sprintf("besadii configuration error: %v", err))
|
|
os.Exit(4)
|
|
}
|
|
|
|
if bin == "patchset-created" {
|
|
trigger, err := buildTriggerFromPatchsetCreated(cfg)
|
|
if err != nil {
|
|
log.Crit(fmt.Sprintf("failed to parse 'patchset-created' invocation from args: %v", err))
|
|
os.Exit(1)
|
|
}
|
|
gerritHookMain(cfg, log, trigger)
|
|
} else if bin == "change-merged" {
|
|
trigger, err := buildTriggerFromChangeMerged(cfg)
|
|
if err != nil {
|
|
log.Crit(fmt.Sprintf("failed to parse 'change-merged' invocation from args: %v", err))
|
|
os.Exit(1)
|
|
}
|
|
gerritHookMain(cfg, log, trigger)
|
|
} else if bin == "post-command" {
|
|
postCommandMain(cfg)
|
|
} else {
|
|
fmt.Fprintf(os.Stderr, "besadii does not know how to be invoked as %q, sorry!", bin)
|
|
os.Exit(1)
|
|
}
|
|
}
|