frontend: show submittable status and URL, add runner, revamp logging

This commit is contained in:
Florian Klink 2019-11-21 16:12:04 +01:00
parent 43f8205e85
commit 057294830e
7 changed files with 186 additions and 56 deletions

View file

@ -10,6 +10,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/rakyll/statik/fs"
"github.com/tweag/gerrit-queue/gerrit"
_ "github.com/tweag/gerrit-queue/statik" // register static assets
"github.com/tweag/gerrit-queue/submitqueue"
)
@ -21,13 +22,13 @@ type Frontend struct {
}
//loadTemplate loads a single template from statikFS and returns a template object
func loadTemplate(templateName string) (*template.Template, error) {
func loadTemplate(templateName string, funcMap template.FuncMap) (*template.Template, error) {
statikFS, err := fs.New()
if err != nil {
return nil, err
}
tmpl := template.New(templateName)
tmpl := template.New(templateName).Funcs(funcMap)
r, err := statikFS.Open("/" + templateName)
if err != nil {
return nil, err
@ -41,9 +42,20 @@ func loadTemplate(templateName string) (*template.Template, error) {
}
// MakeFrontend configures the router and returns a new Frontend struct
func MakeFrontend(router *gin.Engine, submitQueue *submitqueue.SubmitQueue) *Frontend {
func MakeFrontend(runner *submitqueue.Runner, submitQueue *submitqueue.SubmitQueue) *Frontend {
router := gin.Default()
funcMap := template.FuncMap{
"isAutoSubmittable": func(serie *submitqueue.Serie) bool {
return submitQueue.IsAutoSubmittable(serie)
},
"changesetURL": func(changeset *gerrit.Changeset) string {
return submitQueue.GetChangesetURL(changeset)
},
}
tmpl := template.Must(loadTemplate("submit-queue.tmpl.html", funcMap))
tmpl := template.Must(loadTemplate("submit-queue.tmpl.html"))
router.SetHTMLTemplate(tmpl)
router.GET("/submit-queue.json", func(c *gin.Context) {
@ -77,7 +89,6 @@ func MakeFrontend(router *gin.Engine, submitQueue *submitqueue.SubmitQueue) *Fro
}
}
// Run starts the webserver on a given address
func (f *Frontend) Run(addr string) error {
return f.Router.Run(addr)
func (f *Frontend) ServeHTTP(w http.ResponseWriter, r *http.Request) {
f.Router.ServeHTTP(w, r)
}

View file

@ -17,13 +17,15 @@ type IClient interface {
SubmitChangeset(changeset *Changeset) (*Changeset, error)
RebaseChangeset(changeset *Changeset, ref string) (*Changeset, error)
RemoveTag(changeset *Changeset, tag string) (*Changeset, error)
GetBaseURL() string
}
var _ IClient = &Client{}
// Client provides some ways to interact with a gerrit instance
type Client struct {
client *goGerrit.Client
client *goGerrit.Client
baseURL string
}
// NewClient initializes a new gerrit client
@ -38,7 +40,10 @@ func NewClient(URL, username, password string) (*Client, error) {
if err != nil {
return nil, err
}
return &Client{client: goGerritClient}, nil
return &Client{
client: goGerritClient,
baseURL: URL,
}, nil
}
// SearchChangesets fetches a list of changesets matching a passed query string
@ -117,3 +122,8 @@ func (gerrit *Client) RemoveTag(changeset *Changeset, tag string) (*Changeset, e
// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#set-hashtags
return changeset, nil
}
// GetBaseURL returns the gerrit base URL
func (gerrit *Client) GetBaseURL() string {
return gerrit.baseURL
}

42
main.go
View file

@ -4,16 +4,16 @@ package main
import (
"os"
"time"
"net/http"
"github.com/tweag/gerrit-queue/frontend"
"github.com/tweag/gerrit-queue/gerrit"
"github.com/tweag/gerrit-queue/submitqueue"
"github.com/gin-gonic/gin"
"github.com/urfave/cli"
"fmt"
log "github.com/sirupsen/logrus"
)
@ -88,31 +88,29 @@ func main() {
log.Printf("Successfully connected to gerrit at %s", URL)
submitQueue := submitqueue.MakeSubmitQueue(gerritClient, projectName, branchName, submitQueueTag)
runner := submitqueue.NewRunner(submitQueue)
router := gin.Default()
frontend := frontend.MakeFrontend(router, &submitQueue)
handler := frontend.MakeFrontend(runner, submitQueue)
err = submitQueue.LoadSeries()
if err != nil {
log.Errorf("Error loading submit queue: %s", err)
}
// fetch only on first run
runner.Trigger(true)
fmt.Println()
fmt.Println()
fmt.Println()
fmt.Println()
for _, serie := range submitQueue.Series {
fmt.Println(fmt.Sprintf("%s", serie))
for _, changeset := range serie.ChangeSets {
fmt.Println(fmt.Sprintf(" - %s", changeset.String()))
// ticker
go func() {
for {
time.Sleep(time.Minute * 10)
runner.Trigger(fetchOnly)
}
fmt.Println()
}()
server := http.Server{
Addr: ":8080",
Handler: handler,
}
frontend.Run(":8080")
if fetchOnly {
//return backlog.Run()
server.ListenAndServe()
if err != nil {
log.Fatal(err)
}
return nil

View file

@ -9,13 +9,17 @@
<h1>Gerrit Submit Queue</h1>
<h2>{{ .projectName }}/{{ .branchName }} is at {{ printf "%.7s" .HEAD }}</h2>
<h2>Current Queue:</h2>
<div class="card-columns">
{{ range $serie := .series }}
<div class="card" style="width: 18rem">
<div class="list-group">
{{ range $changeset := $serie.ChangeSets}}
<div class="list-group-item">
<div class="list-group-item{{ if not ($serie | isAutoSubmittable) }} disabled{{ end }}">
<div class="d-flex w-100 justify-content-between">
<h5>{{ $changeset.Subject }}</h5>
<small>#{{ $changeset.Number }}</small>
<small><a href="{{ changesetURL $changeset }}" target="_blank">#{{ $changeset.Number }}</a></small>
</div>
<div class="d-flex w-100 justify-content-between">
<small>{{ $changeset.OwnerName }}</small>
@ -32,7 +36,8 @@
</div>
{{ end }}
</div>
<br />
</div>
{{ end }}
</div>
</body>
</html>

60
submitqueue/runner.go Normal file
View file

@ -0,0 +1,60 @@
package submitqueue
import (
"sync"
"time"
)
// Runner supervises the submit queue and records historical data about it
type Runner struct {
mut sync.Mutex
submitQueue *SubmitQueue
currentlyRunning *time.Time
results []*Result
}
func NewRunner(sq *SubmitQueue) *Runner {
return &Runner{
submitQueue: sq,
results: []*Result{},
}
}
// For the frontend to consume the data
// TODO: extend to return all the submitQueue results
func (r *Runner) GetResults() (*time.Time, []*Result) {
r.mut.Lock()
defer r.mut.Unlock()
return r.currentlyRunning, r.results
}
// Trigger starts a new batch job
// TODO: make sure only one batch job is started at the same time
// if a batch job is already started, ignore the newest request
// TODO: be more granular in dry-run mode
func (r *Runner) Trigger(fetchOnly bool) {
r.mut.Lock()
if r.currentlyRunning != nil {
return
}
now := time.Now()
r.currentlyRunning = &now
r.mut.Unlock()
defer func() {
r.mut.Lock()
r.currentlyRunning = nil
r.mut.Unlock()
}()
result := r.submitQueue.Run(fetchOnly)
r.mut.Lock()
// drop tail if size > 10
if len(r.results) > 10 {
r.results = append([]*Result{result}, r.results[:9]...)
} else {
r.results = append([]*Result{result}, r.results...)
}
r.mut.Unlock()
}

View file

@ -5,7 +5,7 @@ import (
"github.com/tweag/gerrit-queue/gerrit"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus"
)
// AssembleSeries consumes a list of `Changeset`, and groups them together to series
@ -17,12 +17,12 @@ import (
// and mapParentToSeries, which allows to lookup all series having a certain parent commit id,
// to prepend to any of the existing series
// if we can't find anything, we create a new series
func AssembleSeries(changesets []*gerrit.Changeset) ([]*Serie, error) {
func AssembleSeries(changesets []*gerrit.Changeset, log *logrus.Logger) ([]*Serie, error) {
series := make([]*Serie, 0)
mapLeafToSerie := make(map[string]*Serie, 0)
for _, changeset := range changesets {
logger := log.WithFields(log.Fields{
logger := log.WithFields(logrus.Fields{
"changeset": changeset.String(),
})
@ -90,7 +90,7 @@ func AssembleSeries(changesets []*gerrit.Changeset) ([]*Serie, error) {
// Check integrity, just to be on the safe side.
for _, serie := range series {
logger := log.WithFields(log.Fields{
logger := log.WithFields(logrus.Fields{
"serie": serie.String(),
})
logger.Debugf("checking integrity")

View file

@ -2,10 +2,11 @@ package submitqueue
import (
"fmt"
"time"
"github.com/tweag/gerrit-queue/gerrit"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus"
)
// SubmitQueue contains a list of series, a gerrit connection, and some project configuration
@ -16,11 +17,12 @@ type SubmitQueue struct {
BranchName string
HEAD string
SubmitQueueTag string // the tag used to submit something to the submit queue
URL string
}
// MakeSubmitQueue builds a new submit queue
func MakeSubmitQueue(gerritClient gerrit.IClient, projectName string, branchName string, submitQueueTag string) SubmitQueue {
return SubmitQueue{
func MakeSubmitQueue(gerritClient gerrit.IClient, projectName string, branchName string, submitQueueTag string) *SubmitQueue {
return &SubmitQueue{
Series: make([]*Serie, 0),
gerrit: gerritClient,
ProjectName: projectName,
@ -30,7 +32,7 @@ func MakeSubmitQueue(gerritClient gerrit.IClient, projectName string, branchName
}
// LoadSeries fills .Series by searching changesets, and assembling them to Series.
func (s *SubmitQueue) LoadSeries() error {
func (s *SubmitQueue) LoadSeries(log *logrus.Logger) error {
var queryString = fmt.Sprintf("status:open project:%s branch:%s", s.ProjectName, s.BranchName)
log.Debugf("Running query %s", queryString)
@ -41,7 +43,7 @@ func (s *SubmitQueue) LoadSeries() error {
}
// Assemble to series
series, err := AssembleSeries(changesets)
series, err := AssembleSeries(changesets, log)
if err != nil {
return err
}
@ -75,12 +77,19 @@ func (s *SubmitQueue) IsAutoSubmittable(serie *Serie) bool {
})
}
// GetChangesetURL returns the URL to view a given changeset
func (s *SubmitQueue) GetChangesetURL(changeset *gerrit.Changeset) string {
return fmt.Sprintf("%s/c/%s/+/%d", s.gerrit.GetBaseURL(), s.ProjectName, changeset.Number)
}
// DoSubmit submits changes that can be submitted,
// and updates `Series` to contain the remaining ones
// Also updates `HEAD`.
func (s *SubmitQueue) DoSubmit() error {
func (s *SubmitQueue) DoSubmit(log *logrus.Logger) error {
var remainingSeries []*Serie
// TODO: actually log more!
for _, serie := range s.Series {
serieParentCommitIDs, err := serie.GetParentCommitIDs()
if err != nil {
@ -124,12 +133,12 @@ func (s *SubmitQueue) DoSubmit() error {
// After a DoRebase, consumers are supposed to fetch state again via LoadSeries,
// as things most likely have changed, and error handling during partially failed rebases
// is really tricky
func (s *SubmitQueue) DoRebase() error {
func (s *SubmitQueue) DoRebase(log *logrus.Logger) error {
if s.HEAD == "" {
return fmt.Errorf("current HEAD is an empty string, bailing out")
}
for _, serie := range s.Series {
logger := log.WithFields(log.Fields{
logger := log.WithFields(logrus.Fields{
"serie": serie,
})
if !s.IsAutoSubmittable(serie) {
@ -157,36 +166,73 @@ func (s *SubmitQueue) DoRebase() error {
return nil
}
// Problem: no inspection during the run
// Problem: record the state
type Result struct {
LogEntries []*logrus.Entry
Series []Serie
Error error
}
func (r Result) StartTime() time.Time {
return r.LogEntries[0].Time
}
func (r Result) EndTime() time.Time {
return r.LogEntries[len(r.LogEntries)-1].Time
}
func (r *Result) Fire(entry *logrus.Entry) error {
r.LogEntries = append(r.LogEntries, entry)
return nil
}
func (r *Result) Levels() []logrus.Level {
return logrus.AllLevels
}
// Run starts the submit and rebase logic.
func (s *SubmitQueue) Run() error {
func (s *SubmitQueue) Run(fetchOnly bool) *Result {
r := &Result{}
//TODO: log decisions made and add to some ring buffer
var err error
log := logrus.New()
log.AddHook(r)
commitID, err := s.gerrit.GetHEAD(s.ProjectName, s.BranchName)
if err != nil {
log.Errorf("Unable to retrieve HEAD of branch %s at project %s: %s", s.BranchName, s.ProjectName, err)
return err
r.Error = err
return r
}
s.HEAD = commitID
err = s.LoadSeries()
err = s.LoadSeries(log)
if err != nil {
return err
r.Error = err
return r
}
if len(s.Series) == 0 {
// Nothing to do!
log.Warn("Nothing to do here")
return nil
return r
}
err = s.DoSubmit()
if fetchOnly {
return r
}
err = s.DoSubmit(log)
if err != nil {
return err
r.Error = err
return r
}
err = s.DoRebase()
err = s.DoRebase(log)
if err != nil {
return err
r.Error = err
return r
}
return nil
return r
}
// RebaseSerie rebases a whole serie on top of a given ref