frontend: show submittable status and URL, add runner, revamp logging

This commit is contained in:
Florian Klink 2019-11-21 16:12:04 +01:00
parent 43f8205e85
commit 057294830e
7 changed files with 186 additions and 56 deletions

View file

@ -10,6 +10,7 @@ import (
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/rakyll/statik/fs" "github.com/rakyll/statik/fs"
"github.com/tweag/gerrit-queue/gerrit"
_ "github.com/tweag/gerrit-queue/statik" // register static assets _ "github.com/tweag/gerrit-queue/statik" // register static assets
"github.com/tweag/gerrit-queue/submitqueue" "github.com/tweag/gerrit-queue/submitqueue"
) )
@ -21,13 +22,13 @@ type Frontend struct {
} }
//loadTemplate loads a single template from statikFS and returns a template object //loadTemplate loads a single template from statikFS and returns a template object
func loadTemplate(templateName string) (*template.Template, error) { func loadTemplate(templateName string, funcMap template.FuncMap) (*template.Template, error) {
statikFS, err := fs.New() statikFS, err := fs.New()
if err != nil { if err != nil {
return nil, err return nil, err
} }
tmpl := template.New(templateName) tmpl := template.New(templateName).Funcs(funcMap)
r, err := statikFS.Open("/" + templateName) r, err := statikFS.Open("/" + templateName)
if err != nil { if err != nil {
return nil, err return nil, err
@ -41,9 +42,20 @@ func loadTemplate(templateName string) (*template.Template, error) {
} }
// MakeFrontend configures the router and returns a new Frontend struct // MakeFrontend configures the router and returns a new Frontend struct
func MakeFrontend(router *gin.Engine, submitQueue *submitqueue.SubmitQueue) *Frontend { func MakeFrontend(runner *submitqueue.Runner, submitQueue *submitqueue.SubmitQueue) *Frontend {
router := gin.Default()
funcMap := template.FuncMap{
"isAutoSubmittable": func(serie *submitqueue.Serie) bool {
return submitQueue.IsAutoSubmittable(serie)
},
"changesetURL": func(changeset *gerrit.Changeset) string {
return submitQueue.GetChangesetURL(changeset)
},
}
tmpl := template.Must(loadTemplate("submit-queue.tmpl.html", funcMap))
tmpl := template.Must(loadTemplate("submit-queue.tmpl.html"))
router.SetHTMLTemplate(tmpl) router.SetHTMLTemplate(tmpl)
router.GET("/submit-queue.json", func(c *gin.Context) { router.GET("/submit-queue.json", func(c *gin.Context) {
@ -77,7 +89,6 @@ func MakeFrontend(router *gin.Engine, submitQueue *submitqueue.SubmitQueue) *Fro
} }
} }
// Run starts the webserver on a given address func (f *Frontend) ServeHTTP(w http.ResponseWriter, r *http.Request) {
func (f *Frontend) Run(addr string) error { f.Router.ServeHTTP(w, r)
return f.Router.Run(addr)
} }

View file

@ -17,6 +17,7 @@ type IClient interface {
SubmitChangeset(changeset *Changeset) (*Changeset, error) SubmitChangeset(changeset *Changeset) (*Changeset, error)
RebaseChangeset(changeset *Changeset, ref string) (*Changeset, error) RebaseChangeset(changeset *Changeset, ref string) (*Changeset, error)
RemoveTag(changeset *Changeset, tag string) (*Changeset, error) RemoveTag(changeset *Changeset, tag string) (*Changeset, error)
GetBaseURL() string
} }
var _ IClient = &Client{} var _ IClient = &Client{}
@ -24,6 +25,7 @@ var _ IClient = &Client{}
// Client provides some ways to interact with a gerrit instance // Client provides some ways to interact with a gerrit instance
type Client struct { type Client struct {
client *goGerrit.Client client *goGerrit.Client
baseURL string
} }
// NewClient initializes a new gerrit client // NewClient initializes a new gerrit client
@ -38,7 +40,10 @@ func NewClient(URL, username, password string) (*Client, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &Client{client: goGerritClient}, nil return &Client{
client: goGerritClient,
baseURL: URL,
}, nil
} }
// SearchChangesets fetches a list of changesets matching a passed query string // SearchChangesets fetches a list of changesets matching a passed query string
@ -117,3 +122,8 @@ func (gerrit *Client) RemoveTag(changeset *Changeset, tag string) (*Changeset, e
// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#set-hashtags // https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#set-hashtags
return changeset, nil return changeset, nil
} }
// GetBaseURL returns the gerrit base URL
func (gerrit *Client) GetBaseURL() string {
return gerrit.baseURL
}

48
main.go
View file

@ -4,16 +4,16 @@ package main
import ( import (
"os" "os"
"time"
"net/http"
"github.com/tweag/gerrit-queue/frontend" "github.com/tweag/gerrit-queue/frontend"
"github.com/tweag/gerrit-queue/gerrit" "github.com/tweag/gerrit-queue/gerrit"
"github.com/tweag/gerrit-queue/submitqueue" "github.com/tweag/gerrit-queue/submitqueue"
"github.com/gin-gonic/gin"
"github.com/urfave/cli" "github.com/urfave/cli"
"fmt"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
@ -88,31 +88,29 @@ func main() {
log.Printf("Successfully connected to gerrit at %s", URL) log.Printf("Successfully connected to gerrit at %s", URL)
submitQueue := submitqueue.MakeSubmitQueue(gerritClient, projectName, branchName, submitQueueTag) submitQueue := submitqueue.MakeSubmitQueue(gerritClient, projectName, branchName, submitQueueTag)
runner := submitqueue.NewRunner(submitQueue)
router := gin.Default() handler := frontend.MakeFrontend(runner, submitQueue)
frontend := frontend.MakeFrontend(router, &submitQueue)
err = submitQueue.LoadSeries() // fetch only on first run
runner.Trigger(true)
// ticker
go func() {
for {
time.Sleep(time.Minute * 10)
runner.Trigger(fetchOnly)
}
}()
server := http.Server{
Addr: ":8080",
Handler: handler,
}
server.ListenAndServe()
if err != nil { if err != nil {
log.Errorf("Error loading submit queue: %s", err) log.Fatal(err)
}
fmt.Println()
fmt.Println()
fmt.Println()
fmt.Println()
for _, serie := range submitQueue.Series {
fmt.Println(fmt.Sprintf("%s", serie))
for _, changeset := range serie.ChangeSets {
fmt.Println(fmt.Sprintf(" - %s", changeset.String()))
}
fmt.Println()
}
frontend.Run(":8080")
if fetchOnly {
//return backlog.Run()
} }
return nil return nil

View file

@ -9,13 +9,17 @@
<h1>Gerrit Submit Queue</h1> <h1>Gerrit Submit Queue</h1>
<h2>{{ .projectName }}/{{ .branchName }} is at {{ printf "%.7s" .HEAD }}</h2> <h2>{{ .projectName }}/{{ .branchName }} is at {{ printf "%.7s" .HEAD }}</h2>
<h2>Current Queue:</h2> <h2>Current Queue:</h2>
<div class="card-columns">
{{ range $serie := .series }} {{ range $serie := .series }}
<div class="card" style="width: 18rem">
<div class="list-group"> <div class="list-group">
{{ range $changeset := $serie.ChangeSets}} {{ range $changeset := $serie.ChangeSets}}
<div class="list-group-item"> <div class="list-group-item{{ if not ($serie | isAutoSubmittable) }} disabled{{ end }}">
<div class="d-flex w-100 justify-content-between"> <div class="d-flex w-100 justify-content-between">
<h5>{{ $changeset.Subject }}</h5> <h5>{{ $changeset.Subject }}</h5>
<small>#{{ $changeset.Number }}</small> <small><a href="{{ changesetURL $changeset }}" target="_blank">#{{ $changeset.Number }}</a></small>
</div> </div>
<div class="d-flex w-100 justify-content-between"> <div class="d-flex w-100 justify-content-between">
<small>{{ $changeset.OwnerName }}</small> <small>{{ $changeset.OwnerName }}</small>
@ -32,7 +36,8 @@
</div> </div>
{{ end }} {{ end }}
</div> </div>
<br /> </div>
{{ end }} {{ end }}
</div>
</body> </body>
</html> </html>

60
submitqueue/runner.go Normal file
View file

@ -0,0 +1,60 @@
package submitqueue
import (
"sync"
"time"
)
// Runner supervises the submit queue and records historical data about it
type Runner struct {
mut sync.Mutex
submitQueue *SubmitQueue
currentlyRunning *time.Time
results []*Result
}
func NewRunner(sq *SubmitQueue) *Runner {
return &Runner{
submitQueue: sq,
results: []*Result{},
}
}
// For the frontend to consume the data
// TODO: extend to return all the submitQueue results
func (r *Runner) GetResults() (*time.Time, []*Result) {
r.mut.Lock()
defer r.mut.Unlock()
return r.currentlyRunning, r.results
}
// Trigger starts a new batch job
// TODO: make sure only one batch job is started at the same time
// if a batch job is already started, ignore the newest request
// TODO: be more granular in dry-run mode
func (r *Runner) Trigger(fetchOnly bool) {
r.mut.Lock()
if r.currentlyRunning != nil {
return
}
now := time.Now()
r.currentlyRunning = &now
r.mut.Unlock()
defer func() {
r.mut.Lock()
r.currentlyRunning = nil
r.mut.Unlock()
}()
result := r.submitQueue.Run(fetchOnly)
r.mut.Lock()
// drop tail if size > 10
if len(r.results) > 10 {
r.results = append([]*Result{result}, r.results[:9]...)
} else {
r.results = append([]*Result{result}, r.results...)
}
r.mut.Unlock()
}

View file

@ -5,7 +5,7 @@ import (
"github.com/tweag/gerrit-queue/gerrit" "github.com/tweag/gerrit-queue/gerrit"
log "github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
// AssembleSeries consumes a list of `Changeset`, and groups them together to series // AssembleSeries consumes a list of `Changeset`, and groups them together to series
@ -17,12 +17,12 @@ import (
// and mapParentToSeries, which allows to lookup all series having a certain parent commit id, // and mapParentToSeries, which allows to lookup all series having a certain parent commit id,
// to prepend to any of the existing series // to prepend to any of the existing series
// if we can't find anything, we create a new series // if we can't find anything, we create a new series
func AssembleSeries(changesets []*gerrit.Changeset) ([]*Serie, error) { func AssembleSeries(changesets []*gerrit.Changeset, log *logrus.Logger) ([]*Serie, error) {
series := make([]*Serie, 0) series := make([]*Serie, 0)
mapLeafToSerie := make(map[string]*Serie, 0) mapLeafToSerie := make(map[string]*Serie, 0)
for _, changeset := range changesets { for _, changeset := range changesets {
logger := log.WithFields(log.Fields{ logger := log.WithFields(logrus.Fields{
"changeset": changeset.String(), "changeset": changeset.String(),
}) })
@ -90,7 +90,7 @@ func AssembleSeries(changesets []*gerrit.Changeset) ([]*Serie, error) {
// Check integrity, just to be on the safe side. // Check integrity, just to be on the safe side.
for _, serie := range series { for _, serie := range series {
logger := log.WithFields(log.Fields{ logger := log.WithFields(logrus.Fields{
"serie": serie.String(), "serie": serie.String(),
}) })
logger.Debugf("checking integrity") logger.Debugf("checking integrity")

View file

@ -2,10 +2,11 @@ package submitqueue
import ( import (
"fmt" "fmt"
"time"
"github.com/tweag/gerrit-queue/gerrit" "github.com/tweag/gerrit-queue/gerrit"
log "github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
// SubmitQueue contains a list of series, a gerrit connection, and some project configuration // SubmitQueue contains a list of series, a gerrit connection, and some project configuration
@ -16,11 +17,12 @@ type SubmitQueue struct {
BranchName string BranchName string
HEAD string HEAD string
SubmitQueueTag string // the tag used to submit something to the submit queue SubmitQueueTag string // the tag used to submit something to the submit queue
URL string
} }
// MakeSubmitQueue builds a new submit queue // MakeSubmitQueue builds a new submit queue
func MakeSubmitQueue(gerritClient gerrit.IClient, projectName string, branchName string, submitQueueTag string) SubmitQueue { func MakeSubmitQueue(gerritClient gerrit.IClient, projectName string, branchName string, submitQueueTag string) *SubmitQueue {
return SubmitQueue{ return &SubmitQueue{
Series: make([]*Serie, 0), Series: make([]*Serie, 0),
gerrit: gerritClient, gerrit: gerritClient,
ProjectName: projectName, ProjectName: projectName,
@ -30,7 +32,7 @@ func MakeSubmitQueue(gerritClient gerrit.IClient, projectName string, branchName
} }
// LoadSeries fills .Series by searching changesets, and assembling them to Series. // LoadSeries fills .Series by searching changesets, and assembling them to Series.
func (s *SubmitQueue) LoadSeries() error { func (s *SubmitQueue) LoadSeries(log *logrus.Logger) error {
var queryString = fmt.Sprintf("status:open project:%s branch:%s", s.ProjectName, s.BranchName) var queryString = fmt.Sprintf("status:open project:%s branch:%s", s.ProjectName, s.BranchName)
log.Debugf("Running query %s", queryString) log.Debugf("Running query %s", queryString)
@ -41,7 +43,7 @@ func (s *SubmitQueue) LoadSeries() error {
} }
// Assemble to series // Assemble to series
series, err := AssembleSeries(changesets) series, err := AssembleSeries(changesets, log)
if err != nil { if err != nil {
return err return err
} }
@ -75,12 +77,19 @@ func (s *SubmitQueue) IsAutoSubmittable(serie *Serie) bool {
}) })
} }
// GetChangesetURL returns the URL to view a given changeset
func (s *SubmitQueue) GetChangesetURL(changeset *gerrit.Changeset) string {
return fmt.Sprintf("%s/c/%s/+/%d", s.gerrit.GetBaseURL(), s.ProjectName, changeset.Number)
}
// DoSubmit submits changes that can be submitted, // DoSubmit submits changes that can be submitted,
// and updates `Series` to contain the remaining ones // and updates `Series` to contain the remaining ones
// Also updates `HEAD`. // Also updates `HEAD`.
func (s *SubmitQueue) DoSubmit() error { func (s *SubmitQueue) DoSubmit(log *logrus.Logger) error {
var remainingSeries []*Serie var remainingSeries []*Serie
// TODO: actually log more!
for _, serie := range s.Series { for _, serie := range s.Series {
serieParentCommitIDs, err := serie.GetParentCommitIDs() serieParentCommitIDs, err := serie.GetParentCommitIDs()
if err != nil { if err != nil {
@ -124,12 +133,12 @@ func (s *SubmitQueue) DoSubmit() error {
// After a DoRebase, consumers are supposed to fetch state again via LoadSeries, // After a DoRebase, consumers are supposed to fetch state again via LoadSeries,
// as things most likely have changed, and error handling during partially failed rebases // as things most likely have changed, and error handling during partially failed rebases
// is really tricky // is really tricky
func (s *SubmitQueue) DoRebase() error { func (s *SubmitQueue) DoRebase(log *logrus.Logger) error {
if s.HEAD == "" { if s.HEAD == "" {
return fmt.Errorf("current HEAD is an empty string, bailing out") return fmt.Errorf("current HEAD is an empty string, bailing out")
} }
for _, serie := range s.Series { for _, serie := range s.Series {
logger := log.WithFields(log.Fields{ logger := log.WithFields(logrus.Fields{
"serie": serie, "serie": serie,
}) })
if !s.IsAutoSubmittable(serie) { if !s.IsAutoSubmittable(serie) {
@ -157,36 +166,73 @@ func (s *SubmitQueue) DoRebase() error {
return nil return nil
} }
// Problem: no inspection during the run
// Problem: record the state
type Result struct {
LogEntries []*logrus.Entry
Series []Serie
Error error
}
func (r Result) StartTime() time.Time {
return r.LogEntries[0].Time
}
func (r Result) EndTime() time.Time {
return r.LogEntries[len(r.LogEntries)-1].Time
}
func (r *Result) Fire(entry *logrus.Entry) error {
r.LogEntries = append(r.LogEntries, entry)
return nil
}
func (r *Result) Levels() []logrus.Level {
return logrus.AllLevels
}
// Run starts the submit and rebase logic. // Run starts the submit and rebase logic.
func (s *SubmitQueue) Run() error { func (s *SubmitQueue) Run(fetchOnly bool) *Result {
r := &Result{}
//TODO: log decisions made and add to some ring buffer //TODO: log decisions made and add to some ring buffer
var err error var err error
log := logrus.New()
log.AddHook(r)
commitID, err := s.gerrit.GetHEAD(s.ProjectName, s.BranchName) commitID, err := s.gerrit.GetHEAD(s.ProjectName, s.BranchName)
if err != nil { if err != nil {
log.Errorf("Unable to retrieve HEAD of branch %s at project %s: %s", s.BranchName, s.ProjectName, err) log.Errorf("Unable to retrieve HEAD of branch %s at project %s: %s", s.BranchName, s.ProjectName, err)
return err r.Error = err
return r
} }
s.HEAD = commitID s.HEAD = commitID
err = s.LoadSeries() err = s.LoadSeries(log)
if err != nil { if err != nil {
return err r.Error = err
return r
} }
if len(s.Series) == 0 { if len(s.Series) == 0 {
// Nothing to do! // Nothing to do!
log.Warn("Nothing to do here") log.Warn("Nothing to do here")
return nil return r
} }
err = s.DoSubmit() if fetchOnly {
return r
}
err = s.DoSubmit(log)
if err != nil { if err != nil {
return err r.Error = err
return r
} }
err = s.DoRebase() err = s.DoRebase(log)
if err != nil { if err != nil {
return err r.Error = err
return r
} }
return nil return r
} }
// RebaseSerie rebases a whole serie on top of a given ref // RebaseSerie rebases a whole serie on top of a given ref