refactor(server): Change setup to create new storage backends

tools/nixery/server/builder/builder.go

@@ -27,7 +27,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"os/exec"
 	"sort"
@@ -45,9 +44,6 @@ import (
 // use up is set at a lower point.
 const LayerBudget int = 94
 
-// HTTP client to use for direct calls to APIs that are not part of the SDK
-var client = &http.Client{}
-
 // State holds the runtime state that is carried around in Nixery and
 // passed to builder functions.
 type State struct {

tools/nixery/server/builder/cache.go

@@ -125,7 +125,7 @@ func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessa
 		log.WithError(err).WithFields(log.Fields{
 			"manifest": key,
 			"backend":  s.Storage.Name(),
-		})
+		}).Error("failed to fetch manifest from cache")
 
 		return nil, false
 	}

tools/nixery/server/config/config.go

@@ -37,6 +37,13 @@ func getConfig(key, desc, def string) string {
 	return value
 }
 
+// Backend represents the possible storage backend types
+type Backend int
+
+const (
+	GCS = iota
+)
+
 // Config holds the Nixery configuration options.
 type Config struct {
 	Port    string    // Port on which to launch HTTP server
@@ -44,6 +51,7 @@ type Config struct {
 	Timeout string    // Timeout for a single Nix builder (seconds)
 	WebDir  string    // Directory with static web assets
 	PopUrl  string    // URL to the Nix package popularity count
+	Backend Backend   // Storage backend to use for Nixery
 }
 
 func FromEnv() (Config, error) {
@@ -52,11 +60,22 @@ func FromEnv() (Config, error) {
 		return Config{}, err
 	}
 
+	var b Backend
+	switch os.Getenv("NIXERY_STORAGE_BACKEND") {
+	case "gcs":
+		b = GCS
+	default:
+		log.WithField("values", []string{
+			"gcs",
+		}).Fatal("NIXERY_STORAGE_BUCKET must be set to a supported value")
+	}
+
 	return Config{
 		Port:    getConfig("PORT", "HTTP port", ""),
 		Pkgs:    pkgs,
 		Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"),
 		WebDir:  getConfig("WEB_DIR", "Static web file dir", ""),
 		PopUrl:  os.Getenv("NIX_POPULARITY_URL"),
+		Backend: b,
 	}, nil
 }

tools/nixery/server/main.go

@@ -36,6 +36,7 @@ import (
 	"github.com/google/nixery/server/builder"
 	"github.com/google/nixery/server/config"
 	"github.com/google/nixery/server/layers"
+	"github.com/google/nixery/server/storage"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -196,6 +197,18 @@ func main() {
 		log.WithError(err).Fatal("failed to load configuration")
 	}
 
+	var s storage.Backend
+
+	switch cfg.Backend {
+	case config.GCS:
+		s, err = storage.NewGCSBackend()
+	}
+	if err != nil {
+		log.WithError(err).Fatal("failed to initialise storage backend")
+	}
+
+	log.WithField("backend", s.Name()).Info("initialised storage backend")
+
 	ctx := context.Background()
 	cache, err := builder.NewCache()
 	if err != nil {
@@ -212,9 +225,10 @@ func main() {
 	}
 
 	state := builder.State{
-		Cache: &cache,
-		Cfg:   cfg,
-		Pop:   pop,
+		Cache:   &cache,
+		Cfg:     cfg,
+		Pop:     pop,
+		Storage: s,
 	}
 
 	log.WithFields(log.Fields{

tools/nixery/server/storage/gcs.go

@@ -30,10 +30,10 @@ type GCSBackend struct {
 
 // Constructs a new GCS bucket backend based on the configured
 // environment variables.
-func New() (GCSBackend, error) {
+func NewGCSBackend() (*GCSBackend, error) {
 	bucket := os.Getenv("GCS_BUCKET")
 	if bucket == "" {
-		return GCSBackend{}, fmt.Errorf("GCS_BUCKET must be configured for GCS usage")
+		return nil, fmt.Errorf("GCS_BUCKET must be configured for GCS usage")
 	}
 
 	ctx := context.Background()
@@ -46,16 +46,16 @@ func New() (GCSBackend, error) {
 
 	if _, err := handle.Attrs(ctx); err != nil {
 		log.WithError(err).WithField("bucket", bucket).Error("could not access configured bucket")
-		return GCSBackend{}, err
+		return nil, err
 	}
 
 	signing, err := signingOptsFromEnv()
 	if err != nil {
 		log.WithError(err).Error("failed to configure GCS bucket signing")
-		return GCSBackend{}, err
+		return nil, err
 	}
 
-	return GCSBackend{
+	return &GCSBackend{
 		bucket:  bucket,
 		handle:  handle,
 		signing: signing,
@@ -66,7 +66,7 @@ func (b *GCSBackend) Name() string {
 	return "Google Cloud Storage (" + b.bucket + ")"
 }
 
-func (b *GCSBackend) Persist(path string, f func(io.Writer) (string, int, error)) (string, int, error) {
+func (b *GCSBackend) Persist(path string, f func(io.Writer) (string, int64, error)) (string, int64, error) {
 	ctx := context.Background()
 	obj := b.handle.Object(path)
 	w := obj.NewWriter(ctx)
@@ -139,7 +139,7 @@ func (b *GCSBackend) Move(old, new string) error {
 	return nil
 }
 
-func (b *GCSBackend) Serve(digest string, w http.ResponseWriter) error {
+func (b *GCSBackend) ServeLayer(digest string, w http.ResponseWriter) error {
 	url, err := b.constructLayerUrl(digest)
 	if err != nil {
 		log.WithError(err).WithFields(log.Fields{