feat(server): Reimplement local manifest cache backed by files

Implements a local manifest cache that uses the temporary directory to
cache manifest builds.

This is necessary due to the size of manifests: Keeping them entirely
in-memory would quickly balloon the memory usage of Nixery, unless
some mechanism for cache eviction is implemented.

tools/nixery/server/builder/builder.go

@@ -34,6 +34,8 @@ import (
 	"sort"
 	"strings"
 
+	"cloud.google.com/go/storage"
+	"github.com/google/nixery/config"
 	"github.com/google/nixery/layers"
 	"github.com/google/nixery/manifest"
 	"golang.org/x/oauth2/google"
@@ -50,6 +52,15 @@ const gcsScope = "https://www.googleapis.com/auth/devstorage.read_write"
 // HTTP client to use for direct calls to APIs that are not part of the SDK
 var client = &http.Client{}
 
+// State holds the runtime state that is carried around in Nixery and
+// passed to builder functions.
+type State struct {
+	Bucket *storage.BucketHandle
+	Cache  *LocalCache
+	Cfg    config.Config
+	Pop    layers.Popularity
+}
+
 // Image represents the information necessary for building a container image.
 // This can be either a list of package names (corresponding to keys in the
 // nixpkgs set) or a Nix expression that results in a *list* of derivations.

tools/nixery/server/builder/cache.go

@@ -20,6 +20,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"os"
 	"sync"
 
 	"github.com/google/nixery/manifest"
@@ -30,39 +31,63 @@ import (
 type LocalCache struct {
 	// Manifest cache
 	mmtx sync.RWMutex
-	mcache map[string]string
+	mdir string
 
 	// Layer cache
 	lmtx   sync.RWMutex
 	lcache map[string]manifest.Entry
 }
 
-func NewCache() LocalCache {
+// Creates an in-memory cache and ensures that the local file path for
-	return LocalCache{
+// manifest caching exists.
-		mcache: make(map[string]string),
+func NewCache() (LocalCache, error) {
-		lcache: make(map[string]manifest.Entry),
+	path := os.TempDir() + "/nixery"
+	err := os.MkdirAll(path, 0755)
+	if err != nil {
+		return LocalCache{}, err
 	}
+
+	return LocalCache{
+		mdir:   path + "/",
+		lcache: make(map[string]manifest.Entry),
+	}, nil
 }
 
 // Retrieve a cached manifest if the build is cacheable and it exists.
-func (c *LocalCache) manifestFromLocalCache(key string) (string, bool) {
+func (c *LocalCache) manifestFromLocalCache(key string) (json.RawMessage, bool) {
 	c.mmtx.RLock()
-	path, ok := c.mcache[key]
+	defer c.mmtx.RUnlock()
-	c.mmtx.RUnlock()
 
-	if !ok {
+	f, err := os.Open(c.mdir + key)
-		return "", false
+	if err != nil {
+		// TODO(tazjin): Once log levels are available, this
+		// might warrant a debug log.
+		return nil, false
+	}
+	defer f.Close()
+
+	m, err := ioutil.ReadAll(f)
+	if err != nil {
+		log.Printf("Failed to read manifest '%s' from local cache: %s\n", key, err)
+		return nil, false
 	}
 
-	return path, true
+	return json.RawMessage(m), true
 }
 
 // Adds the result of a manifest build to the local cache, if the
 // manifest is considered cacheable.
-func (c *LocalCache) localCacheManifest(key, path string) {
+//
+// Manifests can be quite large and are cached on disk instead of in
+// memory.
+func (c *LocalCache) localCacheManifest(key string, m json.RawMessage) {
 	c.mmtx.Lock()
-	c.mcache[key] = path
+	defer c.mmtx.Unlock()
-	c.mmtx.Unlock()
+
+	err := ioutil.WriteFile(c.mdir+key, []byte(m), 0644)
+	if err != nil {
+		log.Printf("Failed to locally cache manifest for '%s': %s\n", key, err)
+	}
 }
 
 // Retrieve a layer build from the local cache.
@@ -84,11 +109,9 @@ func (c *LocalCache) localCacheLayer(key string, e manifest.Entry) {
 // Retrieve a manifest from the cache(s). First the local cache is
 // checked, then the GCS-bucket cache.
 func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessage, bool) {
-	// path, cached := s.Cache.manifestFromLocalCache(key)
+	if m, cached := s.Cache.manifestFromLocalCache(key); cached {
-	// if cached {
+		return m, true
-	// 	return path, true
+	}
-	// }
-	// TODO: local cache?
 
 	obj := s.Bucket.Object("manifests/" + key)
 
@@ -110,15 +133,15 @@ func manifestFromCache(ctx context.Context, s *State, key string) (json.RawMessa
 		log.Printf("Failed to read cached manifest for '%s': %s\n", key, err)
 	}
 
-	// TODO: locally cache manifest, but the cache needs to be changed
+	go s.Cache.localCacheManifest(key, m)
 	log.Printf("Retrieved manifest for sha1:%s from GCS\n", key)
+
 	return json.RawMessage(m), true
 }
 
 // Add a manifest to the bucket & local caches
 func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage) {
-	// go s.Cache.localCacheManifest(key, path)
+	go s.Cache.localCacheManifest(key, m)
-	// TODO local cache
 
 	obj := s.Bucket.Object("manifests/" + key)
 	w := obj.NewWriter(ctx)

tools/nixery/server/builder/state.go

@@ -1,24 +0,0 @@
-package builder
-
-import (
-	"cloud.google.com/go/storage"
-	"github.com/google/nixery/config"
-	"github.com/google/nixery/layers"
-)
-
-// State holds the runtime state that is carried around in Nixery and
-// passed to builder functions.
-type State struct {
-	Bucket *storage.BucketHandle
-	Cache  LocalCache
-	Cfg    config.Config
-	Pop    layers.Popularity
-}
-
-func NewState(bucket *storage.BucketHandle, cfg config.Config) State {
-	return State{
-		Bucket: bucket,
-		Cfg:    cfg,
-		Cache:  NewCache(),
-	}
-}

tools/nixery/server/config/config.go

@@ -71,13 +71,13 @@ type Config struct {
 	PopUrl  string                    // URL to the Nix package popularity count
 }
 
-func FromEnv() (*Config, error) {
+func FromEnv() (Config, error) {
 	pkgs, err := pkgSourceFromEnv()
 	if err != nil {
-		return nil, err
+		return Config{}, err
 	}
 
-	return &Config{
+	return Config{
 		Bucket:  getConfig("BUCKET", "GCS bucket for layer storage", ""),
 		Port:    getConfig("PORT", "HTTP port", ""),
 		Pkgs:    pkgs,

tools/nixery/server/main.go

@@ -194,8 +194,17 @@ func main() {
 	}
 
 	ctx := context.Background()
-	bucket := prepareBucket(ctx, cfg)
+	bucket := prepareBucket(ctx, &cfg)
-	state := builder.NewState(bucket, *cfg)
+	cache, err := builder.NewCache()
+	if err != nil {
+		log.Fatalln("Failed to instantiate build cache", err)
+	}
+
+	state := builder.State{
+		Bucket: bucket,
+		Cache:  &cache,
+		Cfg:    cfg,
+	}
 
 	log.Printf("Starting Nixery on port %s\n", cfg.Port)