feat(server): Cache built manifests to the GCS bucket

Caches manifests under `manifests/$cacheKey` in the GCS bucket and
introduces two-tiered retrieval of manifests from the caches (local
first, bucket second).

There is some cleanup to be done in this code, but the initial version
works.
This commit is contained in:
Vincent Ambo 2019-09-09 16:41:52 +01:00 committed by Vincent Ambo
parent 051eb77b3d
commit 4a58b0ab4d
3 changed files with 97 additions and 24 deletions

View file

@ -109,8 +109,8 @@ func convenienceNames(packages []string) []string {
// Call out to Nix and request that an image be built. Nix will, upon success,
// return a manifest for the container image.
func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) {
resultFile, cached := cache.manifestFromCache(cfg.Pkgs, image)
func BuildImage(ctx *context.Context, cfg *config.Config, cache *LocalCache, image *Image, bucket *storage.BucketHandle) (*BuildResult, error) {
resultFile, cached := manifestFromCache(ctx, bucket, cfg.Pkgs, cache, image)
if !cached {
packages, err := json.Marshal(image.Packages)
@ -158,7 +158,7 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima
log.Println("Finished Nix image build")
resultFile = strings.TrimSpace(string(stdout))
cache.cacheManifest(cfg.Pkgs, image, resultFile)
cacheManifest(ctx, bucket, cfg.Pkgs, cache, image, resultFile)
}
buildOutput, err := ioutil.ReadFile(resultFile)

View file

@ -14,13 +14,21 @@
package builder
import (
"github.com/google/nixery/config"
"context"
"io"
"log"
"os"
"sync"
"cloud.google.com/go/storage"
"github.com/google/nixery/config"
)
type void struct{}
type BuildCache struct {
// LocalCache implements the structure used for local caching of
// manifests and layer uploads.
type LocalCache struct {
mmtx sync.RWMutex
mcache map[string]string
@ -28,8 +36,8 @@ type BuildCache struct {
lcache map[string]void
}
func NewCache() BuildCache {
return BuildCache{
func NewCache() LocalCache {
return LocalCache{
mcache: make(map[string]string),
lcache: make(map[string]void),
}
@ -38,7 +46,7 @@ func NewCache() BuildCache {
// Has this layer hash already been seen by this Nixery instance? If
// yes, we can skip upload checking and such because it has already
// been done.
func (c *BuildCache) hasSeenLayer(hash string) bool {
func (c *LocalCache) hasSeenLayer(hash string) bool {
c.lmtx.RLock()
defer c.lmtx.RUnlock()
_, seen := c.lcache[hash]
@ -46,19 +54,14 @@ func (c *BuildCache) hasSeenLayer(hash string) bool {
}
// Layer has now been seen and should be stored.
func (c *BuildCache) sawLayer(hash string) {
func (c *LocalCache) sawLayer(hash string) {
c.lmtx.Lock()
defer c.lmtx.Unlock()
c.lcache[hash] = void{}
}
// Retrieve a cached manifest if the build is cacheable and it exists.
func (c *BuildCache) manifestFromCache(src config.PkgSource, image *Image) (string, bool) {
key := src.CacheKey(image.Packages, image.Tag)
if key == "" {
return "", false
}
func (c *LocalCache) manifestFromLocalCache(key string) (string, bool) {
c.mmtx.RLock()
path, ok := c.mcache[key]
c.mmtx.RUnlock()
@ -70,15 +73,85 @@ func (c *BuildCache) manifestFromCache(src config.PkgSource, image *Image) (stri
return path, true
}
// Adds the result of a manifest build to the cache, if the manifest
// is considered cacheable.
func (c *BuildCache) cacheManifest(src config.PkgSource, image *Image, path string) {
key := src.CacheKey(image.Packages, image.Tag)
if key == "" {
return
}
// Adds the result of a manifest build to the local cache, if the
// manifest is considered cacheable.
func (c *LocalCache) localCacheManifest(key, path string) {
c.mmtx.Lock()
c.mcache[key] = path
c.mmtx.Unlock()
}
// Retrieve a manifest from the cache(s). First the local cache is
// checked, then the GCS-bucket cache.
func manifestFromCache(ctx *context.Context, bucket *storage.BucketHandle, pkgs config.PkgSource, cache *LocalCache, image *Image) (string, bool) {
key := pkgs.CacheKey(image.Packages, image.Tag)
if key == "" {
return "", false
}
path, cached := cache.manifestFromLocalCache(key)
if cached {
return path, true
}
obj := bucket.Object("manifests/" + key)
// Probe whether the file exists before trying to fetch it.
_, err := obj.Attrs(*ctx)
if err != nil {
return "", false
}
r, err := obj.NewReader(*ctx)
if err != nil {
log.Printf("Failed to retrieve manifest '%s' from cache: %s\n", key, err)
return "", false
}
defer r.Close()
path = os.TempDir() + "/" + key
f, _ := os.Create(path)
defer f.Close()
_, err = io.Copy(f, r)
if err != nil {
log.Printf("Failed to read cached manifest for '%s': %s\n", key, err)
}
log.Printf("Retrieved manifest for '%s' (%s) from GCS\n", image.Name, key)
cache.localCacheManifest(key, path)
return path, true
}
func cacheManifest(ctx *context.Context, bucket *storage.BucketHandle, pkgs config.PkgSource, cache *LocalCache, image *Image, path string) {
key := pkgs.CacheKey(image.Packages, image.Tag)
if key == "" {
return
}
cache.localCacheManifest(key, path)
obj := bucket.Object("manifests/" + key)
w := obj.NewWriter(*ctx)
f, err := os.Open(path)
if err != nil {
log.Printf("failed to open '%s' manifest for cache upload: %s\n", image.Name, err)
return
}
defer f.Close()
size, err := io.Copy(w, f)
if err != nil {
log.Printf("failed to cache manifest sha1:%s: %s\n", key, err)
return
}
if err = w.Close(); err != nil {
log.Printf("failed to cache manifest sha1:%s: %s\n", key, err)
return
}
log.Printf("Cached manifest sha1:%s (%v bytes written)\n", key, size)
}

View file

@ -125,7 +125,7 @@ type registryHandler struct {
cfg *config.Config
ctx *context.Context
bucket *storage.BucketHandle
cache *builder.BuildCache
cache *builder.LocalCache
}
func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {