feat(storage): Add support for content-types (GCS only)

Extends storage.Persist to accept a Content-Type argument, which in
the GCS backend is persisted with the object to ensure that the object
is served back with this content-type.

This is not yet implemented for the filesystem backend, where the
parameter is simply ignored.

This should help in the case of clients which expect the returned
objects to have content-types set when, for example, fetching layers
by digest.
This commit is contained in:
Vincent Ambo 2020-10-29 16:13:53 +01:00 committed by Vincent Ambo
parent 8a5c446bab
commit cc35bf0fc3
7 changed files with 34 additions and 13 deletions

View file

@ -420,7 +420,7 @@ func (b *byteCounter) Write(p []byte) (n int, err error) {
// image manifest.
func uploadHashLayer(ctx context.Context, s *State, key string, lw layerWriter) (*manifest.Entry, error) {
path := "staging/" + key
sha256sum, size, err := s.Storage.Persist(ctx, path, func(sw io.Writer) (string, int64, error) {
sha256sum, size, err := s.Storage.Persist(ctx, path, manifest.LayerType, func(sw io.Writer) (string, int64, error) {
// Sets up a "multiwriter" that simultaneously runs both hash
// algorithms and uploads to the storage backend.
shasum := sha256.New()

View file

@ -152,7 +152,7 @@ func cacheManifest(ctx context.Context, s *State, key string, m json.RawMessage)
go s.Cache.localCacheManifest(key, m)
path := "manifests/" + key
_, size, err := s.Storage.Persist(ctx, path, func(w io.Writer) (string, int64, error) {
_, size, err := s.Storage.Persist(ctx, path, manifest.ManifestType, func(w io.Writer) (string, int64, error) {
size, err := io.Copy(w, bytes.NewReader([]byte(m)))
return "", size, err
})
@ -220,7 +220,7 @@ func cacheLayer(ctx context.Context, s *State, key string, entry manifest.Entry)
j, _ := json.Marshal(&entry)
path := "builds/" + key
_, _, err := s.Storage.Persist(ctx, path, func(w io.Writer) (string, int64, error) {
_, _, err := s.Storage.Persist(ctx, path, "", func(w io.Writer) (string, int64, error) {
size, err := io.Copy(w, bytes.NewReader(j))
return "", size, err
})

View file

@ -38,6 +38,7 @@ import (
"github.com/google/nixery/builder"
"github.com/google/nixery/config"
"github.com/google/nixery/logs"
mf "github.com/google/nixery/manifest"
"github.com/google/nixery/storage"
log "github.com/sirupsen/logrus"
)
@ -170,7 +171,7 @@ func (h *registryHandler) serveManifestTag(w http.ResponseWriter, r *http.Reques
path := "layers/" + sha256sum
ctx := context.TODO()
_, _, err = h.state.Storage.Persist(ctx, path, func(sw io.Writer) (string, int64, error) {
_, _, err = h.state.Storage.Persist(ctx, path, mf.ManifestType, func(sw io.Writer) (string, int64, error) {
// We already know the hash, so no additional hash needs to be
// constructed here.
written, err := sw.Write(manifest)

View file

@ -28,8 +28,8 @@ const (
schemaVersion = 2
// media types
manifestType = "application/vnd.docker.distribution.manifest.v2+json"
layerType = "application/vnd.docker.image.rootfs.diff.tar.gzip"
ManifestType = "application/vnd.docker.distribution.manifest.v2+json"
LayerType = "application/vnd.docker.image.rootfs.diff.tar.gzip"
configType = "application/vnd.docker.container.image.v1+json"
// image config constants
@ -117,7 +117,7 @@ func Manifest(arch string, layers []Entry) (json.RawMessage, ConfigLayer) {
hashes := make([]string, len(layers))
for i, l := range layers {
hashes[i] = l.TarHash
l.MediaType = layerType
l.MediaType = LayerType
l.TarHash = ""
layers[i] = l
}
@ -126,7 +126,7 @@ func Manifest(arch string, layers []Entry) (json.RawMessage, ConfigLayer) {
m := manifest{
SchemaVersion: schemaVersion,
MediaType: manifestType,
MediaType: ManifestType,
Config: Entry{
MediaType: configType,
Size: int64(len(c.Config)),

View file

@ -49,7 +49,8 @@ func (b *FSBackend) Name() string {
return fmt.Sprintf("Filesystem (%s)", b.path)
}
func (b *FSBackend) Persist(ctx context.Context, key string, f Persister) (string, int64, error) {
// TODO(tazjin): Implement support for persisting content-types for the filesystem backend.
func (b *FSBackend) Persist(ctx context.Context, key, _type string, f Persister) (string, int64, error) {
full := path.Join(b.path, key)
dir := path.Dir(full)
err := os.MkdirAll(dir, 0755)

View file

@ -80,17 +80,36 @@ func (b *GCSBackend) Name() string {
return "Google Cloud Storage (" + b.bucket + ")"
}
func (b *GCSBackend) Persist(ctx context.Context, path string, f Persister) (string, int64, error) {
func (b *GCSBackend) Persist(ctx context.Context, path, contentType string, f Persister) (string, int64, error) {
obj := b.handle.Object(path)
w := obj.NewWriter(ctx)
hash, size, err := f(w)
if err != nil {
log.WithError(err).WithField("path", path).Error("failed to upload to GCS")
log.WithError(err).WithField("path", path).Error("failed to write to GCS")
return hash, size, err
}
return hash, size, w.Close()
err = w.Close()
if err != nil {
log.WithError(err).WithField("path", path).Error("failed to complete GCS upload")
return hash, size, err
}
// GCS natively supports content types for objects, which will be
// used when serving them back.
if contentType != "" {
_, err = obj.Update(ctx, storage.ObjectAttrsToUpdate{
ContentType: contentType,
})
if err != nil {
log.WithError(err).WithField("path", path).Error("failed to update object attrs")
return hash, size, err
}
}
return hash, size, nil
}
func (b *GCSBackend) Fetch(ctx context.Context, path string) (io.ReadCloser, error) {

View file

@ -36,7 +36,7 @@ type Backend interface {
// It needs to return the SHA256 hash of the data written as
// well as the total number of bytes, as those are required
// for the image manifest.
Persist(context.Context, string, Persister) (string, int64, error)
Persist(ctx context.Context, path, contentType string, f Persister) (string, int64, error)
// Fetch retrieves data from the storage backend.
Fetch(ctx context.Context, path string) (io.ReadCloser, error)