feat(server): Order layers in image manifest based on merge rating
Image layers in manifests are now sorted in a stable (descending) order based on their merge rating, meaning that layers more likely to be shared between images come first. The reason for this change is Docker's handling of image layers on overlayfs2: Images are condensed into a single representation on disk after downloading. Due to this Docker will constantly redownload all layers that are applied in a different order in different images (layer order matters in imperatively created images), based on something it calls the 'ChainID'. Sorting the layers this way raises the likelihood of a long chain of matching layers at the beginning of an image. This relates to #39.
This commit is contained in:
parent
0d820423e9
commit
48a5ecda97
3 changed files with 20 additions and 4 deletions
|
@ -250,6 +250,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
entry.MergeRating = l.MergeRating
|
||||||
|
|
||||||
go cacheLayer(ctx, s, l.Hash(), *entry)
|
go cacheLayer(ctx, s, l.Hash(), *entry)
|
||||||
entries = append(entries, *entry)
|
entries = append(entries, *entry)
|
||||||
|
|
|
@ -141,7 +141,7 @@ type Popularity = map[string]int
|
||||||
// build for the container image.
|
// build for the container image.
|
||||||
type Layer struct {
|
type Layer struct {
|
||||||
Contents []string `json:"contents"`
|
Contents []string `json:"contents"`
|
||||||
mergeRating uint64
|
MergeRating uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// Hash the contents of a layer to create a deterministic identifier that can be
|
// Hash the contents of a layer to create a deterministic identifier that can be
|
||||||
|
@ -153,7 +153,7 @@ func (l *Layer) Hash() string {
|
||||||
|
|
||||||
func (a Layer) merge(b Layer) Layer {
|
func (a Layer) merge(b Layer) Layer {
|
||||||
a.Contents = append(a.Contents, b.Contents...)
|
a.Contents = append(a.Contents, b.Contents...)
|
||||||
a.mergeRating += b.mergeRating
|
a.MergeRating += b.MergeRating
|
||||||
return a
|
return a
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -291,7 +291,7 @@ func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
|
||||||
// both the size and the popularity when making merge
|
// both the size and the popularity when making merge
|
||||||
// decisions, but there might be a smarter way to do
|
// decisions, but there might be a smarter way to do
|
||||||
// it than a plain multiplication.
|
// it than a plain multiplication.
|
||||||
mergeRating: uint64(root.Popularity) * size,
|
MergeRating: uint64(root.Popularity) * size,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -309,7 +309,7 @@ func dominate(budget int, graph *simple.DirectedGraph) []Layer {
|
||||||
}
|
}
|
||||||
|
|
||||||
sort.Slice(layers, func(i, j int) bool {
|
sort.Slice(layers, func(i, j int) bool {
|
||||||
return layers[i].mergeRating < layers[j].mergeRating
|
return layers[i].MergeRating < layers[j].MergeRating
|
||||||
})
|
})
|
||||||
|
|
||||||
if len(layers) > budget {
|
if len(layers) > budget {
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -27,6 +28,10 @@ type Entry struct {
|
||||||
MediaType string `json:"mediaType,omitempty"`
|
MediaType string `json:"mediaType,omitempty"`
|
||||||
Size int64 `json:"size"`
|
Size int64 `json:"size"`
|
||||||
Digest string `json:"digest"`
|
Digest string `json:"digest"`
|
||||||
|
|
||||||
|
// This field is internal to Nixery and not part of the
|
||||||
|
// serialised entry.
|
||||||
|
MergeRating uint64 `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type manifest struct {
|
type manifest struct {
|
||||||
|
@ -85,6 +90,16 @@ func configLayer(hashes []string) ConfigLayer {
|
||||||
//
|
//
|
||||||
// Callers do not need to set the media type for the layer entries.
|
// Callers do not need to set the media type for the layer entries.
|
||||||
func Manifest(layers []Entry) (json.RawMessage, ConfigLayer) {
|
func Manifest(layers []Entry) (json.RawMessage, ConfigLayer) {
|
||||||
|
// Sort layers by their merge rating, from highest to lowest.
|
||||||
|
// This makes it likely for a contiguous chain of shared image
|
||||||
|
// layers to appear at the beginning of a layer.
|
||||||
|
//
|
||||||
|
// Due to moby/moby#38446 Docker considers the order of layers
|
||||||
|
// when deciding which layers to download again.
|
||||||
|
sort.Slice(layers, func(i, j int) bool {
|
||||||
|
return layers[i].MergeRating > layers[j].MergeRating
|
||||||
|
})
|
||||||
|
|
||||||
hashes := make([]string, len(layers))
|
hashes := make([]string, len(layers))
|
||||||
for i, l := range layers {
|
for i, l := range layers {
|
||||||
l.MediaType = "application/vnd.docker.image.rootfs.diff.tar"
|
l.MediaType = "application/vnd.docker.image.rootfs.diff.tar"
|
||||||
|
|
Loading…
Reference in a new issue