refactor: Remove remaining MD5-hash mentions and computations
This commit is contained in:
parent
f6b40ed6c7
commit
53906024ff
5 changed files with 18 additions and 30 deletions
|
@ -137,11 +137,10 @@ let
|
|||
buildInputs = with pkgs; [ coreutils jq openssl ];
|
||||
}''
|
||||
layerSha256=$(sha256sum ${symlinkLayer} | cut -d ' ' -f1)
|
||||
layerMd5=$(openssl dgst -md5 -binary ${symlinkLayer} | openssl enc -base64)
|
||||
layerSize=$(stat --printf '%s' ${symlinkLayer})
|
||||
|
||||
jq -n -c --arg sha256 $layerSha256 --arg md5 $layerMd5 --arg size $layerSize --arg path ${symlinkLayer} \
|
||||
'{ size: ($size | tonumber), sha256: $sha256, md5: $md5, path: $path }' >> $out
|
||||
jq -n -c --arg sha256 $layerSha256 --arg size $layerSize --arg path ${symlinkLayer} \
|
||||
'{ size: ($size | tonumber), sha256: $sha256, path: $path }' >> $out
|
||||
''));
|
||||
|
||||
# Final output structure returned to Nixery if the build succeeded
|
||||
|
|
|
@ -46,9 +46,8 @@ They are stored content-addressably at `$BUCKET/layers/$SHA256HASH` and layer
|
|||
requests sent to Nixery will redirect directly to this storage location.
|
||||
|
||||
The effect of this cache is that Nixery does not need to upload identical layers
|
||||
repeatedly. When Nixery notices that a layer already exists in GCS, it will use
|
||||
the object metadata to compare its MD5-hash with the locally computed one and
|
||||
skip uploading.
|
||||
repeatedly. When Nixery notices that a layer already exists in GCS it will skip
|
||||
uploading this layer.
|
||||
|
||||
Removing layers from the cache is *potentially problematic* if there are cached
|
||||
manifests or layer builds referencing those layers.
|
||||
|
@ -61,8 +60,8 @@ reference these layers.
|
|||
Layer builds are cached at `$BUCKET/builds/$HASH`, where `$HASH` is a SHA1 of
|
||||
the Nix store paths included in the layer.
|
||||
|
||||
The content of the cached entries is a JSON-object that contains the MD5 and
|
||||
SHA256 hashes of the built layer.
|
||||
The content of the cached entries is a JSON-object that contains the SHA256
|
||||
hashes and sizes of the built layer.
|
||||
|
||||
The effect of this cache is that different instances of Nixery will not build,
|
||||
hash and upload layers that have identical contents across different instances.
|
||||
|
|
|
@ -67,8 +67,7 @@ just ... hang, for a moment.
|
|||
|
||||
Nixery inspects the returned manifest and uploads each layer to the configured
|
||||
[Google Cloud Storage][gcs] bucket. To avoid unnecessary uploading, it will
|
||||
first check whether layers are already present in the bucket and - just to be
|
||||
safe - compare their MD5-hashes against what was built.
|
||||
check whether layers are already present in the bucket.
|
||||
|
||||
## 4. The image manifest is sent back
|
||||
|
||||
|
|
|
@ -21,7 +21,6 @@ import (
|
|||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
@ -108,7 +107,6 @@ type ImageResult struct {
|
|||
SymlinkLayer struct {
|
||||
Size int `json:"size"`
|
||||
SHA256 string `json:"sha256"`
|
||||
MD5 string `json:"md5"`
|
||||
Path string `json:"path"`
|
||||
} `json:"symlinkLayer"`
|
||||
}
|
||||
|
@ -328,8 +326,7 @@ func uploadHashLayer(ctx context.Context, s *State, key string, data io.Reader)
|
|||
// algorithms and uploads to the bucket
|
||||
sw := staging.NewWriter(ctx)
|
||||
shasum := sha256.New()
|
||||
md5sum := md5.New()
|
||||
multi := io.MultiWriter(sw, shasum, md5sum)
|
||||
multi := io.MultiWriter(sw, shasum)
|
||||
|
||||
size, err := io.Copy(multi, data)
|
||||
if err != nil {
|
||||
|
@ -342,27 +339,24 @@ func uploadHashLayer(ctx context.Context, s *State, key string, data io.Reader)
|
|||
return nil, err
|
||||
}
|
||||
|
||||
build := Build{
|
||||
SHA256: fmt.Sprintf("%x", shasum.Sum([]byte{})),
|
||||
MD5: fmt.Sprintf("%x", md5sum.Sum([]byte{})),
|
||||
}
|
||||
sha256sum := fmt.Sprintf("%x", shasum.Sum([]byte{}))
|
||||
|
||||
// Hashes are now known and the object is in the bucket, what
|
||||
// remains is to move it to the correct location and cache it.
|
||||
err = renameObject(ctx, s, "staging/"+key, "layers/"+build.SHA256)
|
||||
err = renameObject(ctx, s, "staging/"+key, "layers/"+sha256sum)
|
||||
if err != nil {
|
||||
log.Printf("failed to move layer '%s' from staging: %s\n", key, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cacheBuild(ctx, s, key, build)
|
||||
log.Printf("Uploaded layer sha256:%s (%v bytes written)", sha256sum, size)
|
||||
|
||||
log.Printf("Uploaded layer sha256:%s (%v bytes written)", build.SHA256, size)
|
||||
|
||||
return &manifest.Entry{
|
||||
Digest: "sha256:" + build.SHA256,
|
||||
entry := manifest.Entry{
|
||||
Digest: "sha256:" + sha256sum,
|
||||
Size: size,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return &entry, nil
|
||||
}
|
||||
|
||||
func BuildImage(ctx context.Context, s *State, image *Image) (*BuildResult, error) {
|
||||
|
|
|
@ -3,7 +3,6 @@
|
|||
package manifest
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
@ -52,12 +51,11 @@ type imageConfig struct {
|
|||
}
|
||||
|
||||
// ConfigLayer represents the configuration layer to be included in
|
||||
// the manifest, containing its JSON-serialised content and the SHA256
|
||||
// & MD5 hashes of its input.
|
||||
// the manifest, containing its JSON-serialised content and SHA256
|
||||
// hash.
|
||||
type ConfigLayer struct {
|
||||
Config []byte
|
||||
SHA256 string
|
||||
MD5 string
|
||||
}
|
||||
|
||||
// imageConfig creates an image configuration with the values set to
|
||||
|
@ -78,7 +76,6 @@ func configLayer(hashes []string) ConfigLayer {
|
|||
return ConfigLayer{
|
||||
Config: j,
|
||||
SHA256: fmt.Sprintf("%x", sha256.Sum256(j)),
|
||||
MD5: fmt.Sprintf("%x", md5.Sum(j)),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue