refactor: Reshuffle file structure for better code layout
This gets rid of the package called "server" and instead moves everything into the project root, such that Go actually builds us a binary called `nixery`. This is the first step towards factoring out CLI-based functionality for Nixery.
This commit is contained in:
parent
df88da126a
commit
2b82f1b71a
21 changed files with 83 additions and 114 deletions
|
@ -19,7 +19,6 @@ package builder
|
|||
// The tarball is written straight to the supplied reader, which makes it
|
||||
// possible to create an image layer from the specified store paths, hash it and
|
||||
// upload it in one reading pass.
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
|
@ -28,8 +27,6 @@ import (
|
|||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/google/nixery/server/layers"
|
||||
)
|
||||
|
||||
// Create a new compressed tarball from each of the paths in the list
|
||||
|
@ -37,7 +34,7 @@ import (
|
|||
//
|
||||
// The uncompressed tarball is hashed because image manifests must
|
||||
// contain both the hashes of compressed and uncompressed layers.
|
||||
func packStorePaths(l *layers.Layer, w io.Writer) (string, error) {
|
||||
func packStorePaths(l *layer, w io.Writer) (string, error) {
|
||||
shasum := sha256.New()
|
||||
gz := gzip.NewWriter(w)
|
||||
multi := io.MultiWriter(shasum, gz)
|
|
@ -12,9 +12,10 @@
|
|||
// License for the specific language governing permissions and limitations under
|
||||
// the License.
|
||||
|
||||
// Package builder implements the code required to build images via Nix. Image
|
||||
// build data is cached for up to 24 hours to avoid duplicated calls to Nix
|
||||
// (which are costly even if no building is performed).
|
||||
// Package builder implements the logic for assembling container
|
||||
// images. It shells out to Nix to retrieve all required Nix-packages
|
||||
// and assemble the symlink layer and then creates the required
|
||||
// tarballs in-process.
|
||||
package builder
|
||||
|
||||
import (
|
||||
|
@ -32,10 +33,9 @@ import (
|
|||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/google/nixery/server/config"
|
||||
"github.com/google/nixery/server/layers"
|
||||
"github.com/google/nixery/server/manifest"
|
||||
"github.com/google/nixery/server/storage"
|
||||
"github.com/google/nixery/config"
|
||||
"github.com/google/nixery/manifest"
|
||||
"github.com/google/nixery/storage"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
|
@ -50,7 +50,7 @@ type State struct {
|
|||
Storage storage.Backend
|
||||
Cache *LocalCache
|
||||
Cfg config.Config
|
||||
Pop layers.Popularity
|
||||
Pop Popularity
|
||||
}
|
||||
|
||||
// Architecture represents the possible CPU architectures for which
|
||||
|
@ -128,7 +128,7 @@ type ImageResult struct {
|
|||
Pkgs []string `json:"pkgs"`
|
||||
|
||||
// These fields are populated in case of success
|
||||
Graph layers.RuntimeGraph `json:"runtimeGraph"`
|
||||
Graph runtimeGraph `json:"runtimeGraph"`
|
||||
SymlinkLayer struct {
|
||||
Size int `json:"size"`
|
||||
TarHash string `json:"tarHash"`
|
||||
|
@ -265,7 +265,7 @@ func prepareImage(s *State, image *Image) (*ImageResult, error) {
|
|||
"--argstr", "system", image.Arch.nixSystem,
|
||||
}
|
||||
|
||||
output, err := callNix("nixery-build-image", image.Name, args)
|
||||
output, err := callNix("nixery-prepare-image", image.Name, args)
|
||||
if err != nil {
|
||||
// granular error logging is performed in callNix already
|
||||
return nil, err
|
||||
|
@ -292,7 +292,7 @@ func prepareImage(s *State, image *Image) (*ImageResult, error) {
|
|||
// added only after successful uploads, which guarantees that entries
|
||||
// retrieved from the cache are present in the bucket.
|
||||
func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageResult) ([]manifest.Entry, error) {
|
||||
grouped := layers.Group(&result.Graph, &s.Pop, LayerBudget)
|
||||
grouped := groupLayers(&result.Graph, &s.Pop, LayerBudget)
|
||||
|
||||
var entries []manifest.Entry
|
||||
|
||||
|
@ -329,7 +329,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
|
|||
|
||||
var pkgs []string
|
||||
for _, p := range l.Contents {
|
||||
pkgs = append(pkgs, layers.PackageFromPath(p))
|
||||
pkgs = append(pkgs, packageFromPath(p))
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
|
@ -22,7 +22,7 @@ import (
|
|||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/google/nixery/server/manifest"
|
||||
"github.com/google/nixery/manifest"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
|
@ -114,7 +114,7 @@
|
|||
//
|
||||
// Layer budget: 10
|
||||
// Layers: { E }, { D, F }, { A }, { B }, { C }
|
||||
package layers
|
||||
package builder
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
|
@ -128,11 +128,11 @@ import (
|
|||
"gonum.org/v1/gonum/graph/simple"
|
||||
)
|
||||
|
||||
// RuntimeGraph represents structured information from Nix about the runtime
|
||||
// runtimeGraph represents structured information from Nix about the runtime
|
||||
// dependencies of a derivation.
|
||||
//
|
||||
// This is generated in Nix by using the exportReferencesGraph feature.
|
||||
type RuntimeGraph struct {
|
||||
type runtimeGraph struct {
|
||||
References struct {
|
||||
Graph []string `json:"graph"`
|
||||
} `json:"exportReferencesGraph"`
|
||||
|
@ -153,19 +153,19 @@ type Popularity = map[string]int
|
|||
|
||||
// Layer represents the data returned for each layer that Nix should
|
||||
// build for the container image.
|
||||
type Layer struct {
|
||||
type layer struct {
|
||||
Contents []string `json:"contents"`
|
||||
MergeRating uint64
|
||||
}
|
||||
|
||||
// Hash the contents of a layer to create a deterministic identifier that can be
|
||||
// used for caching.
|
||||
func (l *Layer) Hash() string {
|
||||
func (l *layer) Hash() string {
|
||||
sum := sha1.Sum([]byte(strings.Join(l.Contents, ":")))
|
||||
return fmt.Sprintf("%x", sum)
|
||||
}
|
||||
|
||||
func (a Layer) merge(b Layer) Layer {
|
||||
func (a layer) merge(b layer) layer {
|
||||
a.Contents = append(a.Contents, b.Contents...)
|
||||
a.MergeRating += b.MergeRating
|
||||
return a
|
||||
|
@ -188,12 +188,15 @@ var nixRegexp = regexp.MustCompile(`^/nix/store/[a-z0-9]+-`)
|
|||
|
||||
// PackageFromPath returns the name of a Nix package based on its
|
||||
// output store path.
|
||||
func PackageFromPath(path string) string {
|
||||
func packageFromPath(path string) string {
|
||||
return nixRegexp.ReplaceAllString(path, "")
|
||||
}
|
||||
|
||||
// DOTID provides a human-readable package name. The name stems from
|
||||
// the dot format used by GraphViz, into which the dependency graph
|
||||
// can be rendered.
|
||||
func (c *closure) DOTID() string {
|
||||
return PackageFromPath(c.Path)
|
||||
return packageFromPath(c.Path)
|
||||
}
|
||||
|
||||
// bigOrPopular checks whether this closure should be considered for
|
||||
|
@ -236,7 +239,7 @@ func insertEdges(graph *simple.DirectedGraph, cmap *map[string]*closure, node *c
|
|||
}
|
||||
|
||||
// Create a graph structure from the references supplied by Nix.
|
||||
func buildGraph(refs *RuntimeGraph, pop *Popularity) *simple.DirectedGraph {
|
||||
func buildGraph(refs *runtimeGraph, pop *Popularity) *simple.DirectedGraph {
|
||||
cmap := make(map[string]*closure)
|
||||
graph := simple.NewDirectedGraph()
|
||||
|
||||
|
@ -296,7 +299,7 @@ func buildGraph(refs *RuntimeGraph, pop *Popularity) *simple.DirectedGraph {
|
|||
// Extracts a subgraph starting at the specified root from the
|
||||
// dominator tree. The subgraph is converted into a flat list of
|
||||
// layers, each containing the store paths and merge rating.
|
||||
func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
|
||||
func groupLayer(dt *flow.DominatorTree, root *closure) layer {
|
||||
size := root.Size
|
||||
contents := []string{root.Path}
|
||||
children := dt.DominatedBy(root.ID())
|
||||
|
@ -313,7 +316,7 @@ func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
|
|||
// Contents are sorted to ensure that hashing is consistent
|
||||
sort.Strings(contents)
|
||||
|
||||
return Layer{
|
||||
return layer{
|
||||
Contents: contents,
|
||||
MergeRating: uint64(root.Popularity) * size,
|
||||
}
|
||||
|
@ -324,10 +327,10 @@ func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
|
|||
//
|
||||
// Layers are merged together until they fit into the layer budget,
|
||||
// based on their merge rating.
|
||||
func dominate(budget int, graph *simple.DirectedGraph) []Layer {
|
||||
func dominate(budget int, graph *simple.DirectedGraph) []layer {
|
||||
dt := flow.Dominators(graph.Node(0), graph)
|
||||
|
||||
var layers []Layer
|
||||
var layers []layer
|
||||
for _, n := range dt.DominatedBy(dt.Root().ID()) {
|
||||
layers = append(layers, groupLayer(&dt, n.(*closure)))
|
||||
}
|
||||
|
@ -352,10 +355,10 @@ func dominate(budget int, graph *simple.DirectedGraph) []Layer {
|
|||
return layers
|
||||
}
|
||||
|
||||
// GroupLayers applies the algorithm described above the its input and returns a
|
||||
// groupLayers applies the algorithm described above the its input and returns a
|
||||
// list of layers, each consisting of a list of Nix store paths that it should
|
||||
// contain.
|
||||
func Group(refs *RuntimeGraph, pop *Popularity, budget int) []Layer {
|
||||
func groupLayers(refs *runtimeGraph, pop *Popularity, budget int) []layer {
|
||||
graph := buildGraph(refs, pop)
|
||||
return dominate(budget, graph)
|
||||
}
|
|
@ -20,6 +20,8 @@
|
|||
with pkgs;
|
||||
|
||||
let
|
||||
inherit (pkgs) buildGoPackage;
|
||||
|
||||
# Hash of all Nixery sources - this is used as the Nixery version in
|
||||
# builds to distinguish errors between deployed versions, see
|
||||
# server/logs.go for details.
|
||||
|
@ -30,13 +32,41 @@ let
|
|||
# Go implementation of the Nixery server which implements the
|
||||
# container registry interface.
|
||||
#
|
||||
# Users should use the nixery-bin derivation below instead.
|
||||
nixery-server = callPackage ./server {
|
||||
srcHash = nixery-src-hash;
|
||||
# Users should use the nixery-bin derivation below instead as it
|
||||
# provides the paths of files needed at runtime.
|
||||
nixery-server = buildGoPackage rec {
|
||||
name = "nixery-server";
|
||||
goDeps = ./go-deps.nix;
|
||||
src = ./.;
|
||||
|
||||
goPackagePath = "github.com/google/nixery";
|
||||
doCheck = true;
|
||||
|
||||
# Simplify the Nix build instructions for Go to just the basics
|
||||
# required to get Nixery up and running with the additional linker
|
||||
# flags required.
|
||||
outputs = [ "out" ];
|
||||
preConfigure = "bin=$out";
|
||||
buildPhase = ''
|
||||
runHook preBuild
|
||||
runHook renameImport
|
||||
|
||||
export GOBIN="$out/bin"
|
||||
go install -ldflags "-X main.version=$(cat ${nixery-src-hash})" ${goPackagePath}
|
||||
'';
|
||||
|
||||
fixupPhase = ''
|
||||
remove-references-to -t ${go} $out/bin/nixery
|
||||
'';
|
||||
|
||||
checkPhase = ''
|
||||
go vet ${goPackagePath}
|
||||
go test ${goPackagePath}
|
||||
'';
|
||||
};
|
||||
in rec {
|
||||
# Implementation of the Nix image building logic
|
||||
nixery-build-image = import ./build-image { inherit pkgs; };
|
||||
nixery-prepare-image = import ./prepare-image { inherit pkgs; };
|
||||
|
||||
# Use mdBook to build a static asset page which Nixery can then
|
||||
# serve. This is primarily used for the public instance at
|
||||
|
@ -50,8 +80,8 @@ in rec {
|
|||
# are installing Nixery directly.
|
||||
nixery-bin = writeShellScriptBin "nixery" ''
|
||||
export WEB_DIR="${nixery-book}"
|
||||
export PATH="${nixery-build-image}/bin:$PATH"
|
||||
exec ${nixery-server}/bin/server
|
||||
export PATH="${nixery-prepare-image}/bin:$PATH"
|
||||
exec ${nixery-server}/bin/nixery
|
||||
'';
|
||||
|
||||
nixery-popcount = callPackage ./popcount { };
|
||||
|
@ -104,7 +134,7 @@ in rec {
|
|||
gzip
|
||||
iana-etc
|
||||
nix
|
||||
nixery-build-image
|
||||
nixery-prepare-image
|
||||
nixery-launch-script
|
||||
openssh
|
||||
zlib
|
||||
|
|
|
@ -11,7 +11,7 @@
|
|||
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
// License for the specific language governing permissions and limitations under
|
||||
// the License.
|
||||
package main
|
||||
package logs
|
||||
|
||||
// This file configures different log formatters via logrus. The
|
||||
// standard formatter uses a structured JSON format that is compatible
|
||||
|
@ -112,7 +112,7 @@ func (f stackdriverFormatter) Format(e *log.Entry) ([]byte, error) {
|
|||
return b.Bytes(), err
|
||||
}
|
||||
|
||||
func init() {
|
||||
func Init(version string) {
|
||||
nixeryContext.Version = version
|
||||
log.SetReportCaller(true)
|
||||
log.SetFormatter(stackdriverFormatter{})
|
|
@ -32,10 +32,10 @@ import (
|
|||
"net/http"
|
||||
"regexp"
|
||||
|
||||
"github.com/google/nixery/server/builder"
|
||||
"github.com/google/nixery/server/config"
|
||||
"github.com/google/nixery/server/layers"
|
||||
"github.com/google/nixery/server/storage"
|
||||
"github.com/google/nixery/builder"
|
||||
"github.com/google/nixery/config"
|
||||
"github.com/google/nixery/logs"
|
||||
"github.com/google/nixery/storage"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
|
@ -59,7 +59,7 @@ var (
|
|||
|
||||
// Downloads the popularity information for the package set from the
|
||||
// URL specified in Nixery's configuration.
|
||||
func downloadPopularity(url string) (layers.Popularity, error) {
|
||||
func downloadPopularity(url string) (builder.Popularity, error) {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -74,7 +74,7 @@ func downloadPopularity(url string) (layers.Popularity, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
var pop layers.Popularity
|
||||
var pop builder.Popularity
|
||||
err = json.Unmarshal(j, &pop)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -190,6 +190,7 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
func main() {
|
||||
logs.Init(version)
|
||||
cfg, err := config.FromEnv()
|
||||
if err != nil {
|
||||
log.WithError(err).Fatal("failed to load configuration")
|
||||
|
@ -214,7 +215,7 @@ func main() {
|
|||
log.WithError(err).Fatal("failed to instantiate build cache")
|
||||
}
|
||||
|
||||
var pop layers.Popularity
|
||||
var pop builder.Popularity
|
||||
if cfg.PopUrl != "" {
|
||||
pop, err = downloadPopularity(cfg.PopUrl)
|
||||
if err != nil {
|
|
@ -175,7 +175,7 @@ func fetchNarInfo(i *item) (string, error) {
|
|||
narinfo, err := ioutil.ReadAll(resp.Body)
|
||||
|
||||
// best-effort write the file to the cache
|
||||
ioutil.WriteFile("popcache/" + i.hash, narinfo, 0644)
|
||||
ioutil.WriteFile("popcache/"+i.hash, narinfo, 0644)
|
||||
|
||||
return string(narinfo), err
|
||||
}
|
||||
|
|
|
@ -20,10 +20,10 @@
|
|||
|
||||
{ pkgs ? import <nixpkgs> {} }:
|
||||
|
||||
pkgs.writeShellScriptBin "nixery-build-image" ''
|
||||
pkgs.writeShellScriptBin "nixery-prepare-image" ''
|
||||
exec ${pkgs.nix}/bin/nix-build \
|
||||
--show-trace \
|
||||
--no-out-link "$@" \
|
||||
--argstr loadPkgs ${./load-pkgs.nix} \
|
||||
${./build-image.nix}
|
||||
${./prepare-image.nix}
|
||||
''
|
|
@ -1,62 +0,0 @@
|
|||
# Copyright 2019 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{ buildGoPackage, go, lib, srcHash }:
|
||||
|
||||
buildGoPackage rec {
|
||||
name = "nixery-server";
|
||||
goDeps = ./go-deps.nix;
|
||||
src = ./.;
|
||||
|
||||
goPackagePath = "github.com/google/nixery/server";
|
||||
doCheck = true;
|
||||
|
||||
# The following phase configurations work around the overengineered
|
||||
# Nix build configuration for Go.
|
||||
#
|
||||
# All I want this to do is produce a binary in the standard Nix
|
||||
# output path, so pretty much all the phases except for the initial
|
||||
# configuration of the "dependency forest" in $GOPATH have been
|
||||
# overridden.
|
||||
#
|
||||
# This is necessary because the upstream builder does wonky things
|
||||
# with the build arguments to the compiler, but I need to set some
|
||||
# complex flags myself
|
||||
|
||||
outputs = [ "out" ];
|
||||
preConfigure = "bin=$out";
|
||||
buildPhase = ''
|
||||
runHook preBuild
|
||||
runHook renameImport
|
||||
|
||||
export GOBIN="$out/bin"
|
||||
go install -ldflags "-X main.version=$(cat ${srcHash})" ${goPackagePath}
|
||||
'';
|
||||
|
||||
fixupPhase = ''
|
||||
remove-references-to -t ${go} $out/bin/server
|
||||
'';
|
||||
|
||||
checkPhase = ''
|
||||
go vet ${goPackagePath}
|
||||
go test ${goPackagePath}
|
||||
'';
|
||||
|
||||
meta = {
|
||||
description = "Container image builder serving Nix-backed images";
|
||||
homepage = "https://github.com/google/nixery";
|
||||
license = lib.licenses.asl20;
|
||||
maintainers = [ lib.maintainers.tazjin ];
|
||||
};
|
||||
}
|
|
@ -20,5 +20,5 @@ let nixery = import ./default.nix { inherit pkgs; };
|
|||
in pkgs.stdenv.mkDerivation {
|
||||
name = "nixery-dev-shell";
|
||||
|
||||
buildInputs = with pkgs; [ jq nixery.nixery-build-image ];
|
||||
buildInputs = with pkgs; [ jq nixery.nixery-prepare-image ];
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue