refactor: Reshuffle file structure for better code layout

This gets rid of the package called "server" and instead moves
everything into the project root, such that Go actually builds us a
binary called `nixery`.

This is the first step towards factoring out CLI-based functionality
for Nixery.
This commit is contained in:
Vincent Ambo 2019-11-11 21:07:16 +00:00 committed by Vincent Ambo
parent df88da126a
commit 2b82f1b71a
21 changed files with 83 additions and 114 deletions

View file

@ -19,7 +19,6 @@ package builder
// The tarball is written straight to the supplied reader, which makes it
// possible to create an image layer from the specified store paths, hash it and
// upload it in one reading pass.
import (
"archive/tar"
"compress/gzip"
@ -28,8 +27,6 @@ import (
"io"
"os"
"path/filepath"
"github.com/google/nixery/server/layers"
)
// Create a new compressed tarball from each of the paths in the list
@ -37,7 +34,7 @@ import (
//
// The uncompressed tarball is hashed because image manifests must
// contain both the hashes of compressed and uncompressed layers.
func packStorePaths(l *layers.Layer, w io.Writer) (string, error) {
func packStorePaths(l *layer, w io.Writer) (string, error) {
shasum := sha256.New()
gz := gzip.NewWriter(w)
multi := io.MultiWriter(shasum, gz)

View file

@ -12,9 +12,10 @@
// License for the specific language governing permissions and limitations under
// the License.
// Package builder implements the code required to build images via Nix. Image
// build data is cached for up to 24 hours to avoid duplicated calls to Nix
// (which are costly even if no building is performed).
// Package builder implements the logic for assembling container
// images. It shells out to Nix to retrieve all required Nix-packages
// and assemble the symlink layer and then creates the required
// tarballs in-process.
package builder
import (
@ -32,10 +33,9 @@ import (
"sort"
"strings"
"github.com/google/nixery/server/config"
"github.com/google/nixery/server/layers"
"github.com/google/nixery/server/manifest"
"github.com/google/nixery/server/storage"
"github.com/google/nixery/config"
"github.com/google/nixery/manifest"
"github.com/google/nixery/storage"
log "github.com/sirupsen/logrus"
)
@ -50,7 +50,7 @@ type State struct {
Storage storage.Backend
Cache *LocalCache
Cfg config.Config
Pop layers.Popularity
Pop Popularity
}
// Architecture represents the possible CPU architectures for which
@ -128,7 +128,7 @@ type ImageResult struct {
Pkgs []string `json:"pkgs"`
// These fields are populated in case of success
Graph layers.RuntimeGraph `json:"runtimeGraph"`
Graph runtimeGraph `json:"runtimeGraph"`
SymlinkLayer struct {
Size int `json:"size"`
TarHash string `json:"tarHash"`
@ -265,7 +265,7 @@ func prepareImage(s *State, image *Image) (*ImageResult, error) {
"--argstr", "system", image.Arch.nixSystem,
}
output, err := callNix("nixery-build-image", image.Name, args)
output, err := callNix("nixery-prepare-image", image.Name, args)
if err != nil {
// granular error logging is performed in callNix already
return nil, err
@ -292,7 +292,7 @@ func prepareImage(s *State, image *Image) (*ImageResult, error) {
// added only after successful uploads, which guarantees that entries
// retrieved from the cache are present in the bucket.
func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageResult) ([]manifest.Entry, error) {
grouped := layers.Group(&result.Graph, &s.Pop, LayerBudget)
grouped := groupLayers(&result.Graph, &s.Pop, LayerBudget)
var entries []manifest.Entry
@ -329,7 +329,7 @@ func prepareLayers(ctx context.Context, s *State, image *Image, result *ImageRes
var pkgs []string
for _, p := range l.Contents {
pkgs = append(pkgs, layers.PackageFromPath(p))
pkgs = append(pkgs, packageFromPath(p))
}
log.WithFields(log.Fields{

View file

@ -22,7 +22,7 @@ import (
"os"
"sync"
"github.com/google/nixery/server/manifest"
"github.com/google/nixery/manifest"
log "github.com/sirupsen/logrus"
)

View file

@ -114,7 +114,7 @@
//
// Layer budget: 10
// Layers: { E }, { D, F }, { A }, { B }, { C }
package layers
package builder
import (
"crypto/sha1"
@ -128,11 +128,11 @@ import (
"gonum.org/v1/gonum/graph/simple"
)
// RuntimeGraph represents structured information from Nix about the runtime
// runtimeGraph represents structured information from Nix about the runtime
// dependencies of a derivation.
//
// This is generated in Nix by using the exportReferencesGraph feature.
type RuntimeGraph struct {
type runtimeGraph struct {
References struct {
Graph []string `json:"graph"`
} `json:"exportReferencesGraph"`
@ -153,19 +153,19 @@ type Popularity = map[string]int
// Layer represents the data returned for each layer that Nix should
// build for the container image.
type Layer struct {
type layer struct {
Contents []string `json:"contents"`
MergeRating uint64
}
// Hash the contents of a layer to create a deterministic identifier that can be
// used for caching.
func (l *Layer) Hash() string {
func (l *layer) Hash() string {
sum := sha1.Sum([]byte(strings.Join(l.Contents, ":")))
return fmt.Sprintf("%x", sum)
}
func (a Layer) merge(b Layer) Layer {
func (a layer) merge(b layer) layer {
a.Contents = append(a.Contents, b.Contents...)
a.MergeRating += b.MergeRating
return a
@ -188,12 +188,15 @@ var nixRegexp = regexp.MustCompile(`^/nix/store/[a-z0-9]+-`)
// PackageFromPath returns the name of a Nix package based on its
// output store path.
func PackageFromPath(path string) string {
func packageFromPath(path string) string {
return nixRegexp.ReplaceAllString(path, "")
}
// DOTID provides a human-readable package name. The name stems from
// the dot format used by GraphViz, into which the dependency graph
// can be rendered.
func (c *closure) DOTID() string {
return PackageFromPath(c.Path)
return packageFromPath(c.Path)
}
// bigOrPopular checks whether this closure should be considered for
@ -236,7 +239,7 @@ func insertEdges(graph *simple.DirectedGraph, cmap *map[string]*closure, node *c
}
// Create a graph structure from the references supplied by Nix.
func buildGraph(refs *RuntimeGraph, pop *Popularity) *simple.DirectedGraph {
func buildGraph(refs *runtimeGraph, pop *Popularity) *simple.DirectedGraph {
cmap := make(map[string]*closure)
graph := simple.NewDirectedGraph()
@ -296,7 +299,7 @@ func buildGraph(refs *RuntimeGraph, pop *Popularity) *simple.DirectedGraph {
// Extracts a subgraph starting at the specified root from the
// dominator tree. The subgraph is converted into a flat list of
// layers, each containing the store paths and merge rating.
func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
func groupLayer(dt *flow.DominatorTree, root *closure) layer {
size := root.Size
contents := []string{root.Path}
children := dt.DominatedBy(root.ID())
@ -313,7 +316,7 @@ func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
// Contents are sorted to ensure that hashing is consistent
sort.Strings(contents)
return Layer{
return layer{
Contents: contents,
MergeRating: uint64(root.Popularity) * size,
}
@ -324,10 +327,10 @@ func groupLayer(dt *flow.DominatorTree, root *closure) Layer {
//
// Layers are merged together until they fit into the layer budget,
// based on their merge rating.
func dominate(budget int, graph *simple.DirectedGraph) []Layer {
func dominate(budget int, graph *simple.DirectedGraph) []layer {
dt := flow.Dominators(graph.Node(0), graph)
var layers []Layer
var layers []layer
for _, n := range dt.DominatedBy(dt.Root().ID()) {
layers = append(layers, groupLayer(&dt, n.(*closure)))
}
@ -352,10 +355,10 @@ func dominate(budget int, graph *simple.DirectedGraph) []Layer {
return layers
}
// GroupLayers applies the algorithm described above the its input and returns a
// groupLayers applies the algorithm described above the its input and returns a
// list of layers, each consisting of a list of Nix store paths that it should
// contain.
func Group(refs *RuntimeGraph, pop *Popularity, budget int) []Layer {
func groupLayers(refs *runtimeGraph, pop *Popularity, budget int) []layer {
graph := buildGraph(refs, pop)
return dominate(budget, graph)
}

View file

@ -20,6 +20,8 @@
with pkgs;
let
inherit (pkgs) buildGoPackage;
# Hash of all Nixery sources - this is used as the Nixery version in
# builds to distinguish errors between deployed versions, see
# server/logs.go for details.
@ -30,13 +32,41 @@ let
# Go implementation of the Nixery server which implements the
# container registry interface.
#
# Users should use the nixery-bin derivation below instead.
nixery-server = callPackage ./server {
srcHash = nixery-src-hash;
# Users should use the nixery-bin derivation below instead as it
# provides the paths of files needed at runtime.
nixery-server = buildGoPackage rec {
name = "nixery-server";
goDeps = ./go-deps.nix;
src = ./.;
goPackagePath = "github.com/google/nixery";
doCheck = true;
# Simplify the Nix build instructions for Go to just the basics
# required to get Nixery up and running with the additional linker
# flags required.
outputs = [ "out" ];
preConfigure = "bin=$out";
buildPhase = ''
runHook preBuild
runHook renameImport
export GOBIN="$out/bin"
go install -ldflags "-X main.version=$(cat ${nixery-src-hash})" ${goPackagePath}
'';
fixupPhase = ''
remove-references-to -t ${go} $out/bin/nixery
'';
checkPhase = ''
go vet ${goPackagePath}
go test ${goPackagePath}
'';
};
in rec {
# Implementation of the Nix image building logic
nixery-build-image = import ./build-image { inherit pkgs; };
nixery-prepare-image = import ./prepare-image { inherit pkgs; };
# Use mdBook to build a static asset page which Nixery can then
# serve. This is primarily used for the public instance at
@ -50,8 +80,8 @@ in rec {
# are installing Nixery directly.
nixery-bin = writeShellScriptBin "nixery" ''
export WEB_DIR="${nixery-book}"
export PATH="${nixery-build-image}/bin:$PATH"
exec ${nixery-server}/bin/server
export PATH="${nixery-prepare-image}/bin:$PATH"
exec ${nixery-server}/bin/nixery
'';
nixery-popcount = callPackage ./popcount { };
@ -104,7 +134,7 @@ in rec {
gzip
iana-etc
nix
nixery-build-image
nixery-prepare-image
nixery-launch-script
openssh
zlib

View file

@ -11,7 +11,7 @@
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
package main
package logs
// This file configures different log formatters via logrus. The
// standard formatter uses a structured JSON format that is compatible
@ -112,7 +112,7 @@ func (f stackdriverFormatter) Format(e *log.Entry) ([]byte, error) {
return b.Bytes(), err
}
func init() {
func Init(version string) {
nixeryContext.Version = version
log.SetReportCaller(true)
log.SetFormatter(stackdriverFormatter{})

View file

@ -32,10 +32,10 @@ import (
"net/http"
"regexp"
"github.com/google/nixery/server/builder"
"github.com/google/nixery/server/config"
"github.com/google/nixery/server/layers"
"github.com/google/nixery/server/storage"
"github.com/google/nixery/builder"
"github.com/google/nixery/config"
"github.com/google/nixery/logs"
"github.com/google/nixery/storage"
log "github.com/sirupsen/logrus"
)
@ -59,7 +59,7 @@ var (
// Downloads the popularity information for the package set from the
// URL specified in Nixery's configuration.
func downloadPopularity(url string) (layers.Popularity, error) {
func downloadPopularity(url string) (builder.Popularity, error) {
resp, err := http.Get(url)
if err != nil {
return nil, err
@ -74,7 +74,7 @@ func downloadPopularity(url string) (layers.Popularity, error) {
return nil, err
}
var pop layers.Popularity
var pop builder.Popularity
err = json.Unmarshal(j, &pop)
if err != nil {
return nil, err
@ -190,6 +190,7 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
func main() {
logs.Init(version)
cfg, err := config.FromEnv()
if err != nil {
log.WithError(err).Fatal("failed to load configuration")
@ -214,7 +215,7 @@ func main() {
log.WithError(err).Fatal("failed to instantiate build cache")
}
var pop layers.Popularity
var pop builder.Popularity
if cfg.PopUrl != "" {
pop, err = downloadPopularity(cfg.PopUrl)
if err != nil {

View file

@ -175,7 +175,7 @@ func fetchNarInfo(i *item) (string, error) {
narinfo, err := ioutil.ReadAll(resp.Body)
// best-effort write the file to the cache
ioutil.WriteFile("popcache/" + i.hash, narinfo, 0644)
ioutil.WriteFile("popcache/"+i.hash, narinfo, 0644)
return string(narinfo), err
}

View file

@ -20,10 +20,10 @@
{ pkgs ? import <nixpkgs> {} }:
pkgs.writeShellScriptBin "nixery-build-image" ''
pkgs.writeShellScriptBin "nixery-prepare-image" ''
exec ${pkgs.nix}/bin/nix-build \
--show-trace \
--no-out-link "$@" \
--argstr loadPkgs ${./load-pkgs.nix} \
${./build-image.nix}
${./prepare-image.nix}
''

View file

@ -1,62 +0,0 @@
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
{ buildGoPackage, go, lib, srcHash }:
buildGoPackage rec {
name = "nixery-server";
goDeps = ./go-deps.nix;
src = ./.;
goPackagePath = "github.com/google/nixery/server";
doCheck = true;
# The following phase configurations work around the overengineered
# Nix build configuration for Go.
#
# All I want this to do is produce a binary in the standard Nix
# output path, so pretty much all the phases except for the initial
# configuration of the "dependency forest" in $GOPATH have been
# overridden.
#
# This is necessary because the upstream builder does wonky things
# with the build arguments to the compiler, but I need to set some
# complex flags myself
outputs = [ "out" ];
preConfigure = "bin=$out";
buildPhase = ''
runHook preBuild
runHook renameImport
export GOBIN="$out/bin"
go install -ldflags "-X main.version=$(cat ${srcHash})" ${goPackagePath}
'';
fixupPhase = ''
remove-references-to -t ${go} $out/bin/server
'';
checkPhase = ''
go vet ${goPackagePath}
go test ${goPackagePath}
'';
meta = {
description = "Container image builder serving Nix-backed images";
homepage = "https://github.com/google/nixery";
license = lib.licenses.asl20;
maintainers = [ lib.maintainers.tazjin ];
};
}

View file

@ -20,5 +20,5 @@ let nixery = import ./default.nix { inherit pkgs; };
in pkgs.stdenv.mkDerivation {
name = "nixery-dev-shell";
buildInputs = with pkgs; [ jq nixery.nixery-build-image ];
buildInputs = with pkgs; [ jq nixery.nixery-prepare-image ];
}