tvl-depot/tools/nixery/main.go

// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

// Package main provides the implementation of a container registry that
// transparently builds container images based on Nix derivations.
//
// The Nix derivation used for image creation is responsible for creating
// objects that are compatible with the registry API. The targeted registry
// protocol is currently Docker's.
//
// When an image is requested, the required contents are parsed out of the
// request and a Nix-build is initiated that eventually responds with the
// manifest as well as information linking each layer digest to a local
// filesystem path.
//
// Nixery caches the filesystem paths and returns the manifest to the client.
// Subsequent requests for layer content per digest are then fulfilled by
// serving the files from disk.
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"cloud.google.com/go/storage"
)

// pkgSource represents the source from which the Nix package set used
// by Nixery is imported. Users configure the source by setting one of
// the supported environment variables.
type pkgSource struct {
	srcType string
	args    string
}

// Convert the package source into the representation required by Nix.
func (p *pkgSource) renderSource(tag string) string {
	// The 'git' source requires a tag to be present.
	if p.srcType == "git" {
		if tag == "latest" || tag == "" {
			tag = "master"
		}

		return fmt.Sprintf("git!%s!%s", p.args, tag)
	}

	return fmt.Sprintf("%s!%s", p.srcType, p.args)
}

// Retrieve a package source from the environment. If no source is
// specified, the Nix code will default to a recent NixOS channel.
func pkgSourceFromEnv() *pkgSource {
	if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" {
		log.Printf("Using Nix package set from Nix channel %q\n", channel)
		return &pkgSource{
			srcType: "nixpkgs",
			args:    channel,
		}
	}

	if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" {
		log.Printf("Using Nix package set from git repository at %q\n", git)
		return &pkgSource{
			srcType: "git",
			args:    git,
		}
	}

	if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" {
		log.Printf("Using Nix package set from path %q\n", path)
		return &pkgSource{
			srcType: "path",
			args:    path,
		}
	}

	return nil
}

// config holds the Nixery configuration options.
type config struct {
	bucket  string     // GCS bucket to cache & serve layers
	builder string     // Nix derivation for building images
	web     string     // Static files to serve over HTTP
	port    string     // Port on which to launch HTTP server
	pkgs    *pkgSource // Source for Nix package set
}

// ManifestMediaType is the Content-Type used for the manifest itself. This
// corresponds to the "Image Manifest V2, Schema 2" described on this page:
//
// https://docs.docker.com/registry/spec/manifest-v2-2/
const manifestMediaType string = "application/vnd.docker.distribution.manifest.v2+json"

// Image represents the information necessary for building a container image.
// This can be either a list of package names (corresponding to keys in the
// nixpkgs set) or a Nix expression that results in a *list* of derivations.
type image struct {
	// Name of the container image.
	name string

	// Tag requested (only relevant for package sets from git repositories)
	tag string

	// Names of packages to include in the image. These must correspond
	// directly to top-level names of Nix packages in the nixpkgs tree.
	packages []string
}

// BuildResult represents the output of calling the Nix derivation responsible
// for building registry images.
//
// The `layerLocations` field contains the local filesystem paths to each
// individual image layer that will need to be served, while the `manifest`
// field contains the JSON-representation of the manifest that needs to be
// served to the client.
//
// The later field is simply treated as opaque JSON and passed through.
type BuildResult struct {
	Error string   `json:"error"`
	Pkgs  []string `json:"pkgs"`

	Manifest       json.RawMessage `json:"manifest"`
	LayerLocations map[string]struct {
		Path string `json:"path"`
		Md5  []byte `json:"md5"`
	} `json:"layerLocations"`
}

// imageFromName parses an image name into the corresponding structure which can
// be used to invoke Nix.
//
// It will expand convenience names under the hood (see the `convenienceNames`
// function below).
func imageFromName(name string, tag string) image {
	packages := strings.Split(name, "/")
	return image{
		name:     name,
		tag:      tag,
		packages: convenienceNames(packages),
	}
}

// convenienceNames expands convenience package names defined by Nixery which
// let users include commonly required sets of tools in a container quickly.
//
// Convenience names must be specified as the first package in an image.
//
// Currently defined convenience names are:
//
// * `shell`: Includes bash, coreutils and other common command-line tools
// * `builder`: All of the above and the standard build environment
func convenienceNames(packages []string) []string {
	shellPackages := []string{"bashInteractive", "coreutils", "moreutils", "nano"}
	builderPackages := append(shellPackages, "stdenv")

	if packages[0] == "shell" {
		return append(packages[1:], shellPackages...)
	} else if packages[0] == "builder" {
		return append(packages[1:], builderPackages...)
	} else {
		return packages
	}
}

// Call out to Nix and request that an image be built. Nix will, upon success,
// return a manifest for the container image.
func buildImage(ctx *context.Context, cfg *config, image *image, bucket *storage.BucketHandle) (*BuildResult, error) {
	packages, err := json.Marshal(image.packages)
	if err != nil {
		return nil, err
	}

	args := []string{
		"--no-out-link",
		"--show-trace",
		"--argstr", "name", image.name,
		"--argstr", "packages", string(packages), cfg.builder,
	}

	if cfg.pkgs != nil {
		args = append(args, "--argstr", "pkgSource", cfg.pkgs.renderSource(image.tag))
	}
	cmd := exec.Command("nix-build", args...)

	outpipe, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	errpipe, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	if err = cmd.Start(); err != nil {
		log.Println("Error starting nix-build:", err)
		return nil, err
	}
	log.Printf("Started Nix image build for '%s'", image.name)

	stdout, _ := ioutil.ReadAll(outpipe)
	stderr, _ := ioutil.ReadAll(errpipe)

	if err = cmd.Wait(); err != nil {
		// TODO(tazjin): Propagate errors upwards in a usable format.
		log.Printf("nix-build execution error: %s\nstdout: %s\nstderr: %s\n", err, stdout, stderr)
		return nil, err
	}

	log.Println("Finished Nix image build")

	buildOutput, err := ioutil.ReadFile(strings.TrimSpace(string(stdout)))
	if err != nil {
		return nil, err
	}

	// The build output returned by Nix is deserialised to add all
	// contained layers to the bucket. Only the manifest itself is
	// re-serialised to JSON and returned.
	var result BuildResult
	err = json.Unmarshal(buildOutput, &result)
	if err != nil {
		return nil, err
	}

	for layer, meta := range result.LayerLocations {
		err = uploadLayer(ctx, bucket, layer, meta.Path, meta.Md5)
		if err != nil {
			return nil, err
		}
	}

	return &result, nil
}

// uploadLayer uploads a single layer to Cloud Storage bucket. Before writing
// any data the bucket is probed to see if the file already exists.
//
// If the file does exist, its MD5 hash is verified to ensure that the stored
// file is not - for example - a fragment of a previous, incomplete upload.
func uploadLayer(ctx *context.Context, bucket *storage.BucketHandle, layer string, path string, md5 []byte) error {
	layerKey := fmt.Sprintf("layers/%s", layer)
	obj := bucket.Object(layerKey)

	// Before uploading a layer to the bucket, probe whether it already
	// exists.
	//
	// If it does and the MD5 checksum matches the expected one, the layer
	// upload can be skipped.
	attrs, err := obj.Attrs(*ctx)

	if err == nil && bytes.Equal(attrs.MD5, md5) {
		log.Printf("Layer sha256:%s already exists in bucket, skipping upload", layer)
	} else {
		writer := obj.NewWriter(*ctx)
		file, err := os.Open(path)

		if err != nil {
			return fmt.Errorf("failed to open layer %s from path %s: %v", layer, path, err)
		}

		size, err := io.Copy(writer, file)
		if err != nil {
			return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err)
		}

		if err = writer.Close(); err != nil {
			return fmt.Errorf("failed to write layer %s to Cloud Storage: %v", layer, err)
		}

		log.Printf("Uploaded layer sha256:%s (%v bytes written)\n", layer, size)
	}

	return nil
}

// layerRedirect constructs the public URL of the layer object in the Cloud
// Storage bucket and redirects the client there.
//
// The Docker client is known to follow redirects, but this might not be true
// for all other registry clients.
func layerRedirect(w http.ResponseWriter, cfg *config, digest string) {
	log.Printf("Redirecting layer '%s' request to bucket '%s'\n", digest, cfg.bucket)
	url := fmt.Sprintf("https://storage.googleapis.com/%s/layers/%s", cfg.bucket, digest)
	w.Header().Set("Location", url)
	w.WriteHeader(303)
}

// prepareBucket configures the handle to a Cloud Storage bucket in which
// individual layers will be stored after Nix builds. Nixery does not directly
// serve layers to registry clients, instead it redirects them to the public
// URLs of the Cloud Storage bucket.
//
// The bucket is required for Nixery to function correctly, hence fatal errors
// are generated in case it fails to be set up correctly.
func prepareBucket(ctx *context.Context, cfg *config) *storage.BucketHandle {
	client, err := storage.NewClient(*ctx)
	if err != nil {
		log.Fatalln("Failed to set up Cloud Storage client:", err)
	}

	bkt := client.Bucket(cfg.bucket)

	if _, err := bkt.Attrs(*ctx); err != nil {
		log.Fatalln("Could not access configured bucket", err)
	}

	return bkt
}

// Regexes matching the V2 Registry API routes. This only includes the
// routes required for serving images, since pushing and other such
// functionality is not available.
var (
	manifestRegex = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/manifests/([\w|\-|\.|\_]+)$`)
	layerRegex    = regexp.MustCompile(`^/v2/([\w|\-|\.|\_|\/]+)/blobs/sha256:(\w+)$`)
)

// Error format corresponding to the registry protocol V2 specification. This
// allows feeding back errors to clients in a way that can be presented to
// users.
type registryError struct {
	Code    string `json:"code"`
	Message string `json:"message"`
}

type registryErrors struct {
	Errors []registryError `json:"errors"`
}

func writeError(w http.ResponseWriter, status int, code, message string) {
	err := registryErrors{
		Errors: []registryError{
			{code, message},
		},
	}
	json, _ := json.Marshal(err)

	w.WriteHeader(status)
	w.Header().Add("Content-Type", "application/json")
	w.Write(json)
}

type registryHandler struct {
	cfg    *config
	ctx    *context.Context
	bucket *storage.BucketHandle
}

func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Serve the manifest (straight from Nix)
	manifestMatches := manifestRegex.FindStringSubmatch(r.RequestURI)
	if len(manifestMatches) == 3 {
		imageName := manifestMatches[1]
		imageTag := manifestMatches[2]
		log.Printf("Requesting manifest for image %q at tag %q", imageName, imageTag)
		image := imageFromName(imageName, imageTag)
		buildResult, err := buildImage(h.ctx, h.cfg, &image, h.bucket)

		if err != nil {
			writeError(w, 500, "UNKNOWN", "image build failure")
			log.Println("Failed to build image manifest", err)
			return
		}

		// Some error types have special handling, which is applied
		// here.
		if buildResult.Error == "not_found" {
			s := fmt.Sprintf("Could not find Nix packages: %v", buildResult.Pkgs)
			writeError(w, 404, "MANIFEST_UNKNOWN", s)
			log.Println(s)
			return
		}

		// This marshaling error is ignored because we know that this
		// field represents valid JSON data.
		manifest, _ := json.Marshal(buildResult.Manifest)
		w.Header().Add("Content-Type", manifestMediaType)
		w.Write(manifest)
		return
	}

	// Serve an image layer. For this we need to first ask Nix for
	// the manifest, then proceed to extract the correct layer from
	// it.
	layerMatches := layerRegex.FindStringSubmatch(r.RequestURI)
	if len(layerMatches) == 3 {
		digest := layerMatches[2]
		layerRedirect(w, h.cfg, digest)
		return
	}

	log.Printf("Unsupported registry route: %s\n", r.RequestURI)
	w.WriteHeader(404)
}

func getConfig(key, desc string) string {
	value := os.Getenv(key)
	if value == "" {
		log.Fatalln(desc + " must be specified")
	}

	return value
}

func main() {
	cfg := &config{
		bucket:  getConfig("BUCKET", "GCS bucket for layer storage"),
		builder: getConfig("NIX_BUILDER", "Nix image builder code"),
		web:     getConfig("WEB_DIR", "Static web file dir"),
		port:    getConfig("PORT", "HTTP port"),
		pkgs:    pkgSourceFromEnv(),
	}

	ctx := context.Background()
	bucket := prepareBucket(&ctx, cfg)

	log.Printf("Starting Kubernetes Nix controller on port %s\n", cfg.port)

	// Acknowledge that we speak V2
	http.HandleFunc("/v2", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w)
	})

	// All other /v2/ requests belong to the registry handler.
	http.Handle("/v2/", &registryHandler{
		cfg:    cfg,
		ctx:    &ctx,
		bucket: bucket,
	})

	// All other roots are served by the static file server.
	http.Handle("/", http.FileServer(http.Dir(cfg.web)))

	log.Fatal(http.ListenAndServe(":"+cfg.port, nil))
}