refactor(server): Move package source management logic to server

Introduces three new types representing each of the possible package
sources and moves the logic for specifying the package source to the
server.

Concrete changes:

* Determining whether a specified git reference is a commit vs. a
  branch/tag is now done in the server, and is done more precisely by
  using a regular expression.

* Package sources now have a new `CacheKey` function which can be used
  to retrieve a key under which a build manifest can be cached *if*
  the package source is not a moving target (i.e. a full git commit
  hash of either nixpkgs or a private repository).

  This function is not yet used.

* Users *must* now specify a package source, Nixery no longer defaults
  to anything and will fail to launch if no source is configured.

tools/nixery/.travis.yml

@@ -31,6 +31,7 @@ script:
       -e GOOGLE_APPLICATION_CREDENTIALS=/var/nixery/key.json \
       -e GCS_SIGNING_ACCOUNT="${GCS_SIGNING_ACCOUNT}" \
       -e GCS_SIGNING_KEY=/var/nixery/gcs.pem \
+      -e NIXERY_CHANNEL=nixos-unstable \
       ${IMG}
 
   # print all of the container's logs regardless of success

tools/nixery/build-image/default.nix

@@ -20,11 +20,12 @@
 
   # Because of the insanity occuring below, this function must mirror
   # all arguments of build-image.nix.
-, pkgSource ? "nixpkgs!nixos-19.03"
+, srcType ? "nixpkgs"
+, srcArgs ? "nixos-19.03"
 , tag ? null, name ? null, packages ? null, maxLayers ? null
 }@args:
 
-let pkgs = import ./load-pkgs.nix { inherit pkgSource; };
+let pkgs = import ./load-pkgs.nix { inherit srcType srcArgs; };
 in with pkgs; rec {
 
   groupLayers = buildGoPackage {

tools/nixery/build-image/load-pkgs.nix

@@ -12,17 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Load a Nix package set from a source specified in one of the following
+# Load a Nix package set from one of the supported source types
-# formats:
+# (nixpkgs, git, path).
-#
+{ srcType, srcArgs, importArgs ? { } }:
-# 1. nixpkgs!$channel (e.g. nixpkgs!nixos-19.03)
-# 2. git!$repo!$rev (e.g. git!git@github.com:NixOS/nixpkgs.git!master)
-# 3. path!$path (e.g. path!/var/local/nixpkgs)
-#
-# '!' was chosen as the separator because `builtins.split` does not
-# support regex escapes and there are few other candidates. It
-# doesn't matter much because this is invoked by the server.
-{ pkgSource, args ? { } }:
 
 with builtins;
 let
@@ -32,42 +24,22 @@ let
     let
       url =
         "https://github.com/NixOS/nixpkgs-channels/archive/${channel}.tar.gz";
-    in import (fetchTarball url) args;
+    in import (fetchTarball url) importArgs;
 
   # If a git repository is requested, it is retrieved via
   # builtins.fetchGit which defaults to the git configuration of the
   # outside environment. This means that user-configured SSH
   # credentials etc. are going to work as expected.
-  fetchImportGit = url: rev:
+  fetchImportGit = spec: import (fetchGit spec) importArgs;
-    let
-      # builtins.fetchGit needs to know whether 'rev' is a reference
-      # (e.g. a branch/tag) or a revision (i.e. a commit hash)
-      #
-      # Since this data is being extrapolated from the supplied image
-      # tag, we have to guess if we want to avoid specifying a format.
-      #
-      # There are some additional caveats around whether the default
-      # branch contains the specified revision, which need to be
-      # explained to users.
-      spec = if (stringLength rev) == 40 then {
-        inherit url rev;
-      } else {
-        inherit url;
-        ref = rev;
-      };
-    in import (fetchGit spec) args;
 
   # No special handling is used for paths, so users are expected to pass one
   # that will work natively with Nix.
-  importPath = path: import (toPath path) args;
+  importPath = path: import (toPath path) importArgs;
-
+in if srcType == "nixpkgs" then
-  source = split "!" pkgSource;
+  fetchImportChannel srcArgs
-  sourceType = elemAt source 0;
+else if srcType == "git" then
-in if sourceType == "nixpkgs" then
+  fetchImportGit (fromJSON srcArgs)
-  fetchImportChannel (elemAt source 2)
+else if srcType == "path" then
-else if sourceType == "git" then
+  importPath srcArgs
-  fetchImportGit (elemAt source 2) (elemAt source 4)
-else if sourceType == "path" then
-  importPath (elemAt source 2)
 else
-  throw ("Invalid package set source specification: ${pkgSource}")
+  throw ("Invalid package set source specification: ${srcType} (${srcArgs})")

tools/nixery/default.nix

@@ -28,7 +28,8 @@ rec {
 
   # Implementation of the image building & layering logic
   nixery-build-image = (import ./build-image {
-    pkgSource = "path!${<nixpkgs>}";
+    srcType = "path";
+    srcArgs = <nixpkgs>;
   }).wrapper;
 
   # Use mdBook to build a static asset page which Nixery can then

tools/nixery/server/builder/builder.go

@@ -118,15 +118,16 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima
 			return nil, err
 		}
 
+		srcType, srcArgs := cfg.Pkgs.Render(image.Tag)
+
 		args := []string{
 			"--timeout", cfg.Timeout,
 			"--argstr", "name", image.Name,
 			"--argstr", "packages", string(packages),
+			"--argstr", "srcType", srcType,
+			"--argstr", "srcArgs", srcArgs,
 		}
 
-		if cfg.Pkgs != nil {
-			args = append(args, "--argstr", "pkgSource", cfg.Pkgs.Render(image.Tag))
-		}
 		cmd := exec.Command("nixery-build-image", args...)
 
 		outpipe, err := cmd.StdoutPipe()

tools/nixery/server/config/config.go

@@ -18,7 +18,6 @@
 package config
 
 import (
-	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
@@ -26,58 +25,6 @@ import (
 	"cloud.google.com/go/storage"
 )
 
-// pkgSource represents the source from which the Nix package set used
-// by Nixery is imported. Users configure the source by setting one of
-// the supported environment variables.
-type PkgSource struct {
-	srcType string
-	args    string
-}
-
-// Convert the package source into the representation required by Nix.
-func (p *PkgSource) Render(tag string) string {
-	// The 'git' source requires a tag to be present.
-	if p.srcType == "git" {
-		if tag == "latest" || tag == "" {
-			tag = "master"
-		}
-
-		return fmt.Sprintf("git!%s!%s", p.args, tag)
-	}
-
-	return fmt.Sprintf("%s!%s", p.srcType, p.args)
-}
-
-// Retrieve a package source from the environment. If no source is
-// specified, the Nix code will default to a recent NixOS channel.
-func pkgSourceFromEnv() *PkgSource {
-	if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" {
-		log.Printf("Using Nix package set from Nix channel %q\n", channel)
-		return &PkgSource{
-			srcType: "nixpkgs",
-			args:    channel,
-		}
-	}
-
-	if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" {
-		log.Printf("Using Nix package set from git repository at %q\n", git)
-		return &PkgSource{
-			srcType: "git",
-			args:    git,
-		}
-	}
-
-	if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" {
-		log.Printf("Using Nix package set from path %q\n", path)
-		return &PkgSource{
-			srcType: "path",
-			args:    path,
-		}
-	}
-
-	return nil
-}
-
 // Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and
 // GCS_SIGNING_ACCOUNT envvars.
 func signingOptsFromEnv() *storage.SignedURLOptions {
@@ -118,18 +65,23 @@ type Config struct {
 	Bucket  string                    // GCS bucket to cache & serve layers
 	Signing *storage.SignedURLOptions // Signing options to use for GCS URLs
 	Port    string                    // Port on which to launch HTTP server
-	Pkgs    *PkgSource                // Source for Nix package set
+	Pkgs    PkgSource                 // Source for Nix package set
 	Timeout string                    // Timeout for a single Nix builder (seconds)
 	WebDir  string                    // Directory with static web assets
 }
 
-func FromEnv() *Config {
+func FromEnv() (*Config, error) {
+	pkgs, err := pkgSourceFromEnv()
+	if err != nil {
+		return nil, err
+	}
+
 	return &Config{
 		Bucket:  getConfig("BUCKET", "GCS bucket for layer storage", ""),
 		Port:    getConfig("PORT", "HTTP port", ""),
-		Pkgs:    pkgSourceFromEnv(),
+		Pkgs:    pkgs,
 		Signing: signingOptsFromEnv(),
 		Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"),
 		WebDir:  getConfig("WEB_DIR", "Static web file dir", ""),
-	}
+	}, nil
 }

tools/nixery/server/config/pkgsource.go

@@ -0,0 +1,155 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+package config
+
+import (
+	"crypto/sha1"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+)
+
+// PkgSource represents the source from which the Nix package set used
+// by Nixery is imported. Users configure the source by setting one of
+// the supported environment variables.
+type PkgSource interface {
+	// Convert the package source into the representation required
+	// for calling Nix.
+	Render(tag string) (string, string)
+
+	// Create a key by which builds for this source and iamge
+	// combination can be cached.
+	//
+	// The empty string means that this value is not cacheable due
+	// to the package source being a moving target (such as a
+	// channel).
+	CacheKey(pkgs []string, tag string) string
+}
+
+type GitSource struct {
+	repository string
+}
+
+// Regex to determine whether a git reference is a commit hash or
+// something else (branch/tag).
+//
+// Used to check whether a git reference is cacheable, and to pass the
+// correct git structure to Nix.
+//
+// Note: If a user creates a branch or tag with the name of a commit
+// and references it intentionally, this heuristic will fail.
+var commitRegex = regexp.MustCompile(`^[0-9a-f]{40}$`)
+
+func (g *GitSource) Render(tag string) (string, string) {
+	args := map[string]string{
+		"url": g.repository,
+	}
+
+	// The 'git' source requires a tag to be present. If the user
+	// has not specified one, it is assumed that the default
+	// 'master' branch should be used.
+	if tag == "latest" || tag == "" {
+		tag = "master"
+	}
+
+	if commitRegex.MatchString(tag) {
+		args["rev"] = tag
+	} else {
+		args["ref"] = tag
+	}
+
+	j, _ := json.Marshal(args)
+
+	return "git", string(j)
+}
+
+func (g *GitSource) CacheKey(pkgs []string, tag string) string {
+	// Only full commit hashes can be used for caching, as
+	// everything else is potentially a moving target.
+	if !commitRegex.MatchString(tag) {
+		return ""
+	}
+
+	unhashed := strings.Join(pkgs, "") + tag
+	hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed)))
+
+	return hashed
+}
+
+type NixChannel struct {
+	channel string
+}
+
+func (n *NixChannel) Render(tag string) (string, string) {
+	return "nixpkgs", n.channel
+}
+
+func (n *NixChannel) CacheKey(pkgs []string, tag string) string {
+	// Since Nix channels are downloaded from the nixpkgs-channels
+	// Github, users can specify full commit hashes as the
+	// "channel", in which case builds are cacheable.
+	if !commitRegex.MatchString(n.channel) {
+		return ""
+	}
+
+	unhashed := strings.Join(pkgs, "") + n.channel
+	hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed)))
+
+	return hashed
+}
+
+type PkgsPath struct {
+	path string
+}
+
+func (p *PkgsPath) Render(tag string) (string, string) {
+	return "path", p.path
+}
+
+func (p *PkgsPath) CacheKey(pkgs []string, tag string) string {
+	// Path-based builds are not currently cacheable because we
+	// have no local hash of the package folder's state easily
+	// available.
+	return ""
+}
+
+// Retrieve a package source from the environment. If no source is
+// specified, the Nix code will default to a recent NixOS channel.
+func pkgSourceFromEnv() (PkgSource, error) {
+	if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" {
+		log.Printf("Using Nix package set from Nix channel %q\n", channel)
+		return &NixChannel{
+			channel: channel,
+		}, nil
+	}
+
+	if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" {
+		log.Printf("Using Nix package set from git repository at %q\n", git)
+		return &GitSource{
+			repository: git,
+		}, nil
+	}
+
+	if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" {
+		log.Printf("Using Nix package set from path %q\n", path)
+		return &PkgsPath{
+			path: path,
+		}, nil
+	}
+
+	return nil, fmt.Errorf("no valid package source has been specified")
+}

tools/nixery/server/main.go

@@ -190,7 +190,11 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func main() {
-	cfg := config.FromEnv()
+	cfg, err := config.FromEnv()
+	if err != nil {
+		log.Fatalln("Failed to load configuration", err)
+	}
+
 	ctx := context.Background()
 	bucket := prepareBucket(&ctx, cfg)
 	cache := builder.NewCache()