refactor(server): Move package source management logic to server

Introduces three new types representing each of the possible package
sources and moves the logic for specifying the package source to the
server.

Concrete changes:

* Determining whether a specified git reference is a commit vs. a
  branch/tag is now done in the server, and is done more precisely by
  using a regular expression.

* Package sources now have a new `CacheKey` function which can be used
  to retrieve a key under which a build manifest can be cached *if*
  the package source is not a moving target (i.e. a full git commit
  hash of either nixpkgs or a private repository).

  This function is not yet used.

* Users *must* now specify a package source, Nixery no longer defaults
  to anything and will fail to launch if no source is configured.

tools/nixery/server/builder/builder.go

@@ -118,15 +118,16 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima
 			return nil, err
 		}
 
+		srcType, srcArgs := cfg.Pkgs.Render(image.Tag)
+
 		args := []string{
 			"--timeout", cfg.Timeout,
 			"--argstr", "name", image.Name,
 			"--argstr", "packages", string(packages),
+			"--argstr", "srcType", srcType,
+			"--argstr", "srcArgs", srcArgs,
 		}
 
-		if cfg.Pkgs != nil {
-			args = append(args, "--argstr", "pkgSource", cfg.Pkgs.Render(image.Tag))
-		}
 		cmd := exec.Command("nixery-build-image", args...)
 
 		outpipe, err := cmd.StdoutPipe()

tools/nixery/server/config/config.go

@@ -18,7 +18,6 @@
 package config
 
 import (
-	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
@@ -26,58 +25,6 @@ import (
 	"cloud.google.com/go/storage"
 )
 
-// pkgSource represents the source from which the Nix package set used
-// by Nixery is imported. Users configure the source by setting one of
-// the supported environment variables.
-type PkgSource struct {
-	srcType string
-	args    string
-}
-
-// Convert the package source into the representation required by Nix.
-func (p *PkgSource) Render(tag string) string {
-	// The 'git' source requires a tag to be present.
-	if p.srcType == "git" {
-		if tag == "latest" || tag == "" {
-			tag = "master"
-		}
-
-		return fmt.Sprintf("git!%s!%s", p.args, tag)
-	}
-
-	return fmt.Sprintf("%s!%s", p.srcType, p.args)
-}
-
-// Retrieve a package source from the environment. If no source is
-// specified, the Nix code will default to a recent NixOS channel.
-func pkgSourceFromEnv() *PkgSource {
-	if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" {
-		log.Printf("Using Nix package set from Nix channel %q\n", channel)
-		return &PkgSource{
-			srcType: "nixpkgs",
-			args:    channel,
-		}
-	}
-
-	if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" {
-		log.Printf("Using Nix package set from git repository at %q\n", git)
-		return &PkgSource{
-			srcType: "git",
-			args:    git,
-		}
-	}
-
-	if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" {
-		log.Printf("Using Nix package set from path %q\n", path)
-		return &PkgSource{
-			srcType: "path",
-			args:    path,
-		}
-	}
-
-	return nil
-}
-
 // Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and
 // GCS_SIGNING_ACCOUNT envvars.
 func signingOptsFromEnv() *storage.SignedURLOptions {
@@ -118,18 +65,23 @@ type Config struct {
 	Bucket  string                    // GCS bucket to cache & serve layers
 	Signing *storage.SignedURLOptions // Signing options to use for GCS URLs
 	Port    string                    // Port on which to launch HTTP server
-	Pkgs    *PkgSource                // Source for Nix package set
+	Pkgs    PkgSource                 // Source for Nix package set
 	Timeout string                    // Timeout for a single Nix builder (seconds)
 	WebDir  string                    // Directory with static web assets
 }
 
-func FromEnv() *Config {
+func FromEnv() (*Config, error) {
+	pkgs, err := pkgSourceFromEnv()
+	if err != nil {
+		return nil, err
+	}
+
 	return &Config{
 		Bucket:  getConfig("BUCKET", "GCS bucket for layer storage", ""),
 		Port:    getConfig("PORT", "HTTP port", ""),
-		Pkgs:    pkgSourceFromEnv(),
+		Pkgs:    pkgs,
 		Signing: signingOptsFromEnv(),
 		Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"),
 		WebDir:  getConfig("WEB_DIR", "Static web file dir", ""),
-	}
+	}, nil
 }

tools/nixery/server/config/pkgsource.go

@@ -0,0 +1,155 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+package config
+
+import (
+	"crypto/sha1"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+)
+
+// PkgSource represents the source from which the Nix package set used
+// by Nixery is imported. Users configure the source by setting one of
+// the supported environment variables.
+type PkgSource interface {
+	// Convert the package source into the representation required
+	// for calling Nix.
+	Render(tag string) (string, string)
+
+	// Create a key by which builds for this source and iamge
+	// combination can be cached.
+	//
+	// The empty string means that this value is not cacheable due
+	// to the package source being a moving target (such as a
+	// channel).
+	CacheKey(pkgs []string, tag string) string
+}
+
+type GitSource struct {
+	repository string
+}
+
+// Regex to determine whether a git reference is a commit hash or
+// something else (branch/tag).
+//
+// Used to check whether a git reference is cacheable, and to pass the
+// correct git structure to Nix.
+//
+// Note: If a user creates a branch or tag with the name of a commit
+// and references it intentionally, this heuristic will fail.
+var commitRegex = regexp.MustCompile(`^[0-9a-f]{40}$`)
+
+func (g *GitSource) Render(tag string) (string, string) {
+	args := map[string]string{
+		"url": g.repository,
+	}
+
+	// The 'git' source requires a tag to be present. If the user
+	// has not specified one, it is assumed that the default
+	// 'master' branch should be used.
+	if tag == "latest" || tag == "" {
+		tag = "master"
+	}
+
+	if commitRegex.MatchString(tag) {
+		args["rev"] = tag
+	} else {
+		args["ref"] = tag
+	}
+
+	j, _ := json.Marshal(args)
+
+	return "git", string(j)
+}
+
+func (g *GitSource) CacheKey(pkgs []string, tag string) string {
+	// Only full commit hashes can be used for caching, as
+	// everything else is potentially a moving target.
+	if !commitRegex.MatchString(tag) {
+		return ""
+	}
+
+	unhashed := strings.Join(pkgs, "") + tag
+	hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed)))
+
+	return hashed
+}
+
+type NixChannel struct {
+	channel string
+}
+
+func (n *NixChannel) Render(tag string) (string, string) {
+	return "nixpkgs", n.channel
+}
+
+func (n *NixChannel) CacheKey(pkgs []string, tag string) string {
+	// Since Nix channels are downloaded from the nixpkgs-channels
+	// Github, users can specify full commit hashes as the
+	// "channel", in which case builds are cacheable.
+	if !commitRegex.MatchString(n.channel) {
+		return ""
+	}
+
+	unhashed := strings.Join(pkgs, "") + n.channel
+	hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed)))
+
+	return hashed
+}
+
+type PkgsPath struct {
+	path string
+}
+
+func (p *PkgsPath) Render(tag string) (string, string) {
+	return "path", p.path
+}
+
+func (p *PkgsPath) CacheKey(pkgs []string, tag string) string {
+	// Path-based builds are not currently cacheable because we
+	// have no local hash of the package folder's state easily
+	// available.
+	return ""
+}
+
+// Retrieve a package source from the environment. If no source is
+// specified, the Nix code will default to a recent NixOS channel.
+func pkgSourceFromEnv() (PkgSource, error) {
+	if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" {
+		log.Printf("Using Nix package set from Nix channel %q\n", channel)
+		return &NixChannel{
+			channel: channel,
+		}, nil
+	}
+
+	if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" {
+		log.Printf("Using Nix package set from git repository at %q\n", git)
+		return &GitSource{
+			repository: git,
+		}, nil
+	}
+
+	if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" {
+		log.Printf("Using Nix package set from path %q\n", path)
+		return &PkgsPath{
+			path: path,
+		}, nil
+	}
+
+	return nil, fmt.Errorf("no valid package source has been specified")
+}

tools/nixery/server/main.go

@@ -190,7 +190,11 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func main() {
-	cfg := config.FromEnv()
+	cfg, err := config.FromEnv()
+	if err != nil {
+		log.Fatalln("Failed to load configuration", err)
+	}
+
 	ctx := context.Background()
 	bucket := prepareBucket(&ctx, cfg)
 	cache := builder.NewCache()