refactor(server): Move package source management logic to server

Introduces three new types representing each of the possible package
sources and moves the logic for specifying the package source to the
server.

Concrete changes:

* Determining whether a specified git reference is a commit vs. a
  branch/tag is now done in the server, and is done more precisely by
  using a regular expression.

* Package sources now have a new `CacheKey` function which can be used
  to retrieve a key under which a build manifest can be cached *if*
  the package source is not a moving target (i.e. a full git commit
  hash of either nixpkgs or a private repository).

  This function is not yet used.

* Users *must* now specify a package source, Nixery no longer defaults
  to anything and will fail to launch if no source is configured.
This commit is contained in:
Vincent Ambo 2019-09-08 21:53:22 +01:00 committed by Vincent Ambo
parent 496a4ab847
commit 980f5e2187
8 changed files with 192 additions and 105 deletions

View file

@ -31,6 +31,7 @@ script:
-e GOOGLE_APPLICATION_CREDENTIALS=/var/nixery/key.json \ -e GOOGLE_APPLICATION_CREDENTIALS=/var/nixery/key.json \
-e GCS_SIGNING_ACCOUNT="${GCS_SIGNING_ACCOUNT}" \ -e GCS_SIGNING_ACCOUNT="${GCS_SIGNING_ACCOUNT}" \
-e GCS_SIGNING_KEY=/var/nixery/gcs.pem \ -e GCS_SIGNING_KEY=/var/nixery/gcs.pem \
-e NIXERY_CHANNEL=nixos-unstable \
${IMG} ${IMG}
# print all of the container's logs regardless of success # print all of the container's logs regardless of success

View file

@ -20,11 +20,12 @@
# Because of the insanity occuring below, this function must mirror # Because of the insanity occuring below, this function must mirror
# all arguments of build-image.nix. # all arguments of build-image.nix.
, pkgSource ? "nixpkgs!nixos-19.03" , srcType ? "nixpkgs"
, srcArgs ? "nixos-19.03"
, tag ? null, name ? null, packages ? null, maxLayers ? null , tag ? null, name ? null, packages ? null, maxLayers ? null
}@args: }@args:
let pkgs = import ./load-pkgs.nix { inherit pkgSource; }; let pkgs = import ./load-pkgs.nix { inherit srcType srcArgs; };
in with pkgs; rec { in with pkgs; rec {
groupLayers = buildGoPackage { groupLayers = buildGoPackage {

View file

@ -12,17 +12,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# Load a Nix package set from a source specified in one of the following # Load a Nix package set from one of the supported source types
# formats: # (nixpkgs, git, path).
# { srcType, srcArgs, importArgs ? { } }:
# 1. nixpkgs!$channel (e.g. nixpkgs!nixos-19.03)
# 2. git!$repo!$rev (e.g. git!git@github.com:NixOS/nixpkgs.git!master)
# 3. path!$path (e.g. path!/var/local/nixpkgs)
#
# '!' was chosen as the separator because `builtins.split` does not
# support regex escapes and there are few other candidates. It
# doesn't matter much because this is invoked by the server.
{ pkgSource, args ? { } }:
with builtins; with builtins;
let let
@ -32,42 +24,22 @@ let
let let
url = url =
"https://github.com/NixOS/nixpkgs-channels/archive/${channel}.tar.gz"; "https://github.com/NixOS/nixpkgs-channels/archive/${channel}.tar.gz";
in import (fetchTarball url) args; in import (fetchTarball url) importArgs;
# If a git repository is requested, it is retrieved via # If a git repository is requested, it is retrieved via
# builtins.fetchGit which defaults to the git configuration of the # builtins.fetchGit which defaults to the git configuration of the
# outside environment. This means that user-configured SSH # outside environment. This means that user-configured SSH
# credentials etc. are going to work as expected. # credentials etc. are going to work as expected.
fetchImportGit = url: rev: fetchImportGit = spec: import (fetchGit spec) importArgs;
let
# builtins.fetchGit needs to know whether 'rev' is a reference
# (e.g. a branch/tag) or a revision (i.e. a commit hash)
#
# Since this data is being extrapolated from the supplied image
# tag, we have to guess if we want to avoid specifying a format.
#
# There are some additional caveats around whether the default
# branch contains the specified revision, which need to be
# explained to users.
spec = if (stringLength rev) == 40 then {
inherit url rev;
} else {
inherit url;
ref = rev;
};
in import (fetchGit spec) args;
# No special handling is used for paths, so users are expected to pass one # No special handling is used for paths, so users are expected to pass one
# that will work natively with Nix. # that will work natively with Nix.
importPath = path: import (toPath path) args; importPath = path: import (toPath path) importArgs;
in if srcType == "nixpkgs" then
source = split "!" pkgSource; fetchImportChannel srcArgs
sourceType = elemAt source 0; else if srcType == "git" then
in if sourceType == "nixpkgs" then fetchImportGit (fromJSON srcArgs)
fetchImportChannel (elemAt source 2) else if srcType == "path" then
else if sourceType == "git" then importPath srcArgs
fetchImportGit (elemAt source 2) (elemAt source 4)
else if sourceType == "path" then
importPath (elemAt source 2)
else else
throw ("Invalid package set source specification: ${pkgSource}") throw ("Invalid package set source specification: ${srcType} (${srcArgs})")

View file

@ -28,7 +28,8 @@ rec {
# Implementation of the image building & layering logic # Implementation of the image building & layering logic
nixery-build-image = (import ./build-image { nixery-build-image = (import ./build-image {
pkgSource = "path!${<nixpkgs>}"; srcType = "path";
srcArgs = <nixpkgs>;
}).wrapper; }).wrapper;
# Use mdBook to build a static asset page which Nixery can then # Use mdBook to build a static asset page which Nixery can then

View file

@ -118,15 +118,16 @@ func BuildImage(ctx *context.Context, cfg *config.Config, cache *BuildCache, ima
return nil, err return nil, err
} }
srcType, srcArgs := cfg.Pkgs.Render(image.Tag)
args := []string{ args := []string{
"--timeout", cfg.Timeout, "--timeout", cfg.Timeout,
"--argstr", "name", image.Name, "--argstr", "name", image.Name,
"--argstr", "packages", string(packages), "--argstr", "packages", string(packages),
"--argstr", "srcType", srcType,
"--argstr", "srcArgs", srcArgs,
} }
if cfg.Pkgs != nil {
args = append(args, "--argstr", "pkgSource", cfg.Pkgs.Render(image.Tag))
}
cmd := exec.Command("nixery-build-image", args...) cmd := exec.Command("nixery-build-image", args...)
outpipe, err := cmd.StdoutPipe() outpipe, err := cmd.StdoutPipe()

View file

@ -18,7 +18,6 @@
package config package config
import ( import (
"fmt"
"io/ioutil" "io/ioutil"
"log" "log"
"os" "os"
@ -26,58 +25,6 @@ import (
"cloud.google.com/go/storage" "cloud.google.com/go/storage"
) )
// pkgSource represents the source from which the Nix package set used
// by Nixery is imported. Users configure the source by setting one of
// the supported environment variables.
type PkgSource struct {
srcType string
args string
}
// Convert the package source into the representation required by Nix.
func (p *PkgSource) Render(tag string) string {
// The 'git' source requires a tag to be present.
if p.srcType == "git" {
if tag == "latest" || tag == "" {
tag = "master"
}
return fmt.Sprintf("git!%s!%s", p.args, tag)
}
return fmt.Sprintf("%s!%s", p.srcType, p.args)
}
// Retrieve a package source from the environment. If no source is
// specified, the Nix code will default to a recent NixOS channel.
func pkgSourceFromEnv() *PkgSource {
if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" {
log.Printf("Using Nix package set from Nix channel %q\n", channel)
return &PkgSource{
srcType: "nixpkgs",
args: channel,
}
}
if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" {
log.Printf("Using Nix package set from git repository at %q\n", git)
return &PkgSource{
srcType: "git",
args: git,
}
}
if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" {
log.Printf("Using Nix package set from path %q\n", path)
return &PkgSource{
srcType: "path",
args: path,
}
}
return nil
}
// Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and // Load (optional) GCS bucket signing data from the GCS_SIGNING_KEY and
// GCS_SIGNING_ACCOUNT envvars. // GCS_SIGNING_ACCOUNT envvars.
func signingOptsFromEnv() *storage.SignedURLOptions { func signingOptsFromEnv() *storage.SignedURLOptions {
@ -118,18 +65,23 @@ type Config struct {
Bucket string // GCS bucket to cache & serve layers Bucket string // GCS bucket to cache & serve layers
Signing *storage.SignedURLOptions // Signing options to use for GCS URLs Signing *storage.SignedURLOptions // Signing options to use for GCS URLs
Port string // Port on which to launch HTTP server Port string // Port on which to launch HTTP server
Pkgs *PkgSource // Source for Nix package set Pkgs PkgSource // Source for Nix package set
Timeout string // Timeout for a single Nix builder (seconds) Timeout string // Timeout for a single Nix builder (seconds)
WebDir string // Directory with static web assets WebDir string // Directory with static web assets
} }
func FromEnv() *Config { func FromEnv() (*Config, error) {
pkgs, err := pkgSourceFromEnv()
if err != nil {
return nil, err
}
return &Config{ return &Config{
Bucket: getConfig("BUCKET", "GCS bucket for layer storage", ""), Bucket: getConfig("BUCKET", "GCS bucket for layer storage", ""),
Port: getConfig("PORT", "HTTP port", ""), Port: getConfig("PORT", "HTTP port", ""),
Pkgs: pkgSourceFromEnv(), Pkgs: pkgs,
Signing: signingOptsFromEnv(), Signing: signingOptsFromEnv(),
Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"), Timeout: getConfig("NIX_TIMEOUT", "Nix builder timeout", "60"),
WebDir: getConfig("WEB_DIR", "Static web file dir", ""), WebDir: getConfig("WEB_DIR", "Static web file dir", ""),
} }, nil
} }

View file

@ -0,0 +1,155 @@
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.
package config
import (
"crypto/sha1"
"encoding/json"
"fmt"
"log"
"os"
"regexp"
"strings"
)
// PkgSource represents the source from which the Nix package set used
// by Nixery is imported. Users configure the source by setting one of
// the supported environment variables.
type PkgSource interface {
// Convert the package source into the representation required
// for calling Nix.
Render(tag string) (string, string)
// Create a key by which builds for this source and iamge
// combination can be cached.
//
// The empty string means that this value is not cacheable due
// to the package source being a moving target (such as a
// channel).
CacheKey(pkgs []string, tag string) string
}
type GitSource struct {
repository string
}
// Regex to determine whether a git reference is a commit hash or
// something else (branch/tag).
//
// Used to check whether a git reference is cacheable, and to pass the
// correct git structure to Nix.
//
// Note: If a user creates a branch or tag with the name of a commit
// and references it intentionally, this heuristic will fail.
var commitRegex = regexp.MustCompile(`^[0-9a-f]{40}$`)
func (g *GitSource) Render(tag string) (string, string) {
args := map[string]string{
"url": g.repository,
}
// The 'git' source requires a tag to be present. If the user
// has not specified one, it is assumed that the default
// 'master' branch should be used.
if tag == "latest" || tag == "" {
tag = "master"
}
if commitRegex.MatchString(tag) {
args["rev"] = tag
} else {
args["ref"] = tag
}
j, _ := json.Marshal(args)
return "git", string(j)
}
func (g *GitSource) CacheKey(pkgs []string, tag string) string {
// Only full commit hashes can be used for caching, as
// everything else is potentially a moving target.
if !commitRegex.MatchString(tag) {
return ""
}
unhashed := strings.Join(pkgs, "") + tag
hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed)))
return hashed
}
type NixChannel struct {
channel string
}
func (n *NixChannel) Render(tag string) (string, string) {
return "nixpkgs", n.channel
}
func (n *NixChannel) CacheKey(pkgs []string, tag string) string {
// Since Nix channels are downloaded from the nixpkgs-channels
// Github, users can specify full commit hashes as the
// "channel", in which case builds are cacheable.
if !commitRegex.MatchString(n.channel) {
return ""
}
unhashed := strings.Join(pkgs, "") + n.channel
hashed := fmt.Sprintf("%x", sha1.Sum([]byte(unhashed)))
return hashed
}
type PkgsPath struct {
path string
}
func (p *PkgsPath) Render(tag string) (string, string) {
return "path", p.path
}
func (p *PkgsPath) CacheKey(pkgs []string, tag string) string {
// Path-based builds are not currently cacheable because we
// have no local hash of the package folder's state easily
// available.
return ""
}
// Retrieve a package source from the environment. If no source is
// specified, the Nix code will default to a recent NixOS channel.
func pkgSourceFromEnv() (PkgSource, error) {
if channel := os.Getenv("NIXERY_CHANNEL"); channel != "" {
log.Printf("Using Nix package set from Nix channel %q\n", channel)
return &NixChannel{
channel: channel,
}, nil
}
if git := os.Getenv("NIXERY_PKGS_REPO"); git != "" {
log.Printf("Using Nix package set from git repository at %q\n", git)
return &GitSource{
repository: git,
}, nil
}
if path := os.Getenv("NIXERY_PKGS_PATH"); path != "" {
log.Printf("Using Nix package set from path %q\n", path)
return &PkgsPath{
path: path,
}, nil
}
return nil, fmt.Errorf("no valid package source has been specified")
}

View file

@ -190,7 +190,11 @@ func (h *registryHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
func main() { func main() {
cfg := config.FromEnv() cfg, err := config.FromEnv()
if err != nil {
log.Fatalln("Failed to load configuration", err)
}
ctx := context.Background() ctx := context.Background()
bucket := prepareBucket(&ctx, cfg) bucket := prepareBucket(&ctx, cfg)
cache := builder.NewCache() cache := builder.NewCache()