From 5126815846202658328325ef716fc2ee383d42e0 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Sat, 23 Nov 2019 15:24:50 +0000
Subject: [PATCH 01/36] chore: initial commit


From 24225580e8dd6dbbb930e01d0b4947f2feaa70c7 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Sat, 23 Nov 2019 15:25:11 +0000
Subject: [PATCH 02/36] feat: Check in buildGo with program & package builders

---
 buildGo.nix | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 buildGo.nix

diff --git a/buildGo.nix b/buildGo.nix
new file mode 100644
index 000000000..090627c59
--- /dev/null
+++ b/buildGo.nix
@@ -0,0 +1,70 @@
+# Copyright 2019 Google LLC.
+# SPDX-License-Identifier: Apache-2.0
+#
+# buildGo provides Nix functions to build Go packages in the style of Bazel's
+# rules_go.
+
+{ pkgs ? import <nixpkgs> {}
+, ... }:
+
+let
+  inherit (builtins)
+    attrNames
+    baseNameOf
+    elemAt
+    filter
+    map
+    match
+    readDir;
+
+  inherit (pkgs) lib go runCommand;
+
+  # Helpers for low-level Go compiler invocations
+  spaceOut = lib.concatStringsSep " ";
+
+  includeDepSrc = dep: "-I ${dep}";
+  includeSources = deps: spaceOut (map includeDepSrc deps);
+
+  includeDepLib = dep: "-L ${dep}";
+  includeLibs = deps: spaceOut (map includeDepLib deps);
+
+  srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1;
+  srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}";
+  srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs);
+
+  isGoFile = f: (match ".*\.go" f) != null;
+  isGoTest = f: (match ".*_test\.go" f) != null;
+  goFileFilter = k: v: (v == "regular") && (isGoFile k) && (!isGoTest k);
+  goFilesIn = dir:
+    let files = readDir dir;
+        goFiles = filter (f: goFileFilter f files."${f}") (attrNames files);
+    in map (f: dir + "/" + f) goFiles;
+
+  allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps)));
+
+  # High-level build functions
+
+  # Build a Go program out of the specified files and dependencies.
+  program = { name, srcs, deps ? [] }:
+  let uniqueDeps = allDeps deps;
+  in runCommand name {} ''
+    ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs}
+    mkdir -p $out/bin
+    ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${includeLibs uniqueDeps} ${name}.a
+  '';
+
+  # Build a Go library assembled out of the specified files.
+  #
+  # This outputs both the sources and compiled binary, as both are
+  # needed when downstream packages depend on it.
+  package = { name, srcs, deps ? [], path ? name }:
+  let uniqueDeps = allDeps deps;
+  in (runCommand "golib-${name}" {} ''
+    mkdir -p $out/${path}
+    ${srcList path (map (s: "${s}") srcs)}
+    ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs}
+  '') // { goDeps = uniqueDeps; };
+in {
+  # Only the high-level builder functions are exposed
+  inherit program package;
+}

From d441e035aabac877948f754289b3ab028408550b Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Sun, 24 Nov 2019 20:17:17 +0000
Subject: [PATCH 03/36] feat: Add support for Protobuf-generated libraries

Adds a 'buildGo.proto' function which takes a single .proto file as
its source and generates a corresponding Go library which can then be
imported.

'proto' takes these arguments (Yants-style type definition):

struct "protoArgs" {
  # required:
  name = string;
  proto = path;

  # optional:
  extraDeps = list goLib; # defaults to [ ]
  protocFlags = option string;
}

Note that proto libraries will automatically have dependencies for the
required protobuf Go libraries added to them.

gRPC is not (yet) supported.
---
 buildGo.nix | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 75 insertions(+), 3 deletions(-)

diff --git a/buildGo.nix b/buildGo.nix
index 090627c59..029271f2f 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -15,9 +15,11 @@ let
     filter
     map
     match
-    readDir;
+    readDir
+    replaceStrings
+    toPath;
 
-  inherit (pkgs) lib go runCommand;
+  inherit (pkgs) lib go runCommand fetchFromGitHub protobuf;
 
   # Helpers for low-level Go compiler invocations
   spaceOut = lib.concatStringsSep " ";
@@ -64,7 +66,77 @@ let
     ${srcList path (map (s: "${s}") srcs)}
     ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs}
   '') // { goDeps = uniqueDeps; };
+
+  # Build a Go library out of the specified protobuf definition.
+  proto = { name, proto, path ? name, protocFlags ? "", extraDeps ? [] }: package {
+    inherit name path;
+    deps = [ goProto ] ++ extraDeps;
+    srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} ''
+      cp ${proto} ${baseNameOf proto}
+      ${protobuf}/bin/protoc --plugin=${protocGo}/bin/protoc-gen-go \
+        --go_out=${protocFlags}import_path=${baseNameOf path}:. ${baseNameOf proto}
+      mv *.pb.go $out
+    '');
+  };
+
+  # Protobuf & gRPC integration requires these dependencies:
+  proto-go-src = fetchFromGitHub {
+    owner = "golang";
+    repo = "protobuf";
+    rev = "ed6926b37a637426117ccab59282c3839528a700";
+    sha256 = "0fynqrim022x9xi2bivkw19npbz4316v4yr7mb677s9s36z4dc4h";
+  };
+
+  protoPart = path: deps: package {
+    inherit deps;
+    name = replaceStrings ["/"] ["_"] path;
+    path = "github.com/golang/protobuf/${path}";
+    srcs = goFilesIn (toPath "${proto-go-src}/${path}");
+  };
+
+  goProto =
+    let
+      protobuf = package {
+        name = "protobuf";
+        path = "github.com/golang/protobuf/proto";
+        # TODO(tazjin): How does this build toggle work?
+        srcs = filter
+          (f: (match "(.*)/pointer_reflect.go" f) == null)
+          (goFilesIn (toPath "${proto-go-src}/proto"));
+      };
+      type = name: protoPart "ptypes/${name}" [ protobuf ];
+      descriptor = protoPart "descriptor" [ protobuf ];
+      ptypes = package {
+        name = "ptypes";
+        path = "github.com/golang/protobuf/ptypes";
+        srcs = goFilesIn (toPath "${proto-go-src}/ptypes");
+        deps = map type [
+          "any"
+          "duration"
+          "empty"
+          "struct"
+          "timestamp"
+          "wrappers"
+        ];
+      };
+    in protobuf // { goDeps = allDeps (protobuf.goDeps ++ [ ptypes ]); };
+
+  protocDescriptor = (protoPart "protoc-gen-go/descriptor" [ goProto ]);
+  protocGo =
+    let
+      generator = protoPart "protoc-gen-go/generator" [
+        (protoPart "protoc-gen-go/generator/internal/remap" [])
+        (protoPart "protoc-gen-go/plugin" [ protocDescriptor ])
+      ];
+      grpc = protoPart "protoc-gen-go/grpc" [ generator ];
+    in program {
+      name = "protoc-gen-go";
+      deps = [ goProto grpc generator ];
+      srcs = filter
+        (f: (match "(.*)/doc.go" f) == null)
+        (goFilesIn (toPath "${proto-go-src}/protoc-gen-go"));
+    };
 in {
   # Only the high-level builder functions are exposed
-  inherit program package;
+  inherit program package proto;
 }

From 071babf14824aee035dd6c3ddff1957662aa7c98 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Sun, 24 Nov 2019 20:34:30 +0000
Subject: [PATCH 04/36] feat(example): Add an example for how to use buildGo
 builders

---
 example/default.nix | 43 +++++++++++++++++++++++++++++++++++++++++++
 example/lib.go      |  9 +++++++++
 example/main.go     | 22 ++++++++++++++++++++++
 example/thing.proto | 10 ++++++++++
 4 files changed, 84 insertions(+)
 create mode 100644 example/default.nix
 create mode 100644 example/lib.go
 create mode 100644 example/main.go
 create mode 100644 example/thing.proto

diff --git a/example/default.nix b/example/default.nix
new file mode 100644
index 000000000..407f55079
--- /dev/null
+++ b/example/default.nix
@@ -0,0 +1,43 @@
+# Copyright 2019 Google LLC.
+# SPDX-License-Identifier: Apache-2.0
+
+# This file provides examples for how to use the various builder
+# functions provided by `buildGo`.
+#
+# The features used in the example are not exhaustive, but should give
+# users a quick introduction to how to use buildGo.
+
+let
+  buildGo = import ../buildGo.nix {};
+
+  # Example use of buildGo.package, which creates an importable Go
+  # package from the specified source files.
+  examplePackage = buildGo.package {
+    name = "example";
+    srcs = [
+      ./lib.go
+    ];
+  };
+
+  # Example use of buildGo.proto, which generates a Go library from a
+  # Protobuf definition file.
+  exampleProto = buildGo.proto {
+    name = "exampleproto";
+    proto = ./thing.proto;
+  };
+
+  # Example use of buildGo.program, which builds an executable using
+  # the specified name and dependencies (which in turn must have been
+  # created via buildGo.package etc.)
+in buildGo.program {
+  name = "example";
+
+  srcs = [
+    ./main.go
+  ];
+
+  deps = [
+    examplePackage
+    exampleProto
+  ];
+}
diff --git a/example/lib.go b/example/lib.go
new file mode 100644
index 000000000..8a61370e9
--- /dev/null
+++ b/example/lib.go
@@ -0,0 +1,9 @@
+// Copyright 2019 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+
+package example
+
+// UUID returns a totally random, carefully chosen UUID
+func UUID() string {
+	return "3640932f-ad40-4bc9-b45d-f504a0f5910a"
+}
diff --git a/example/main.go b/example/main.go
new file mode 100644
index 000000000..1db1bb9e2
--- /dev/null
+++ b/example/main.go
@@ -0,0 +1,22 @@
+// Copyright 2019 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Package main provides a tiny example program for the Bazel-style
+// Nix build system for Go.
+
+package main
+
+import (
+	"example"
+	"exampleproto"
+	"fmt"
+)
+
+func main() {
+	thing := exampleproto.Thing{
+		Id:          example.UUID(),
+		KindOfThing: "test thing",
+	}
+
+	fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing)
+}
diff --git a/example/thing.proto b/example/thing.proto
new file mode 100644
index 000000000..0cb34124d
--- /dev/null
+++ b/example/thing.proto
@@ -0,0 +1,10 @@
+// Copyright 2019 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+
+syntax = "proto3";
+package exampleProto;
+
+message Thing {
+  string id = 1;
+  string kind_of_thing = 2;
+}

From 0fd74669229415907353563b3dbc2847d8a80942 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Sun, 24 Nov 2019 20:41:01 +0000
Subject: [PATCH 05/36] feat(buildGo): Add support for x_defs option in
 buildGo.program

This lets users define an attribute set with link time options.
---
 buildGo.nix | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/buildGo.nix b/buildGo.nix
index 029271f2f..dce550265 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -44,15 +44,17 @@ let
 
   allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps)));
 
+  xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs));
+
   # High-level build functions
 
   # Build a Go program out of the specified files and dependencies.
-  program = { name, srcs, deps ? [] }:
+  program = { name, srcs, deps ? [], x_defs ? {} }:
   let uniqueDeps = allDeps deps;
   in runCommand name {} ''
     ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs}
     mkdir -p $out/bin
-    ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${includeLibs uniqueDeps} ${name}.a
+    ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a
   '';
 
   # Build a Go library assembled out of the specified files.

From 473421dbb81fedbec7f1f6e788c813b5fe9f4604 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Sun, 24 Nov 2019 20:41:29 +0000
Subject: [PATCH 06/36] feat(example): Demonstrate usage of x_defs flag to
 buildGo.program

---
 example/default.nix | 4 ++++
 example/main.go     | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/example/default.nix b/example/default.nix
index 407f55079..5abed1fbb 100644
--- a/example/default.nix
+++ b/example/default.nix
@@ -40,4 +40,8 @@ in buildGo.program {
     examplePackage
     exampleProto
   ];
+
+  x_defs = {
+    "main.Flag" = "successfully";
+  };
 }
diff --git a/example/main.go b/example/main.go
index 1db1bb9e2..bbcedbff8 100644
--- a/example/main.go
+++ b/example/main.go
@@ -12,6 +12,8 @@ import (
 	"fmt"
 )
 
+var Flag string = "unsuccessfully"
+
 func main() {
 	thing := exampleproto.Thing{
 		Id:          example.UUID(),
@@ -19,4 +21,5 @@ func main() {
 	}
 
 	fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing)
+	fmt.Printf("The flag has been %s set\n", Flag)
 }

From 7d26550c11e6fdde5b3a0a052b917aef16dc9816 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Mon, 25 Nov 2019 16:42:49 +0000
Subject: [PATCH 07/36] feat(buildGo): Add support for building "external" Go
 libraries

Adds a buildGo.external function that can build packages following the
default go-tool package layout. Dependencies work the same way as they
do for other buildGo-packages, but instead of being passed straight to
the compiler a fake GOPATH is assembled using a symlink forest.

External currently supports very few direct configuration options and
was primarily created to build the protobuf packages, but it is also
useful for including external dependencies in buildGo-native projects.

The previous complex build logic for the protobuf package has been
replaced with a call to `external`.
---
 buildGo.nix | 104 ++++++++++++++++++++++++++--------------------------
 1 file changed, 51 insertions(+), 53 deletions(-)

diff --git a/buildGo.nix b/buildGo.nix
index dce550265..1a7ed66c7 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -11,15 +11,15 @@ let
   inherit (builtins)
     attrNames
     baseNameOf
+    dirOf
     elemAt
     filter
     map
     match
     readDir
-    replaceStrings
-    toPath;
+    replaceStrings;
 
-  inherit (pkgs) lib go runCommand fetchFromGitHub protobuf;
+  inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin;
 
   # Helpers for low-level Go compiler invocations
   spaceOut = lib.concatStringsSep " ";
@@ -46,6 +46,8 @@ let
 
   xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs));
 
+  pathToName = p: replaceStrings ["/"] ["_"] (toString p);
+
   # High-level build functions
 
   # Build a Go program out of the specified files and dependencies.
@@ -75,12 +77,52 @@ let
     deps = [ goProto ] ++ extraDeps;
     srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} ''
       cp ${proto} ${baseNameOf proto}
-      ${protobuf}/bin/protoc --plugin=${protocGo}/bin/protoc-gen-go \
+      ${protobuf}/bin/protoc --plugin=${goProto}/bin/protoc-gen-go \
         --go_out=${protocFlags}import_path=${baseNameOf path}:. ${baseNameOf proto}
       mv *.pb.go $out
     '');
   };
 
+  # Build an externally defined Go library using `go build` itself.
+  #
+  # Libraries built this way can be included in any standard buildGo
+  # build.
+  #
+  # Contrary to other functions, `src` is expected to point at a
+  # single directory containing the root of the external library.
+  external = { path, src, deps ? [] }:
+    let
+      name = pathToName path;
+      uniqueDeps = allDeps deps;
+      srcDir = runCommand "goext-src-${name}" {} ''
+        mkdir -p $out/${dirOf path}
+        cp -r ${src} $out/${dirOf path}/${baseNameOf path}
+      '';
+      gopathSrc = symlinkJoin {
+        name = "gopath-${name}";
+        paths = uniqueDeps ++ [ srcDir ];
+      };
+      gopathPkg = runCommand "goext-pkg-${name}" {} ''
+        mkdir -p gopath $out
+        export GOPATH=$PWD/gopath
+        ln -s ${gopathSrc} gopath/src
+        ${go}/bin/go install ${path}/...
+
+        if [[ -d gopath/pkg/linux_amd64 ]]; then
+          echo "Installing Go packages for ${path}"
+          mv gopath/pkg/linux_amd64/* $out
+        fi
+
+        if [[ -d gopath/bin ]]; then
+          echo "Installing Go binaries for ${path}"
+          mv gopath/bin $out/bin
+        fi
+      '';
+    in symlinkJoin {
+      name = "goext-${name}";
+      paths = [ gopathSrc gopathPkg ];
+    } // { goDeps = uniqueDeps; };
+
   # Protobuf & gRPC integration requires these dependencies:
   proto-go-src = fetchFromGitHub {
     owner = "golang";
@@ -89,56 +131,12 @@ let
     sha256 = "0fynqrim022x9xi2bivkw19npbz4316v4yr7mb677s9s36z4dc4h";
   };
 
-  protoPart = path: deps: package {
-    inherit deps;
-    name = replaceStrings ["/"] ["_"] path;
-    path = "github.com/golang/protobuf/${path}";
-    srcs = goFilesIn (toPath "${proto-go-src}/${path}");
+  goProto = external {
+    path = "github.com/golang/protobuf";
+    src = proto-go-src;
+    deps = [];
   };
-
-  goProto =
-    let
-      protobuf = package {
-        name = "protobuf";
-        path = "github.com/golang/protobuf/proto";
-        # TODO(tazjin): How does this build toggle work?
-        srcs = filter
-          (f: (match "(.*)/pointer_reflect.go" f) == null)
-          (goFilesIn (toPath "${proto-go-src}/proto"));
-      };
-      type = name: protoPart "ptypes/${name}" [ protobuf ];
-      descriptor = protoPart "descriptor" [ protobuf ];
-      ptypes = package {
-        name = "ptypes";
-        path = "github.com/golang/protobuf/ptypes";
-        srcs = goFilesIn (toPath "${proto-go-src}/ptypes");
-        deps = map type [
-          "any"
-          "duration"
-          "empty"
-          "struct"
-          "timestamp"
-          "wrappers"
-        ];
-      };
-    in protobuf // { goDeps = allDeps (protobuf.goDeps ++ [ ptypes ]); };
-
-  protocDescriptor = (protoPart "protoc-gen-go/descriptor" [ goProto ]);
-  protocGo =
-    let
-      generator = protoPart "protoc-gen-go/generator" [
-        (protoPart "protoc-gen-go/generator/internal/remap" [])
-        (protoPart "protoc-gen-go/plugin" [ protocDescriptor ])
-      ];
-      grpc = protoPart "protoc-gen-go/grpc" [ generator ];
-    in program {
-      name = "protoc-gen-go";
-      deps = [ goProto grpc generator ];
-      srcs = filter
-        (f: (match "(.*)/doc.go" f) == null)
-        (goFilesIn (toPath "${proto-go-src}/protoc-gen-go"));
-    };
 in {
   # Only the high-level builder functions are exposed
-  inherit program package proto;
+  inherit program package proto external;
 }

From 8b6b08b814af72e8b2f6281037f089e099a41e25 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Tue, 26 Nov 2019 12:11:43 +0000
Subject: [PATCH 08/36] feat(buildGo): Add 'srcOnly' and 'targets' parameters
 for external

Adds two new parameters to buildGo.external:

* `srcOnly` toggles whether the created derivation should contain only
  the source code, or the built package.

  This is useful in situations where some sub-packages of a larger
  package are needed and the build should be deferred to the package
  depending on them.

  It defaults to false, meaning that external packages are built by
  default.

* `targets` controls which "sub-packages" of the target package are
  built. It defaults to building all sub-packages.
---
 buildGo.nix | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/buildGo.nix b/buildGo.nix
index 1a7ed66c7..12504ce57 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -90,7 +90,7 @@ let
   #
   # Contrary to other functions, `src` is expected to point at a
   # single directory containing the root of the external library.
-  external = { path, src, deps ? [] }:
+  external = { path, src, deps ? [], srcOnly ? false, targets ? [ "..." ] }:
     let
       name = pathToName path;
       uniqueDeps = allDeps deps;
@@ -106,7 +106,7 @@ let
         mkdir -p gopath $out
         export GOPATH=$PWD/gopath
         ln -s ${gopathSrc} gopath/src
-        ${go}/bin/go install ${path}/...
+        ${go}/bin/go install ${spaceOut (map (t: path + "/" + t) targets)}
 
         if [[ -d gopath/pkg/linux_amd64 ]]; then
           echo "Installing Go packages for ${path}"
@@ -118,10 +118,10 @@ let
           mv gopath/bin $out/bin
         fi
       '';
-    in symlinkJoin {
+    in (if srcOnly then gopathSrc else symlinkJoin {
       name = "goext-${name}";
       paths = [ gopathSrc gopathPkg ];
-    } // { goDeps = uniqueDeps; };
+    }) // { goDeps = uniqueDeps; };
 
   # Protobuf & gRPC integration requires these dependencies:
   proto-go-src = fetchFromGitHub {

From d3e8774a8e159984bfa2864d383a1d9b641a4c6f Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Tue, 26 Nov 2019 12:15:18 +0000
Subject: [PATCH 09/36] feat(proto): Add protobuf & gRPC dependencies via
 external

Moves the Protobuf & gRPC dependencies to a separate file which uses
buildGo.external to build the dependencies.

The versions are pinned at master of 2019-11-26.
---
 buildGo.nix | 14 ++---------
 proto.nix   | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 12 deletions(-)
 create mode 100644 proto.nix

diff --git a/buildGo.nix b/buildGo.nix
index 12504ce57..bf8625c1b 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -123,18 +123,8 @@ let
       paths = [ gopathSrc gopathPkg ];
     }) // { goDeps = uniqueDeps; };
 
-  # Protobuf & gRPC integration requires these dependencies:
-  proto-go-src = fetchFromGitHub {
-    owner = "golang";
-    repo = "protobuf";
-    rev = "ed6926b37a637426117ccab59282c3839528a700";
-    sha256 = "0fynqrim022x9xi2bivkw19npbz4316v4yr7mb677s9s36z4dc4h";
-  };
-
-  goProto = external {
-    path = "github.com/golang/protobuf";
-    src = proto-go-src;
-    deps = [];
+  protoLibs = import ./proto.nix {
+    inherit external;
   };
 in {
   # Only the high-level builder functions are exposed
diff --git a/proto.nix b/proto.nix
new file mode 100644
index 000000000..e773af031
--- /dev/null
+++ b/proto.nix
@@ -0,0 +1,72 @@
+# Copyright 2019 Google LLC.
+# SPDX-License-Identifier: Apache-2.0
+#
+# This file provides derivations for the dependencies of a gRPC
+# service in Go.
+
+{ external }:
+
+let
+  inherit (builtins) fetchGit map;
+in rec {
+  goProto = external {
+    path = "github.com/golang/protobuf";
+    src = fetchGit {
+      url = "https://github.com/golang/protobuf";
+      rev = "ed6926b37a637426117ccab59282c3839528a700";
+    };
+  };
+
+  xnet = external {
+    path = "golang.org/x/net";
+    srcOnly = true;
+    deps = [ xtext ];
+    src = fetchGit {
+      url = "https://go.googlesource.com/net";
+      rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d";
+    };
+  };
+
+  xsys = external {
+    path = "golang.org/x/sys";
+    srcOnly = true;
+    src = fetchGit {
+      url = "https://go.googlesource.com/sys";
+      rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd";
+    };
+  };
+
+  xtext = external {
+    path = "golang.org/x/text";
+    srcOnly = true;
+    src = fetchGit {
+      url = "https://go.googlesource.com/text";
+      rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686";
+    };
+  };
+
+  genproto = external {
+    path = "google.golang.org/genproto";
+    srcOnly = true;
+    src = fetchGit {
+      url = "https://github.com/google/go-genproto";
+      rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6";
+    };
+  };
+
+  goGrpc = external {
+    path = "google.golang.org/grpc";
+    deps = [ goProto xnet xsys genproto ];
+
+    src = fetchGit {
+      url = "https://github.com/grpc/grpc-go";
+      rev = "d8e3da36ac481ef00e510ca119f6b68177713689";
+    };
+
+    targets = [
+      "."
+      "codes"
+      "status"
+    ];
+  };
+}

From 9280cf97151fd14269342310ad1acfb6ccd37369 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Tue, 26 Nov 2019 12:16:27 +0000
Subject: [PATCH 10/36] feat(buildGo): Add support for gRPC packages

Introduces buildGo.grpc which is like buildGo.proto, but adds
dependencies on the gRPC libraries.
---
 buildGo.nix | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/buildGo.nix b/buildGo.nix
index bf8625c1b..fba4e18e9 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -72,17 +72,20 @@ let
   '') // { goDeps = uniqueDeps; };
 
   # Build a Go library out of the specified protobuf definition.
-  proto = { name, proto, path ? name, protocFlags ? "", extraDeps ? [] }: package {
+  proto = { name, proto, path ? name, extraDeps ? [] }: package {
     inherit name path;
-    deps = [ goProto ] ++ extraDeps;
+    deps = [ protoLibs.goProto ] ++ extraDeps;
     srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} ''
       cp ${proto} ${baseNameOf proto}
-      ${protobuf}/bin/protoc --plugin=${goProto}/bin/protoc-gen-go \
-        --go_out=${protocFlags}import_path=${baseNameOf path}:. ${baseNameOf proto}
+      ${protobuf}/bin/protoc --plugin=${protoLibs.goProto}/bin/protoc-gen-go \
+        --go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto}
       mv *.pb.go $out
     '');
   };
 
+  # Build a Go library out of the specified gRPC definition.
+  grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc ]; });
+
   # Build an externally defined Go library using `go build` itself.
   #
   # Libraries built this way can be included in any standard buildGo
@@ -128,5 +131,5 @@ let
   };
 in {
   # Only the high-level builder functions are exposed
-  inherit program package proto external;
+  inherit program package proto grpc external;
 }

From 28e587b348a8aaa7af00a004c05286af9d35ca9a Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Wed, 27 Nov 2019 15:14:30 +0000
Subject: [PATCH 11/36] docs: Add README file that describes project usage &
 background

---
 README.md | 139 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 139 insertions(+)
 create mode 100644 README.md

diff --git a/README.md b/README.md
new file mode 100644
index 000000000..489cf1281
--- /dev/null
+++ b/README.md
@@ -0,0 +1,139 @@
+buildGo.nix
+===========
+
+This is an alternative [Nix][] build system for [Go][]. It supports building Go
+libraries and programs, and even automatically generating Protobuf & gRPC
+libraries.
+
+*Note:* This will probably end up being folded into [Nixery][].
+
+## Background
+
+Most language-specific Nix tooling outsources the build to existing
+language-specific build tooling, which essentially means that Nix ends up being
+a wrapper around all sorts of external build systems.
+
+However, systems like [Bazel][] take an alternative approach in which the
+compiler is invoked directly and the composition of programs and libraries stays
+within a single homogeneous build system.
+
+Users don't need to learn per-language build systems and especially for
+companies with large monorepo-setups ([like Google][]) this has huge
+productivity impact.
+
+This project is an attempt to prove that Nix can be used in a similar style to
+build software directly, rather than shelling out to other build systems.
+
+## Example
+
+Given a program layout like this:
+
+```
+.
+├── lib          <-- some library component
+│   ├── bar.go
+│   └── foo.go
+├── api.proto    <-- gRPC API definition
+├── main.go      <-- program implementation
+└── default.nix  <-- build instructions
+```
+
+The contents of `default.nix` could look like this:
+
+```nix
+{ buildGo }:
+
+let
+  api = buildGo.grpc {
+    name  = "someapi";
+    proto = ./api.proto;
+  };
+
+  lib = buildGo.package {
+    name = "somelib";
+    srcs = [
+      ./lib/bar.go
+      ./lib/foo.go
+    ];
+  };
+in buildGo.program {
+  name = "my-program";
+  deps = [ api lib ];
+
+  srcs = [
+    ./main.go
+  ];
+}
+```
+
+(If you don't know how to read Nix, check out [nix-1p][])
+
+## Usage
+
+`buildGo` exposes five different functions:
+
+* `buildGo.program`: Build a Go binary out of the specified source files.
+
+  | parameter | type                    | use                                            | required? |
+  |-----------|-------------------------|------------------------------------------------|-----------|
+  | `name`    | `string`                | Name of the program (and resulting executable) | yes       |
+  | `srcs`    | `list<path>`            | List of paths to source files                  | yes       |
+  | `deps`    | `list<drv>`             | List of dependencies (i.e. other Go libraries) | no        |
+  | `x_defs`  | `attrs<string, string>` | Attribute set of linker vars (i.e. `-X`-flags) | no        |
+
+* `buildGo.package`: Build a Go library out of the specified source files.
+
+  | parameter | type         | use                                            | required? |
+  |-----------|--------------|------------------------------------------------|-----------|
+  | `name`    | `string`     | Name of the library (and resulting executable) | yes       |
+  | `srcs`    | `list<path>` | List of paths to source files                  | yes       |
+  | `deps`    | `list<drv>`  | List of dependencies (i.e. other Go libraries) | no        |
+  | `path`    | `string`     | Go import path for the resulting library       | no        |
+
+* `buildGo.external`: Build a Go library or program using standard `go` tooling.
+
+  This exists for compatibility with complex external dependencies. In theory it
+  is possible to write `buildGo.package` specifications for each subpackage of
+  an external dependency, but it is often cumbersome to do so.
+
+  | parameter | type           | use                                            | required? |
+  |-----------|----------------|------------------------------------------------|-----------|
+  | `path`    | `string`       | Go import path for the resulting library       | yes       |
+  | `src`     | `path`         | Path to the source **directory**               | yes       |
+  | `deps`    | `list<drv>`    | List of dependencies (i.e. other Go libraries) | no        |
+  | `srcOnly` | `bool`         | Only copy sources, do not perform a build.     | no        |
+  | `targets` | `list<string>` | Sub-packages to build (defaults to all)        | no        |
+
+  For some examples of how `buildGo.external` is used, check out
+  [`proto.nix`](./proto.nix).
+
+* `buildGo.proto`: Build a Go library out of the specified Protobuf definition.
+
+  | parameter   | type        | use                                              | required? |
+  |-------------|-------------|--------------------------------------------------|-----------|
+  | `name`      | `string`    | Name for the resulting library                   | yes       |
+  | `proto`     | `path`      | Path to the Protobuf definition file             | yes       |
+  | `path`      | `string`    | Import path for the resulting Go library         | no        |
+  | `extraDeps` | `list<drv>` | Additional Go dependencies to add to the library | no        |
+
+* `buildGo.grpc`: Build a Go library out of the specified gRPC definition.
+
+  The parameters are identical to `buildGo.proto`.
+
+## Current status
+
+This project is work-in-progress. Crucially it is lacking the following features:
+
+* feature flag parity with Bazel's Go rules
+* documentation building
+* test execution
+
+There are still some open questions around how to structure some of those
+features in Nix.
+
+[Nix]: https://nixos.org/nix/
+[Go]: https://golang.org/
+[Nixery]: https://github.com/google/nixery
+[Bazel]: https://bazel.build/
+[like Google]: https://ai.google/research/pubs/pub45424
+[nix-1p]: https://github.com/tazjin/nix-1p

From 6a526620e25a349642b161eb687dec2e005360e5 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Mon, 9 Dec 2019 00:51:27 +0000
Subject: [PATCH 12/36] feat(buildGo): Add 'overrideGo' argument overriding
 function

This makes it possible to override arguments to the Go builders
downstream in the style of `overrideAttrs` from standard nixpkgs
derivations.

For example, given a Nix value `foo` that builds a binary called `foo`
the name of this binary could be changed and a new dependency on
`somelib` added like so:

    foo.overrideGo(old: {
      name = "bar";
      deps = old.deps ++ [ somelib ];
    })
---
 buildGo.nix | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/buildGo.nix b/buildGo.nix
index fba4e18e9..5f5118e8c 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -48,6 +48,13 @@ let
 
   pathToName = p: replaceStrings ["/"] ["_"] (toString p);
 
+  # Add an `overrideGo` attribute to a function result that works
+  # similar to `overrideAttrs`, but is used specifically for the
+  # arguments passed to Go builders.
+  makeOverridable = f: orig: (f orig) // {
+    overrideGo = new: makeOverridable f (orig // (new orig));
+  };
+
   # High-level build functions
 
   # Build a Go program out of the specified files and dependencies.
@@ -130,6 +137,11 @@ let
     inherit external;
   };
 in {
-  # Only the high-level builder functions are exposed
-  inherit program package proto grpc external;
+  # Only the high-level builder functions are exposed, but made
+  # overrideable.
+  program = makeOverridable program;
+  package = makeOverridable package;
+  proto = makeOverridable proto;
+  grpc = makeOverridable grpc;
+  external = makeOverridable external;
 }

From c40e8a4061b033282031cd1a8d64df9a591aa694 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Mon, 9 Dec 2019 00:27:05 +0000
Subject: [PATCH 13/36] feat(buildGo): Add new traversing external'
 implementation

Adds an alternative implementation of a builder for external packages
which traverses packages and builds up an attribute set tree out of
their structure.

Currently this is not functional because there is no useable method of
specifying dependencies within that package set.
---
 buildGo.nix | 55 ++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 46 insertions(+), 9 deletions(-)

diff --git a/buildGo.nix b/buildGo.nix
index 5f5118e8c..366be770a 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -14,10 +14,12 @@ let
     dirOf
     elemAt
     filter
+    listToAttrs
     map
     match
     readDir
-    replaceStrings;
+    replaceStrings
+    toString;
 
   inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin;
 
@@ -34,14 +36,6 @@ let
   srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}";
   srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs);
 
-  isGoFile = f: (match ".*\.go" f) != null;
-  isGoTest = f: (match ".*_test\.go" f) != null;
-  goFileFilter = k: v: (v == "regular") && (isGoFile k) && (!isGoTest k);
-  goFilesIn = dir:
-    let files = readDir dir;
-        goFiles = filter (f: goFileFilter f files."${f}") (attrNames files);
-    in map (f: dir + "/" + f) goFiles;
-
   allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps)));
 
   xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs));
@@ -93,6 +87,41 @@ let
   # Build a Go library out of the specified gRPC definition.
   grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc ]; });
 
+  # Traverse an externally defined Go library to build up a tree of
+  # its packages.
+  #
+  # TODO(tazjin): Automatically infer which packages depend on which
+  # other packages, which currently requires overriding.
+  #
+  # TODO(tazjin): Add support for rewriting package paths.
+  external' = { src, path, deps ? [] }:
+    let
+      dir = readDir src;
+      isGoFile = f: (match ".*\.go" f) != null;
+      isGoTest = f: (match ".*_test\.go" f) != null;
+      goFileFilter = k: v: (v == "regular") && (isGoFile k) && (!isGoTest k);
+      goSources =
+        let goFiles = filter (f: goFileFilter f dir."${f}") (attrNames dir);
+        in map (f: src + ("/" + f)) goFiles;
+
+      subDirs = filter (n: dir."${n}" == "directory") (attrNames dir);
+      subPackages = map (name: {
+        inherit name;
+        value = external' {
+          inherit deps;
+          src = src + ("/" + name);
+          path = path + ("/" + name);
+        };
+      }) subDirs;
+      subAttrs = listToAttrs (filter (p: p.value != {}) subPackages);
+
+      current = package {
+        inherit deps path;
+        name = pathToName path;
+        srcs = goSources;
+      };
+    in if goSources == [] then subAttrs else (current // subAttrs);
+
   # Build an externally defined Go library using `go build` itself.
   #
   # Libraries built this way can be included in any standard buildGo
@@ -144,4 +173,12 @@ in {
   proto = makeOverridable proto;
   grpc = makeOverridable grpc;
   external = makeOverridable external;
+
+  # TODO: remove
+  inherit external';
+
+  extTest = external' {
+    src = /home/tazjin/go/src/cloud.google.com/go;
+    path = "cloud.google.com/go";
+  };
 }

From 57f37743c2e295b0404023587b92c517e5731a30 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Wed, 11 Dec 2019 09:54:02 +0000
Subject: [PATCH 14/36] fix(buildGo): Ensure 'proto' libraries are overridable

---
 buildGo.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildGo.nix b/buildGo.nix
index 366be770a..b5a6ba2b3 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -73,7 +73,7 @@ let
   '') // { goDeps = uniqueDeps; };
 
   # Build a Go library out of the specified protobuf definition.
-  proto = { name, proto, path ? name, extraDeps ? [] }: package {
+  proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) {
     inherit name path;
     deps = [ protoLibs.goProto ] ++ extraDeps;
     srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} ''

From e60dfabc2122bbc29fc9832d1562826c08772442 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 03:14:27 +0000
Subject: [PATCH 15/36] feat(buildGo): Expose Go import path for packages

---
 buildGo.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildGo.nix b/buildGo.nix
index b5a6ba2b3..c03c30bef 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -70,7 +70,7 @@ let
     mkdir -p $out/${path}
     ${srcList path (map (s: "${s}") srcs)}
     ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs}
-  '') // { goDeps = uniqueDeps; };
+  '') // { goDeps = uniqueDeps; goImportPath = path; };
 
   # Build a Go library out of the specified protobuf definition.
   proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) {

From fb4dd761461de12ccbc276431ea8aa37dcefa9b0 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 03:23:09 +0000
Subject: [PATCH 16/36] feat(external): Implement tool to analyse external
 dependencies

Adds a tool that can analyse dependencies that were not originally
meant to be built with buildGo.nix and return information that can be
used to construct appropriate Nix dependencies.

The tool will return information about package-local and foreign
dependencies separately to let Nix determine whether all required
dependencies are provided and to correctly link together sub-packages.

To avoid listing standard library imports in the dependencies, a list
of all packages in the standard library is generated statically to
allow for those to be filtered out during the analysis.

This tool is still work-in-progress.
---
 external/default.nix |  29 ++++++++
 external/main.go     | 159 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 188 insertions(+)
 create mode 100644 external/default.nix
 create mode 100644 external/main.go

diff --git a/external/default.nix b/external/default.nix
new file mode 100644
index 000000000..79d559c05
--- /dev/null
+++ b/external/default.nix
@@ -0,0 +1,29 @@
+# Copyright 2019 Google LLC.
+# SPDX-License-Identifier: Apache-2.0
+{ runCommand, go, jq, ripgrep, program }:
+
+let
+  # Collect all non-vendored dependencies from the Go standard library
+  # into a file that can be used to filter them out when processing
+  # dependencies.
+  stdlibPackages = runCommand "stdlib-pkgs.json" {} ''
+    export GOPATH=/dev/null
+    ${go}/bin/go list all | \
+      ${ripgrep}/bin/rg -v 'vendor' | \
+      ${jq}/bin/jq -R '.' | \
+      ${jq}/bin/jq -c -s 'map({key: ., value: true}) | from_entries' \
+      > $out
+  '';
+
+  analyser = program {
+    name = "analyser";
+
+    srcs = [
+      ./main.go
+    ];
+
+    x_defs = {
+      "main.stdlibList" = "${stdlibPackages}";
+    };
+  };
+in analyser
diff --git a/external/main.go b/external/main.go
new file mode 100644
index 000000000..0f8834fc9
--- /dev/null
+++ b/external/main.go
@@ -0,0 +1,159 @@
+// Copyright 2019 Google LLC.
+// SPDX-License-Identifier: Apache-2.0
+
+// This tool analyses external (i.e. not built with `buildGo.nix`) Go
+// packages to determine a build plan that Nix can import.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"go/build"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// Path to a JSON file describing all standard library import paths.
+// This file is generated and set here by Nix during the build
+// process.
+var stdlibList string
+
+// pkg describes a single Go package within the specified source
+// directory.
+//
+// Return information includes the local (relative from project root)
+// and external (none-stdlib) dependencies of this package.
+type pkg struct {
+	Name        string   `json:"name"`
+	Source      string   `json:"source"`
+	Files       []string `json:"files"`
+	LocalDeps   []string `json:"localDeps"`
+	ForeignDeps []string `json:"foreignDeps"`
+}
+
+// findGoDirs returns a filepath.WalkFunc that identifies all
+// directories that contain Go source code in a certain tree.
+func findGoDirs(at string) ([]string, error) {
+	var goDirs []string
+	dir := ""
+
+	err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error {
+		// Skip testdata
+		if info.IsDir() && info.Name() == "testdata" {
+			return filepath.SkipDir
+		}
+
+		// Keep track of the last seen directory.
+		if info.IsDir() {
+			dir = path
+			return nil
+		}
+
+		// If the directory has already been "popped", nothing else needs
+		// to happen.
+		if dir == "" {
+			return nil
+		}
+
+		// If the current file is a Go file, then the directory is popped
+		// (i.e. marked as a Go directory).
+		if strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") {
+			goDirs = append(goDirs, dir)
+			dir = ""
+			return nil
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return goDirs, nil
+}
+
+// analysePackage loads and analyses the imports of a single Go
+// package, returning the data that is required by the Nix code to
+// generate a derivation for this package.
+func analysePackage(root, source, path string, stdlib map[string]bool) (pkg, error) {
+	ctx := build.Default
+
+	p, err := ctx.ImportDir(source, build.IgnoreVendor)
+	if err != nil {
+		return pkg{}, err
+	}
+
+	local := []string{}
+	foreign := []string{}
+
+	for _, i := range p.Imports {
+		if stdlib[i] {
+			continue
+		}
+
+		if strings.HasPrefix(i, path) {
+			local = append(local, i)
+		} else {
+			foreign = append(foreign, i)
+		}
+	}
+
+	analysed := pkg{
+		Name:        strings.TrimPrefix(source, root+"/"),
+		Source:      source,
+		Files:       p.GoFiles,
+		LocalDeps:   local,
+		ForeignDeps: foreign,
+	}
+
+	return analysed, nil
+}
+
+func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) {
+	f, err := ioutil.ReadFile(from)
+	if err != nil {
+		return
+	}
+
+	err = json.Unmarshal(f, &pkgs)
+	return
+}
+
+func main() {
+	// TODO(tazjin): Remove default values
+	source := flag.String("source", "/nix/store/fzp67ris29zg5zfs65z0q245x0fahgll-source", "path to directory with sources to process")
+	path := flag.String("path", "github.com/golang/protobuf", "import path for the package")
+
+	flag.Parse()
+
+	if *source == "" {
+		log.Fatalf("-source flag must be specified")
+	}
+
+	stdlibPkgs, err := loadStdlibPkgs(stdlibList)
+	if err != nil {
+		log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err)
+	}
+
+	goDirs, err := findGoDirs(*source)
+	if err != nil {
+		log.Fatalf("failed to walk source directory '%s': %s\n", source, err)
+	}
+
+	all := []pkg{}
+	for _, d := range goDirs {
+		analysed, err := analysePackage(*source, d, *path, stdlibPkgs)
+		if err != nil {
+			log.Fatalf("failed to analyse package at %q: %s", d, err)
+		}
+		all = append(all, analysed)
+	}
+
+	j, _ := json.MarshalIndent(all, "", "  ") // TODO: no indent
+	fmt.Println(string(j))
+}

From a5473293e78fc0b01def5c4c3f7e11854d88d8d0 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 03:46:28 +0000
Subject: [PATCH 17/36] feat(external): Return references in more useable
 format for Nix

Sub-packages of external dependencies are traversed by Nix as a tree
of attribute sets which need to be accessed by "path". To make this
easier, the dependency analyser now returns "paths" as string lists.
---
 external/main.go | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/external/main.go b/external/main.go
index 0f8834fc9..2e5ee4900 100644
--- a/external/main.go
+++ b/external/main.go
@@ -13,6 +13,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 )
@@ -28,11 +29,10 @@ var stdlibList string
 // Return information includes the local (relative from project root)
 // and external (none-stdlib) dependencies of this package.
 type pkg struct {
-	Name        string   `json:"name"`
-	Source      string   `json:"source"`
-	Files       []string `json:"files"`
-	LocalDeps   []string `json:"localDeps"`
-	ForeignDeps []string `json:"foreignDeps"`
+	Name        []string   `json:"name"`
+	Files       []string   `json:"files"`
+	LocalDeps   [][]string `json:"localDeps"`
+	ForeignDeps []string   `json:"foreignDeps"`
 }
 
 // findGoDirs returns a filepath.WalkFunc that identifies all
@@ -80,7 +80,7 @@ func findGoDirs(at string) ([]string, error) {
 // analysePackage loads and analyses the imports of a single Go
 // package, returning the data that is required by the Nix code to
 // generate a derivation for this package.
-func analysePackage(root, source, path string, stdlib map[string]bool) (pkg, error) {
+func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) {
 	ctx := build.Default
 
 	p, err := ctx.ImportDir(source, build.IgnoreVendor)
@@ -88,7 +88,7 @@ func analysePackage(root, source, path string, stdlib map[string]bool) (pkg, err
 		return pkg{}, err
 	}
 
-	local := []string{}
+	local := [][]string{}
 	foreign := []string{}
 
 	for _, i := range p.Imports {
@@ -96,17 +96,22 @@ func analysePackage(root, source, path string, stdlib map[string]bool) (pkg, err
 			continue
 		}
 
-		if strings.HasPrefix(i, path) {
-			local = append(local, i)
+		if strings.HasPrefix(i, importpath) {
+			local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/"))
 		} else {
 			foreign = append(foreign, i)
 		}
 	}
 
+	prefix := strings.TrimPrefix(source, root+"/")
+	files := []string{}
+	for _, f := range p.GoFiles {
+		files = append(files, path.Join(prefix, f))
+	}
+
 	analysed := pkg{
-		Name:        strings.TrimPrefix(source, root+"/"),
-		Source:      source,
-		Files:       p.GoFiles,
+		Name:        strings.Split(prefix, "/"),
+		Files:       files,
 		LocalDeps:   local,
 		ForeignDeps: foreign,
 	}
@@ -125,9 +130,8 @@ func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) {
 }
 
 func main() {
-	// TODO(tazjin): Remove default values
-	source := flag.String("source", "/nix/store/fzp67ris29zg5zfs65z0q245x0fahgll-source", "path to directory with sources to process")
-	path := flag.String("path", "github.com/golang/protobuf", "import path for the package")
+	source := flag.String("source", "", "path to directory with sources to process")
+	path := flag.String("path", "", "import path for the package")
 
 	flag.Parse()
 
@@ -154,6 +158,6 @@ func main() {
 		all = append(all, analysed)
 	}
 
-	j, _ := json.MarshalIndent(all, "", "  ") // TODO: no indent
+	j, _ := json.Marshal(all)
 	fmt.Println(string(j))
 }

From b20e46d60b14d5e367e52ec14989dc27b4829774 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 16:33:10 +0000
Subject: [PATCH 18/36] fix(external): Ensure findGoDirs "finds" top-level
 directory

Due to the lexical walk order of `filepath.Walk` the previous
directory identification logic failed under certain conditions if the
top-level directory contained Go files that showed up *after* the
first subdirectories.

To simplify the logic a set of directories is now gathered instead on
a file-level.
---
 external/main.go | 30 ++++++++++--------------------
 1 file changed, 10 insertions(+), 20 deletions(-)

diff --git a/external/main.go b/external/main.go
index 2e5ee4900..0c1d84d5b 100644
--- a/external/main.go
+++ b/external/main.go
@@ -38,33 +38,18 @@ type pkg struct {
 // findGoDirs returns a filepath.WalkFunc that identifies all
 // directories that contain Go source code in a certain tree.
 func findGoDirs(at string) ([]string, error) {
-	var goDirs []string
-	dir := ""
+	dirSet := make(map[string]bool)
 
 	err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error {
-		// Skip testdata
-		if info.IsDir() && info.Name() == "testdata" {
+		// Skip folders that are guaranteed to not be relevant
+		if info.IsDir() && (info.Name() == "testdata" || info.Name() == ".git") {
 			return filepath.SkipDir
 		}
 
-		// Keep track of the last seen directory.
-		if info.IsDir() {
-			dir = path
-			return nil
-		}
-
-		// If the directory has already been "popped", nothing else needs
-		// to happen.
-		if dir == "" {
-			return nil
-		}
-
 		// If the current file is a Go file, then the directory is popped
 		// (i.e. marked as a Go directory).
-		if strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") {
-			goDirs = append(goDirs, dir)
-			dir = ""
-			return nil
+		if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") {
+			dirSet[filepath.Dir(path)] = true
 		}
 
 		return nil
@@ -74,6 +59,11 @@ func findGoDirs(at string) ([]string, error) {
 		return nil, err
 	}
 
+	goDirs := []string{}
+	for k, _ := range dirSet {
+		goDirs = append(goDirs, k)
+	}
+
 	return goDirs, nil
 }
 

From 1fd80fb20175c28bc500e6d889594d1a5ebb6be7 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 17:04:40 +0000
Subject: [PATCH 19/36] fix(external): Correctly set names for root packages

Fixes the prefix trimming logic for package names and source files if
the source files appear in the package root (which is, unsurprisingly,
very common).
---
 external/main.go | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/external/main.go b/external/main.go
index 0c1d84d5b..028703e38 100644
--- a/external/main.go
+++ b/external/main.go
@@ -94,19 +94,27 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk
 	}
 
 	prefix := strings.TrimPrefix(source, root+"/")
+
+	name := []string{}
+	if len(prefix) != len(source) {
+		name = strings.Split(prefix, "/")
+	} else {
+		// Otherwise, the name is empty since its the root package and no
+		// prefix should be added to files.
+		prefix = ""
+	}
+
 	files := []string{}
 	for _, f := range p.GoFiles {
 		files = append(files, path.Join(prefix, f))
 	}
 
-	analysed := pkg{
-		Name:        strings.Split(prefix, "/"),
+	return pkg{
+		Name:        name,
 		Files:       files,
 		LocalDeps:   local,
 		ForeignDeps: foreign,
-	}
-
-	return analysed, nil
+	}, nil
 }
 
 func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) {

From bbf3a418a5a119e2ab6aaaf1e04f42caa1b683a8 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 22:58:51 +0000
Subject: [PATCH 20/36] feat(external): Add fully qualified import path to
 analyser output

This is used by Nix to build the derivation names for individual
packages.
---
 external/main.go | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/external/main.go b/external/main.go
index 028703e38..aac966172 100644
--- a/external/main.go
+++ b/external/main.go
@@ -29,7 +29,8 @@ var stdlibList string
 // Return information includes the local (relative from project root)
 // and external (none-stdlib) dependencies of this package.
 type pkg struct {
-	Name        []string   `json:"name"`
+	Name        string     `json:"name"`
+	Locator     []string   `json:"locator"`
 	Files       []string   `json:"files"`
 	LocalDeps   [][]string `json:"localDeps"`
 	ForeignDeps []string   `json:"foreignDeps"`
@@ -95,12 +96,12 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk
 
 	prefix := strings.TrimPrefix(source, root+"/")
 
-	name := []string{}
+	locator := []string{}
 	if len(prefix) != len(source) {
-		name = strings.Split(prefix, "/")
+		locator = strings.Split(prefix, "/")
 	} else {
-		// Otherwise, the name is empty since its the root package and no
-		// prefix should be added to files.
+		// Otherwise, the locator is empty since its the root package and
+		// no prefix should be added to files.
 		prefix = ""
 	}
 
@@ -110,7 +111,8 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk
 	}
 
 	return pkg{
-		Name:        name,
+		Name:        path.Join(importpath, prefix),
+		Locator:     locator,
 		Files:       files,
 		LocalDeps:   local,
 		ForeignDeps: foreign,

From c5373a69fe440bada29f7e2fc41882890f7d0872 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 23:00:20 +0000
Subject: [PATCH 21/36] feat(external): Implement builder function for
 externals

Implements a builder function that calls the analysis tool on the
provided source and builds up the required attribute set, including
local dependencies.
---
 external/default.nix | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/external/default.nix b/external/default.nix
index 79d559c05..a065ab333 100644
--- a/external/default.nix
+++ b/external/default.nix
@@ -1,8 +1,13 @@
 # Copyright 2019 Google LLC.
 # SPDX-License-Identifier: Apache-2.0
-{ runCommand, go, jq, ripgrep, program }:
+{ pkgs, program, package }:
 
 let
+  inherit (builtins) foldl'fromJSON head readFile replaceStrings tail throw;
+  inherit (pkgs) lib runCommand go jq ripgrep;
+
+  pathToName = p: replaceStrings ["/"] ["_"] (toString p);
+
   # Collect all non-vendored dependencies from the Go standard library
   # into a file that can be used to filter them out when processing
   # dependencies.
@@ -26,4 +31,26 @@ let
       "main.stdlibList" = "${stdlibPackages}";
     };
   };
-in analyser
+
+  mkset = path: value:
+    if path == [] then { gopkg = value; }
+    else { "${head path}" = mkset (tail path) value; };
+
+  toPackage = self: src: path: entry: package {
+    name = pathToName entry.name entry.name;
+    path = lib.concatStringsSep "/" ([ path ] ++ entry.locator);
+    srcs = map (f: src + ("/" + f)) entry.files;
+    deps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) (
+      throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'"
+    ) self) entry.localDeps;
+  };
+
+in { src, path, deps ? [] }: let
+  name = pathToName path;
+  analysisOutput = runCommand "${name}-structure.json" {} ''
+    ${analyser}/bin/analyser -path ${path} -source ${src} > $out
+  '';
+  analysis = fromJSON (readFile analysisOutput);
+in lib.fix(self: foldl' lib.recursiveUpdate {} (
+  map (entry: mkset entry.locator (toPackage self src path entry)) analysis
+))

From f5e3183de14e73ab6ff0ada0a6d2e9408a5280be Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 23:07:42 +0000
Subject: [PATCH 22/36] feat(buildGo): Expose new external builder

---
 buildGo.nix | 101 ++++++++--------------------------------------------
 1 file changed, 14 insertions(+), 87 deletions(-)

diff --git a/buildGo.nix b/buildGo.nix
index c03c30bef..a470c1db5 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -72,10 +72,23 @@ let
     ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs}
   '') // { goDeps = uniqueDeps; goImportPath = path; };
 
+  # Build a tree of Go libraries out of an external Go source
+  # directory that follows the standard Go layout and was not built
+  # with buildGo.nix.
+  #
+  # The derivation for each actual dependency will reside in an
+  # attribute named "gopkg".
+  external = import ./external { inherit pkgs program package; };
+
+  # Import support libraries needed for protobuf & gRPC support
+  protoLibs = import ./proto.nix {
+    inherit external;
+  };
+
   # Build a Go library out of the specified protobuf definition.
   proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) {
     inherit name path;
-    deps = [ protoLibs.goProto ] ++ extraDeps;
+    deps = [ protoLibs'.protobuf ] ++ extraDeps;
     srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} ''
       cp ${proto} ${baseNameOf proto}
       ${protobuf}/bin/protoc --plugin=${protoLibs.goProto}/bin/protoc-gen-go \
@@ -87,84 +100,6 @@ let
   # Build a Go library out of the specified gRPC definition.
   grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc ]; });
 
-  # Traverse an externally defined Go library to build up a tree of
-  # its packages.
-  #
-  # TODO(tazjin): Automatically infer which packages depend on which
-  # other packages, which currently requires overriding.
-  #
-  # TODO(tazjin): Add support for rewriting package paths.
-  external' = { src, path, deps ? [] }:
-    let
-      dir = readDir src;
-      isGoFile = f: (match ".*\.go" f) != null;
-      isGoTest = f: (match ".*_test\.go" f) != null;
-      goFileFilter = k: v: (v == "regular") && (isGoFile k) && (!isGoTest k);
-      goSources =
-        let goFiles = filter (f: goFileFilter f dir."${f}") (attrNames dir);
-        in map (f: src + ("/" + f)) goFiles;
-
-      subDirs = filter (n: dir."${n}" == "directory") (attrNames dir);
-      subPackages = map (name: {
-        inherit name;
-        value = external' {
-          inherit deps;
-          src = src + ("/" + name);
-          path = path + ("/" + name);
-        };
-      }) subDirs;
-      subAttrs = listToAttrs (filter (p: p.value != {}) subPackages);
-
-      current = package {
-        inherit deps path;
-        name = pathToName path;
-        srcs = goSources;
-      };
-    in if goSources == [] then subAttrs else (current // subAttrs);
-
-  # Build an externally defined Go library using `go build` itself.
-  #
-  # Libraries built this way can be included in any standard buildGo
-  # build.
-  #
-  # Contrary to other functions, `src` is expected to point at a
-  # single directory containing the root of the external library.
-  external = { path, src, deps ? [], srcOnly ? false, targets ? [ "..." ] }:
-    let
-      name = pathToName path;
-      uniqueDeps = allDeps deps;
-      srcDir = runCommand "goext-src-${name}" {} ''
-        mkdir -p $out/${dirOf path}
-        cp -r ${src} $out/${dirOf path}/${baseNameOf path}
-      '';
-      gopathSrc = symlinkJoin {
-        name = "gopath-${name}";
-        paths = uniqueDeps ++ [ srcDir ];
-      };
-      gopathPkg = runCommand "goext-pkg-${name}" {} ''
-        mkdir -p gopath $out
-        export GOPATH=$PWD/gopath
-        ln -s ${gopathSrc} gopath/src
-        ${go}/bin/go install ${spaceOut (map (t: path + "/" + t) targets)}
-
-        if [[ -d gopath/pkg/linux_amd64 ]]; then
-          echo "Installing Go packages for ${path}"
-          mv gopath/pkg/linux_amd64/* $out
-        fi
-
-        if [[ -d gopath/bin ]]; then
-          echo "Installing Go binaries for ${path}"
-          mv gopath/bin $out/bin
-        fi
-      '';
-    in (if srcOnly then gopathSrc else symlinkJoin {
-      name = "goext-${name}";
-      paths = [ gopathSrc gopathPkg ];
-    }) // { goDeps = uniqueDeps; };
-
-  protoLibs = import ./proto.nix {
-    inherit external;
-  };
 in {
   # Only the high-level builder functions are exposed, but made
   # overrideable.
@@ -173,12 +108,4 @@ in {
   proto = makeOverridable proto;
   grpc = makeOverridable grpc;
   external = makeOverridable external;
-
-  # TODO: remove
-  inherit external';
-
-  extTest = external' {
-    src = /home/tazjin/go/src/cloud.google.com/go;
-    path = "cloud.google.com/go";
-  };
 }

From 875628a0978937d708ce9365c9cc2061f39b0bca Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 23:26:08 +0000
Subject: [PATCH 23/36] feat(external): Switch between packages & programs
 automatically

---
 external/default.nix | 28 +++++++++++++++++++---------
 external/main.go     |  2 ++
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/external/default.nix b/external/default.nix
index a065ab333..5854d4f02 100644
--- a/external/default.nix
+++ b/external/default.nix
@@ -3,7 +3,7 @@
 { pkgs, program, package }:
 
 let
-  inherit (builtins) foldl'fromJSON head readFile replaceStrings tail throw;
+  inherit (builtins) elemAt foldl' fromJSON head length readFile replaceStrings tail throw;
   inherit (pkgs) lib runCommand go jq ripgrep;
 
   pathToName = p: replaceStrings ["/"] ["_"] (toString p);
@@ -36,14 +36,24 @@ let
     if path == [] then { gopkg = value; }
     else { "${head path}" = mkset (tail path) value; };
 
-  toPackage = self: src: path: entry: package {
-    name = pathToName entry.name entry.name;
-    path = lib.concatStringsSep "/" ([ path ] ++ entry.locator);
-    srcs = map (f: src + ("/" + f)) entry.files;
-    deps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) (
-      throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'"
-    ) self) entry.localDeps;
-  };
+  last = l: elemAt l ((length l) - 1);
+
+  toPackage = self: src: path: entry:
+    let
+      args = {
+        srcs = map (f: src + ("/" + f)) entry.files;
+        deps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) (
+          throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'"
+        ) self) entry.localDeps;
+      };
+      libArgs = args // {
+        name = pathToName entry.name;
+        path = lib.concatStringsSep "/" ([ path ] ++ entry.locator);
+      };
+      binArgs = args // {
+        name = last ([ path ] ++ entry.locator);
+      };
+    in if entry.isCommand then (program binArgs) else (package libArgs);
 
 in { src, path, deps ? [] }: let
   name = pathToName path;
diff --git a/external/main.go b/external/main.go
index aac966172..23fd53326 100644
--- a/external/main.go
+++ b/external/main.go
@@ -34,6 +34,7 @@ type pkg struct {
 	Files       []string   `json:"files"`
 	LocalDeps   [][]string `json:"localDeps"`
 	ForeignDeps []string   `json:"foreignDeps"`
+	IsCommand   bool       `json:"isCommand"`
 }
 
 // findGoDirs returns a filepath.WalkFunc that identifies all
@@ -116,6 +117,7 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk
 		Files:       files,
 		LocalDeps:   local,
 		ForeignDeps: foreign,
+		IsCommand:   p.IsCommand(),
 	}, nil
 }
 

From 0dfa5d15fae5b8f5992625f03586ba6043bcd475 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 23:27:43 +0000
Subject: [PATCH 24/36] fix(buildGo): Update buildGo.proto for compatibility
 with external

Changes in the structure of buildGo.external meant that the package
layout for the protobuf library is now slightly different.

`proto` has been amended to work with the new structure. Callers of
buildGo.proto do not need to be updated (i.e. the example still
works).
---
 buildGo.nix | 8 ++++----
 proto.nix   | 4 ----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/buildGo.nix b/buildGo.nix
index a470c1db5..0df3c955a 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -76,8 +76,8 @@ let
   # directory that follows the standard Go layout and was not built
   # with buildGo.nix.
   #
-  # The derivation for each actual dependency will reside in an
-  # attribute named "gopkg".
+  # The derivation for each actual package will reside in an attribute
+  # named "gopkg", and an attribute named "gobin" for binaries.
   external = import ./external { inherit pkgs program package; };
 
   # Import support libraries needed for protobuf & gRPC support
@@ -88,10 +88,10 @@ let
   # Build a Go library out of the specified protobuf definition.
   proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) {
     inherit name path;
-    deps = [ protoLibs'.protobuf ] ++ extraDeps;
+    deps = [ protoLibs.goProto.proto.gopkg ] ++ extraDeps;
     srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} ''
       cp ${proto} ${baseNameOf proto}
-      ${protobuf}/bin/protoc --plugin=${protoLibs.goProto}/bin/protoc-gen-go \
+      ${protobuf}/bin/protoc --plugin=${protoLibs.goProto.protoc-gen-go.gopkg}/bin/protoc-gen-go \
         --go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto}
       mv *.pb.go $out
     '');
diff --git a/proto.nix b/proto.nix
index e773af031..dd40def0f 100644
--- a/proto.nix
+++ b/proto.nix
@@ -19,7 +19,6 @@ in rec {
 
   xnet = external {
     path = "golang.org/x/net";
-    srcOnly = true;
     deps = [ xtext ];
     src = fetchGit {
       url = "https://go.googlesource.com/net";
@@ -29,7 +28,6 @@ in rec {
 
   xsys = external {
     path = "golang.org/x/sys";
-    srcOnly = true;
     src = fetchGit {
       url = "https://go.googlesource.com/sys";
       rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd";
@@ -38,7 +36,6 @@ in rec {
 
   xtext = external {
     path = "golang.org/x/text";
-    srcOnly = true;
     src = fetchGit {
       url = "https://go.googlesource.com/text";
       rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686";
@@ -47,7 +44,6 @@ in rec {
 
   genproto = external {
     path = "google.golang.org/genproto";
-    srcOnly = true;
     src = fetchGit {
       url = "https://github.com/google/go-genproto";
       rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6";

From 39e73f42b21e369906442fc3235cf9ce62f25ce1 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 23:30:47 +0000
Subject: [PATCH 25/36] style(external): Minor formatting fixes

---
 external/default.nix | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/external/default.nix b/external/default.nix
index 5854d4f02..f9cb542e6 100644
--- a/external/default.nix
+++ b/external/default.nix
@@ -3,7 +3,17 @@
 { pkgs, program, package }:
 
 let
-  inherit (builtins) elemAt foldl' fromJSON head length readFile replaceStrings tail throw;
+  inherit (builtins)
+    elemAt
+    foldl'
+    fromJSON
+    head
+    length
+    readFile
+    replaceStrings
+    tail
+    throw;
+
   inherit (pkgs) lib runCommand go jq ripgrep;
 
   pathToName = p: replaceStrings ["/"] ["_"] (toString p);

From 218b81c709248784112ae7bd0646c7fc6773fe5f Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 12 Dec 2019 23:53:39 +0000
Subject: [PATCH 26/36] fix(external): Skip folders with "no buildable Go
 files"

This error is returned by the build analysis logic if the target
package does not have any relevant Go files, which might be the case
if `+build` flags and such are used.
---
 external/main.go | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/external/main.go b/external/main.go
index 23fd53326..e0b95605f 100644
--- a/external/main.go
+++ b/external/main.go
@@ -43,14 +43,15 @@ func findGoDirs(at string) ([]string, error) {
 	dirSet := make(map[string]bool)
 
 	err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error {
+		name := info.Name()
 		// Skip folders that are guaranteed to not be relevant
-		if info.IsDir() && (info.Name() == "testdata" || info.Name() == ".git") {
+		if info.IsDir() && (name == "testdata" || name == ".git") {
 			return filepath.SkipDir
 		}
 
 		// If the current file is a Go file, then the directory is popped
 		// (i.e. marked as a Go directory).
-		if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") && !strings.HasSuffix(info.Name(), "_test.go") {
+		if !info.IsDir() && strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") {
 			dirSet[filepath.Dir(path)] = true
 		}
 
@@ -154,6 +155,16 @@ func main() {
 	all := []pkg{}
 	for _, d := range goDirs {
 		analysed, err := analysePackage(*source, d, *path, stdlibPkgs)
+
+		// If the Go source analysis returned "no buildable Go files",
+		// that directory should be skipped.
+		//
+		// This might be due to `+build` flags on the platform and other
+		// reasons (such as test files).
+		if _, ok := err.(*build.NoGoError); ok {
+			continue
+		}
+
 		if err != nil {
 			log.Fatalf("failed to analyse package at %q: %s", d, err)
 		}

From d8ef203f78ba58968d26fb3a749cb32170394500 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Fri, 13 Dec 2019 00:07:47 +0000
Subject: [PATCH 27/36] feat(external): Support foreign dependencies in
 external packages

Users can supply a list of foreign dependencies in calls to
buildGo.external. These are now appropriately inserted into packages
that *need them* and no further, resolving issues with complex
internal recursion in some repositories!
---
 external/default.nix | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/external/default.nix b/external/default.nix
index f9cb542e6..49f8bd931 100644
--- a/external/default.nix
+++ b/external/default.nix
@@ -9,6 +9,7 @@ let
     fromJSON
     head
     length
+    listToAttrs
     readFile
     replaceStrings
     tail
@@ -48,29 +49,45 @@ let
 
   last = l: elemAt l ((length l) - 1);
 
-  toPackage = self: src: path: entry:
+  toPackage = self: src: path: depMap: entry:
     let
+      localDeps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) (
+        throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'"
+      ) self) entry.localDeps;
+
+      foreignDeps = map (d: lib.attrByPath [ d ] (
+        throw "missing foreign dependency '${d}' in '${path}'"
+      ) depMap) entry.foreignDeps;
+
       args = {
         srcs = map (f: src + ("/" + f)) entry.files;
-        deps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) (
-          throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'"
-        ) self) entry.localDeps;
+        deps = localDeps ++ foreignDeps;
       };
+
       libArgs = args // {
         name = pathToName entry.name;
         path = lib.concatStringsSep "/" ([ path ] ++ entry.locator);
       };
+
       binArgs = args // {
         name = last ([ path ] ++ entry.locator);
       };
     in if entry.isCommand then (program binArgs) else (package libArgs);
 
 in { src, path, deps ? [] }: let
+  # Build a map of dependencies (from their import paths to their
+  # derivation) so that they can be conditionally imported only in
+  # sub-packages that require them.
+  depMap = listToAttrs (map (d: {
+    name = d.goImportPath;
+    value = d;
+  }) deps);
+
   name = pathToName path;
   analysisOutput = runCommand "${name}-structure.json" {} ''
     ${analyser}/bin/analyser -path ${path} -source ${src} > $out
   '';
   analysis = fromJSON (readFile analysisOutput);
 in lib.fix(self: foldl' lib.recursiveUpdate {} (
-  map (entry: mkset entry.locator (toPackage self src path entry)) analysis
+  map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis
 ))

From 41c28cf60ec27afde7379d2389ce8b5b7d420c5e Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Fri, 13 Dec 2019 00:08:36 +0000
Subject: [PATCH 28/36] fix(buildGo): Correctly refer to goGrpc package

---
 buildGo.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildGo.nix b/buildGo.nix
index 0df3c955a..759a671fa 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -98,7 +98,7 @@ let
   };
 
   # Build a Go library out of the specified gRPC definition.
-  grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc ]; });
+  grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc.gopkg ]; });
 
 in {
   # Only the high-level builder functions are exposed, but made

From 369c86e0ef8a713d47c2a6b94a3f70d527b84a3c Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Fri, 13 Dec 2019 00:08:56 +0000
Subject: [PATCH 29/36] fix(proto): Refactor gRPC dependencies to match new
 external layout

This is now a lot more fine-grained than before, but it actually works
fine.

This stuff is a bit annoying to write by hand. There are multiple
different options available (e.g. carrying an attribute set of all
subpackages in each `external` and only passing that, having a tool
generate this, etc.).
---
 proto.nix | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/proto.nix b/proto.nix
index dd40def0f..2ece948eb 100644
--- a/proto.nix
+++ b/proto.nix
@@ -19,11 +19,17 @@ in rec {
 
   xnet = external {
     path = "golang.org/x/net";
-    deps = [ xtext ];
+
     src = fetchGit {
       url = "https://go.googlesource.com/net";
       rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d";
     };
+
+    deps = map (p: p.gopkg) [
+      xtext.secure.bidirule
+      xtext.unicode.bidi
+      xtext.unicode.norm
+    ];
   };
 
   xsys = external {
@@ -48,21 +54,31 @@ in rec {
       url = "https://github.com/google/go-genproto";
       rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6";
     };
+
+    deps = with goProto; map (p: p.gopkg) [
+      proto
+      ptypes.any
+    ];
   };
 
   goGrpc = external {
     path = "google.golang.org/grpc";
-    deps = [ goProto xnet xsys genproto ];
+    deps = map (p: p.gopkg) ([
+      xnet.trace
+      xnet.http2
+      xsys.unix
+      xnet.http2.hpack
+      genproto.googleapis.rpc.status
+    ] ++ (with goProto; [
+      proto
+      ptypes
+      ptypes.duration
+      ptypes.timestamp
+    ]));
 
     src = fetchGit {
       url = "https://github.com/grpc/grpc-go";
       rev = "d8e3da36ac481ef00e510ca119f6b68177713689";
     };
-
-    targets = [
-      "."
-      "codes"
-      "status"
-    ];
   };
 }

From 9d5417501b1bdf5f363221a14b2e2a92ef2ba3e2 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Fri, 13 Dec 2019 00:12:33 +0000
Subject: [PATCH 30/36] docs(README): Update description of buildGo.external

---
 README.md | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 489cf1281..e84ede663 100644
--- a/README.md
+++ b/README.md
@@ -90,19 +90,20 @@ in buildGo.program {
   | `deps`    | `list<drv>`  | List of dependencies (i.e. other Go libraries) | no        |
   | `path`    | `string`     | Go import path for the resulting library       | no        |
 
-* `buildGo.external`: Build a Go library or program using standard `go` tooling.
+* `buildGo.external`: Build an externally defined Go library or program.
 
-  This exists for compatibility with complex external dependencies. In theory it
-  is possible to write `buildGo.package` specifications for each subpackage of
-  an external dependency, but it is often cumbersome to do so.
+  This function performs analysis on the supplied source code (which
+  can use the standard Go tooling layout) and creates a tree of all
+  the packages contained within.
 
-  | parameter | type           | use                                            | required? |
-  |-----------|----------------|------------------------------------------------|-----------|
-  | `path`    | `string`       | Go import path for the resulting library       | yes       |
-  | `src`     | `path`         | Path to the source **directory**               | yes       |
-  | `deps`    | `list<drv>`    | List of dependencies (i.e. other Go libraries) | no        |
-  | `srcOnly` | `bool`         | Only copy sources, do not perform a build.     | no        |
-  | `targets` | `list<string>` | Sub-packages to build (defaults to all)        | no        |
+  This exists for compatibility with external libraries that were not
+  defined using buildGo.
+
+  | parameter | type           | use                                           | required? |
+  |-----------|----------------|-----------------------------------------------|-----------|
+  | `path`    | `string`       | Go import path for the resulting package      | yes       |
+  | `src`     | `path`         | Path to the source **directory**              | yes       |
+  | `deps`    | `list<drv>`    | List of dependencies (i.e. other Go packages) | no        |
 
   For some examples of how `buildGo.external` is used, check out
   [`proto.nix`](./proto.nix).

From 5649c75d01dc9058d9a6092ad2a504669cc780a9 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Fri, 13 Dec 2019 12:10:32 +0000
Subject: [PATCH 31/36] fix(external): Fix "inverted" local dependencies

Usually in large packages the root depends on one or more
sub-packages (or there is no root), but some projects (e.g.
golang.org/x/oauth2) do it the other way around.

This fix adds compatibility for both ways.
---
 external/main.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/external/main.go b/external/main.go
index e0b95605f..d73f1540a 100644
--- a/external/main.go
+++ b/external/main.go
@@ -89,7 +89,9 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk
 			continue
 		}
 
-		if strings.HasPrefix(i, importpath) {
+		if i == importpath {
+			local = append(local, []string{})
+		} else if strings.HasPrefix(i, importpath) {
 			local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/"))
 		} else {
 			foreign = append(foreign, i)

From cfae527cc14c55f2cfa3d356674971c627ad1a88 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Fri, 13 Dec 2019 15:15:22 +0000
Subject: [PATCH 32/36] feat(buildGo): Support linking of symabi files for ASM
 sources

---
 buildGo.nix | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/buildGo.nix b/buildGo.nix
index 759a671fa..b61c3904f 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -64,12 +64,17 @@ let
   #
   # This outputs both the sources and compiled binary, as both are
   # needed when downstream packages depend on it.
-  package = { name, srcs, deps ? [], path ? name }:
+  package = { name, srcs, deps ? [], path ? name, sfiles ? [] }:
   let uniqueDeps = allDeps deps;
+      asmBuild = if sfiles == [] then "" else ''
+        ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles}
+      '';
+      asmLink = if sfiles == [] then "-complete" else "-symabis ./symabis";
   in (runCommand "golib-${name}" {} ''
     mkdir -p $out/${path}
     ${srcList path (map (s: "${s}") srcs)}
-    ${go}/bin/go tool compile -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs}
+    ${asmBuild}
+    ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs}
   '') // { goDeps = uniqueDeps; goImportPath = path; };
 
   # Build a tree of Go libraries out of an external Go source

From 859c429b2fbf21e3ea799b1068daa6f123c59a48 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Fri, 13 Dec 2019 15:15:53 +0000
Subject: [PATCH 33/36] feat(external): Include *.s (ASM) files in external
 builds

---
 external/default.nix | 1 +
 external/main.go     | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/external/default.nix b/external/default.nix
index 49f8bd931..ee0bf94fa 100644
--- a/external/default.nix
+++ b/external/default.nix
@@ -67,6 +67,7 @@ let
       libArgs = args // {
         name = pathToName entry.name;
         path = lib.concatStringsSep "/" ([ path ] ++ entry.locator);
+        sfiles = map (f: src + ("/" + f)) entry.sfiles;
       };
 
       binArgs = args // {
diff --git a/external/main.go b/external/main.go
index d73f1540a..aa4a813d3 100644
--- a/external/main.go
+++ b/external/main.go
@@ -32,6 +32,7 @@ type pkg struct {
 	Name        string     `json:"name"`
 	Locator     []string   `json:"locator"`
 	Files       []string   `json:"files"`
+	SFiles      []string   `json:"sfiles"`
 	LocalDeps   [][]string `json:"localDeps"`
 	ForeignDeps []string   `json:"foreignDeps"`
 	IsCommand   bool       `json:"isCommand"`
@@ -75,6 +76,7 @@ func findGoDirs(at string) ([]string, error) {
 // generate a derivation for this package.
 func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) {
 	ctx := build.Default
+	ctx.CgoEnabled = false
 
 	p, err := ctx.ImportDir(source, build.IgnoreVendor)
 	if err != nil {
@@ -114,10 +116,16 @@ func analysePackage(root, source, importpath string, stdlib map[string]bool) (pk
 		files = append(files, path.Join(prefix, f))
 	}
 
+	sfiles := []string{}
+	for _, f := range p.SFiles {
+		sfiles = append(sfiles, path.Join(prefix, f))
+	}
+
 	return pkg{
 		Name:        path.Join(importpath, prefix),
 		Locator:     locator,
 		Files:       files,
+		SFiles:      sfiles,
 		LocalDeps:   local,
 		ForeignDeps: foreign,
 		IsCommand:   p.IsCommand(),

From 7f74980457df843aea542510a406f34366e8b868 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Fri, 13 Dec 2019 21:23:41 +0000
Subject: [PATCH 34/36] feat(external): Compile Go assembly and include it in
 pkg archive

This was the final step required to add support for packages that make
use of Go assembly, such as golang.org/x/sys.
---
 buildGo.nix | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/buildGo.nix b/buildGo.nix
index b61c3904f..140cbf2d9 100644
--- a/buildGo.nix
+++ b/buildGo.nix
@@ -65,16 +65,28 @@ let
   # This outputs both the sources and compiled binary, as both are
   # needed when downstream packages depend on it.
   package = { name, srcs, deps ? [], path ? name, sfiles ? [] }:
-  let uniqueDeps = allDeps deps;
-      asmBuild = if sfiles == [] then "" else ''
-        ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles}
-      '';
-      asmLink = if sfiles == [] then "-complete" else "-symabis ./symabis";
+  let
+    uniqueDeps = allDeps deps;
+
+    # The build steps below need to be executed conditionally for Go
+    # assembly if the analyser detected any *.s files.
+    #
+    # This is required for several popular packages (e.g. x/sys).
+    ifAsm = do: if sfiles == [] then "" else do;
+    asmBuild = ifAsm ''
+      ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles}
+      ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles}
+    '';
+    asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h";
+    asmPack = ifAsm ''
+      ${go}/bin/go tool pack r $out/${path}.a ./asm.o
+    '';
   in (runCommand "golib-${name}" {} ''
     mkdir -p $out/${path}
     ${srcList path (map (s: "${s}") srcs)}
     ${asmBuild}
     ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs}
+    ${asmPack}
   '') // { goDeps = uniqueDeps; goImportPath = path; };
 
   # Build a tree of Go libraries out of an external Go source

From 53630e3b1544de38f509e48ac0f589bcfafa7728 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Sun, 15 Dec 2019 15:57:10 +0000
Subject: [PATCH 35/36] fix(external): Correctly determine binary name if
 locator is empty

If the root of a project is a binary, the previous logic would
generate invalid names.

This ensure that the last path component of the name is used.
---
 external/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/external/default.nix b/external/default.nix
index ee0bf94fa..6e1dce111 100644
--- a/external/default.nix
+++ b/external/default.nix
@@ -71,7 +71,7 @@ let
       };
 
       binArgs = args // {
-        name = last ([ path ] ++ entry.locator);
+        name = (last ((lib.splitString "/" path) ++ entry.locator));
       };
     in if entry.isCommand then (program binArgs) else (package libArgs);
 

From a8fbdc52c2d7170bc8b144bef9d522bef9f0d815 Mon Sep 17 00:00:00 2001
From: Vincent Ambo <tazjin@google.com>
Date: Thu, 19 Dec 2019 15:22:35 +0000
Subject: [PATCH 36/36] chore(buildGo): Relayout for depot merge

---
 README.md => overrides/buildGo/README.md             | 0
 buildGo.nix => overrides/buildGo/default.nix         | 0
 {example => overrides/buildGo/example}/default.nix   | 0
 {example => overrides/buildGo/example}/lib.go        | 0
 {example => overrides/buildGo/example}/main.go       | 0
 {example => overrides/buildGo/example}/thing.proto   | 0
 {external => overrides/buildGo/external}/default.nix | 0
 {external => overrides/buildGo/external}/main.go     | 0
 proto.nix => overrides/buildGo/proto.nix             | 0
 9 files changed, 0 insertions(+), 0 deletions(-)
 rename README.md => overrides/buildGo/README.md (100%)
 rename buildGo.nix => overrides/buildGo/default.nix (100%)
 rename {example => overrides/buildGo/example}/default.nix (100%)
 rename {example => overrides/buildGo/example}/lib.go (100%)
 rename {example => overrides/buildGo/example}/main.go (100%)
 rename {example => overrides/buildGo/example}/thing.proto (100%)
 rename {external => overrides/buildGo/external}/default.nix (100%)
 rename {external => overrides/buildGo/external}/main.go (100%)
 rename proto.nix => overrides/buildGo/proto.nix (100%)

diff --git a/README.md b/overrides/buildGo/README.md
similarity index 100%
rename from README.md
rename to overrides/buildGo/README.md
diff --git a/buildGo.nix b/overrides/buildGo/default.nix
similarity index 100%
rename from buildGo.nix
rename to overrides/buildGo/default.nix
diff --git a/example/default.nix b/overrides/buildGo/example/default.nix
similarity index 100%
rename from example/default.nix
rename to overrides/buildGo/example/default.nix
diff --git a/example/lib.go b/overrides/buildGo/example/lib.go
similarity index 100%
rename from example/lib.go
rename to overrides/buildGo/example/lib.go
diff --git a/example/main.go b/overrides/buildGo/example/main.go
similarity index 100%
rename from example/main.go
rename to overrides/buildGo/example/main.go
diff --git a/example/thing.proto b/overrides/buildGo/example/thing.proto
similarity index 100%
rename from example/thing.proto
rename to overrides/buildGo/example/thing.proto
diff --git a/external/default.nix b/overrides/buildGo/external/default.nix
similarity index 100%
rename from external/default.nix
rename to overrides/buildGo/external/default.nix
diff --git a/external/main.go b/overrides/buildGo/external/main.go
similarity index 100%
rename from external/main.go
rename to overrides/buildGo/external/main.go
diff --git a/proto.nix b/overrides/buildGo/proto.nix
similarity index 100%
rename from proto.nix
rename to overrides/buildGo/proto.nix