mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merge(buildGo): Integrate buildGo.nix into depot

Browse source
This commit is contained in:
Vincent Ambo 2019-12-19 15:24:17 +00:00
commit e996141d2e
9 changed files with 723 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

140
overrides/buildGo/README.md Normal file
View file

@ -0,0 +1,140 @@
buildGo.nix
===========
This is an alternative [Nix][] build system for [Go][]. It supports building Go
libraries and programs, and even automatically generating Protobuf & gRPC
libraries.
*Note:* This will probably end up being folded into [Nixery][].
## Background
Most language-specific Nix tooling outsources the build to existing
language-specific build tooling, which essentially means that Nix ends up being
a wrapper around all sorts of external build systems.
However, systems like [Bazel][] take an alternative approach in which the
compiler is invoked directly and the composition of programs and libraries stays
within a single homogeneous build system.
Users don't need to learn per-language build systems and especially for
companies with large monorepo-setups ([like Google][]) this has huge
productivity impact.
This project is an attempt to prove that Nix can be used in a similar style to
build software directly, rather than shelling out to other build systems.
## Example
Given a program layout like this:
```
.
├── lib <-- some library component
│   ├── bar.go
│   └── foo.go
├── api.proto <-- gRPC API definition
├── main.go <-- program implementation
└── default.nix <-- build instructions
```
The contents of `default.nix` could look like this:
```nix
{ buildGo }:
let
api = buildGo.grpc {
name = "someapi";
proto = ./api.proto;
};
lib = buildGo.package {
name = "somelib";
srcs = [
./lib/bar.go
./lib/foo.go
];
};
in buildGo.program {
name = "my-program";
deps = [ api lib ];
srcs = [
./main.go
];
}
```
(If you don't know how to read Nix, check out [nix-1p][])
## Usage
`buildGo` exposes five different functions:
* `buildGo.program`: Build a Go binary out of the specified source files.
| parameter | type | use | required? |
|-----------|-------------------------|------------------------------------------------|-----------|
| `name` | `string` | Name of the program (and resulting executable) | yes |
| `srcs` | `list<path>` | List of paths to source files | yes |
| `deps` | `list<drv>` | List of dependencies (i.e. other Go libraries) | no |
| `x_defs` | `attrs<string, string>` | Attribute set of linker vars (i.e. `-X`-flags) | no |
* `buildGo.package`: Build a Go library out of the specified source files.
| parameter | type | use | required? |
|-----------|--------------|------------------------------------------------|-----------|
| `name` | `string` | Name of the library (and resulting executable) | yes |
| `srcs` | `list<path>` | List of paths to source files | yes |
| `deps` | `list<drv>` | List of dependencies (i.e. other Go libraries) | no |
| `path` | `string` | Go import path for the resulting library | no |
* `buildGo.external`: Build an externally defined Go library or program.
This function performs analysis on the supplied source code (which
can use the standard Go tooling layout) and creates a tree of all
the packages contained within.
This exists for compatibility with external libraries that were not
defined using buildGo.
| parameter | type | use | required? |
|-----------|----------------|-----------------------------------------------|-----------|
| `path` | `string` | Go import path for the resulting package | yes |
| `src` | `path` | Path to the source **directory** | yes |
| `deps` | `list<drv>` | List of dependencies (i.e. other Go packages) | no |
For some examples of how `buildGo.external` is used, check out
[`proto.nix`](./proto.nix).
* `buildGo.proto`: Build a Go library out of the specified Protobuf definition.
| parameter | type | use | required? |
|-------------|-------------|--------------------------------------------------|-----------|
| `name` | `string` | Name for the resulting library | yes |
| `proto` | `path` | Path to the Protobuf definition file | yes |
| `path` | `string` | Import path for the resulting Go library | no |
| `extraDeps` | `list<drv>` | Additional Go dependencies to add to the library | no |
* `buildGo.grpc`: Build a Go library out of the specified gRPC definition.
The parameters are identical to `buildGo.proto`.
## Current status
This project is work-in-progress. Crucially it is lacking the following features:
* feature flag parity with Bazel's Go rules
* documentation building
* test execution
There are still some open questions around how to structure some of those
features in Nix.
[Nix]: https://nixos.org/nix/
[Go]: https://golang.org/
[Nixery]: https://github.com/google/nixery
[Bazel]: https://bazel.build/
[like Google]: https://ai.google/research/pubs/pub45424
[nix-1p]: https://github.com/tazjin/nix-1p

128
overrides/buildGo/default.nix Normal file
View file

@ -0,0 +1,128 @@
# Copyright 2019 Google LLC.
# SPDX-License-Identifier: Apache-2.0
#
# buildGo provides Nix functions to build Go packages in the style of Bazel's
# rules_go.
{ pkgs ? import <nixpkgs> {}
, ... }:
let
inherit (builtins)
attrNames
baseNameOf
dirOf
elemAt
filter
listToAttrs
map
match
readDir
replaceStrings
toString;
inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin;
# Helpers for low-level Go compiler invocations
spaceOut = lib.concatStringsSep " ";
includeDepSrc = dep: "-I ${dep}";
includeSources = deps: spaceOut (map includeDepSrc deps);
includeDepLib = dep: "-L ${dep}";
includeLibs = deps: spaceOut (map includeDepLib deps);
srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1;
srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}";
srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs);
allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps)));
xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs));
pathToName = p: replaceStrings ["/"] ["_"] (toString p);
# Add an `overrideGo` attribute to a function result that works
# similar to `overrideAttrs`, but is used specifically for the
# arguments passed to Go builders.
makeOverridable = f: orig: (f orig) // {
overrideGo = new: makeOverridable f (orig // (new orig));
};
# High-level build functions
# Build a Go program out of the specified files and dependencies.
program = { name, srcs, deps ? [], x_defs ? {} }:
let uniqueDeps = allDeps deps;
in runCommand name {} ''
${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs}
mkdir -p $out/bin
${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a
'';
# Build a Go library assembled out of the specified files.
#
# This outputs both the sources and compiled binary, as both are
# needed when downstream packages depend on it.
package = { name, srcs, deps ? [], path ? name, sfiles ? [] }:
let
uniqueDeps = allDeps deps;
# The build steps below need to be executed conditionally for Go
# assembly if the analyser detected any *.s files.
#
# This is required for several popular packages (e.g. x/sys).
ifAsm = do: if sfiles == [] then "" else do;
asmBuild = ifAsm ''
${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles}
${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles}
'';
asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h";
asmPack = ifAsm ''
${go}/bin/go tool pack r $out/${path}.a ./asm.o
'';
in (runCommand "golib-${name}" {} ''
mkdir -p $out/${path}
${srcList path (map (s: "${s}") srcs)}
${asmBuild}
${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs}
${asmPack}
'') // { goDeps = uniqueDeps; goImportPath = path; };
# Build a tree of Go libraries out of an external Go source
# directory that follows the standard Go layout and was not built
# with buildGo.nix.
#
# The derivation for each actual package will reside in an attribute
# named "gopkg", and an attribute named "gobin" for binaries.
external = import ./external { inherit pkgs program package; };
# Import support libraries needed for protobuf & gRPC support
protoLibs = import ./proto.nix {
inherit external;
};
# Build a Go library out of the specified protobuf definition.
proto = { name, proto, path ? name, extraDeps ? [] }: (makeOverridable package) {
inherit name path;
deps = [ protoLibs.goProto.proto.gopkg ] ++ extraDeps;
srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} ''
cp ${proto} ${baseNameOf proto}
${protobuf}/bin/protoc --plugin=${protoLibs.goProto.protoc-gen-go.gopkg}/bin/protoc-gen-go \
--go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto}
mv *.pb.go $out
'');
};
# Build a Go library out of the specified gRPC definition.
grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc.gopkg ]; });
in {
# Only the high-level builder functions are exposed, but made
# overrideable.
program = makeOverridable program;
package = makeOverridable package;
proto = makeOverridable proto;
grpc = makeOverridable grpc;
external = makeOverridable external;
}

47
overrides/buildGo/example/default.nix Normal file
View file

@ -0,0 +1,47 @@
# Copyright 2019 Google LLC.
# SPDX-License-Identifier: Apache-2.0
# This file provides examples for how to use the various builder
# functions provided by `buildGo`.
#
# The features used in the example are not exhaustive, but should give
# users a quick introduction to how to use buildGo.
let
buildGo = import ../buildGo.nix {};
# Example use of buildGo.package, which creates an importable Go
# package from the specified source files.
examplePackage = buildGo.package {
name = "example";
srcs = [
./lib.go
];
};
# Example use of buildGo.proto, which generates a Go library from a
# Protobuf definition file.
exampleProto = buildGo.proto {
name = "exampleproto";
proto = ./thing.proto;
};
# Example use of buildGo.program, which builds an executable using
# the specified name and dependencies (which in turn must have been
# created via buildGo.package etc.)
in buildGo.program {
name = "example";
srcs = [
./main.go
];
deps = [
examplePackage
exampleProto
];
x_defs = {
"main.Flag" = "successfully";
};
}

9
overrides/buildGo/example/lib.go Normal file
View file

@ -0,0 +1,9 @@
// Copyright 2019 Google LLC.
// SPDX-License-Identifier: Apache-2.0
package example
// UUID returns a totally random, carefully chosen UUID
func UUID() string {
return "3640932f-ad40-4bc9-b45d-f504a0f5910a"
}

25
overrides/buildGo/example/main.go Normal file
View file

@ -0,0 +1,25 @@
// Copyright 2019 Google LLC.
// SPDX-License-Identifier: Apache-2.0
//
// Package main provides a tiny example program for the Bazel-style
// Nix build system for Go.
package main
import (
"example"
"exampleproto"
"fmt"
)
var Flag string = "unsuccessfully"
func main() {
thing := exampleproto.Thing{
Id: example.UUID(),
KindOfThing: "test thing",
}
fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing)
fmt.Printf("The flag has been %s set\n", Flag)
}

10
overrides/buildGo/example/thing.proto Normal file
View file

@ -0,0 +1,10 @@
// Copyright 2019 Google LLC.
// SPDX-License-Identifier: Apache-2.0
syntax = "proto3";
package exampleProto;
message Thing {
string id = 1;
string kind_of_thing = 2;
}

94
overrides/buildGo/external/default.nix vendored Normal file
View file

@ -0,0 +1,94 @@
# Copyright 2019 Google LLC.
# SPDX-License-Identifier: Apache-2.0
{ pkgs, program, package }:
let
inherit (builtins)
elemAt
foldl'
fromJSON
head
length
listToAttrs
readFile
replaceStrings
tail
throw;
inherit (pkgs) lib runCommand go jq ripgrep;
pathToName = p: replaceStrings ["/"] ["_"] (toString p);
# Collect all non-vendored dependencies from the Go standard library
# into a file that can be used to filter them out when processing
# dependencies.
stdlibPackages = runCommand "stdlib-pkgs.json" {} ''
export GOPATH=/dev/null
${go}/bin/go list all | \
${ripgrep}/bin/rg -v 'vendor' | \
${jq}/bin/jq -R '.' | \
${jq}/bin/jq -c -s 'map({key: ., value: true}) | from_entries' \
> $out
'';
analyser = program {
name = "analyser";
srcs = [
./main.go
];
x_defs = {
"main.stdlibList" = "${stdlibPackages}";
};
};
mkset = path: value:
if path == [] then { gopkg = value; }
else { "${head path}" = mkset (tail path) value; };
last = l: elemAt l ((length l) - 1);
toPackage = self: src: path: depMap: entry:
let
localDeps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) (
throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'"
) self) entry.localDeps;
foreignDeps = map (d: lib.attrByPath [ d ] (
throw "missing foreign dependency '${d}' in '${path}'"
) depMap) entry.foreignDeps;
args = {
srcs = map (f: src + ("/" + f)) entry.files;
deps = localDeps ++ foreignDeps;
};
libArgs = args // {
name = pathToName entry.name;
path = lib.concatStringsSep "/" ([ path ] ++ entry.locator);
sfiles = map (f: src + ("/" + f)) entry.sfiles;
};
binArgs = args // {
name = (last ((lib.splitString "/" path) ++ entry.locator));
};
in if entry.isCommand then (program binArgs) else (package libArgs);
in { src, path, deps ? [] }: let
# Build a map of dependencies (from their import paths to their
# derivation) so that they can be conditionally imported only in
# sub-packages that require them.
depMap = listToAttrs (map (d: {
name = d.goImportPath;
value = d;
}) deps);
name = pathToName path;
analysisOutput = runCommand "${name}-structure.json" {} ''
${analyser}/bin/analyser -path ${path} -source ${src} > $out
'';
analysis = fromJSON (readFile analysisOutput);
in lib.fix(self: foldl' lib.recursiveUpdate {} (
map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis
))

186
overrides/buildGo/external/main.go vendored Normal file
View file

@ -0,0 +1,186 @@
// Copyright 2019 Google LLC.
// SPDX-License-Identifier: Apache-2.0
// This tool analyses external (i.e. not built with `buildGo.nix`) Go
// packages to determine a build plan that Nix can import.
package main
import (
"encoding/json"
"flag"
"fmt"
"go/build"
"io/ioutil"
"log"
"os"
"path"
"path/filepath"
"strings"
)
// Path to a JSON file describing all standard library import paths.
// This file is generated and set here by Nix during the build
// process.
var stdlibList string
// pkg describes a single Go package within the specified source
// directory.
//
// Return information includes the local (relative from project root)
// and external (none-stdlib) dependencies of this package.
type pkg struct {
Name string `json:"name"`
Locator []string `json:"locator"`
Files []string `json:"files"`
SFiles []string `json:"sfiles"`
LocalDeps [][]string `json:"localDeps"`
ForeignDeps []string `json:"foreignDeps"`
IsCommand bool `json:"isCommand"`
}
// findGoDirs returns a filepath.WalkFunc that identifies all
// directories that contain Go source code in a certain tree.
func findGoDirs(at string) ([]string, error) {
dirSet := make(map[string]bool)
err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error {
name := info.Name()
// Skip folders that are guaranteed to not be relevant
if info.IsDir() && (name == "testdata" || name == ".git") {
return filepath.SkipDir
}
// If the current file is a Go file, then the directory is popped
// (i.e. marked as a Go directory).
if !info.IsDir() && strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") {
dirSet[filepath.Dir(path)] = true
}
return nil
})
if err != nil {
return nil, err
}
goDirs := []string{}
for k, _ := range dirSet {
goDirs = append(goDirs, k)
}
return goDirs, nil
}
// analysePackage loads and analyses the imports of a single Go
// package, returning the data that is required by the Nix code to
// generate a derivation for this package.
func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) {
ctx := build.Default
ctx.CgoEnabled = false
p, err := ctx.ImportDir(source, build.IgnoreVendor)
if err != nil {
return pkg{}, err
}
local := [][]string{}
foreign := []string{}
for _, i := range p.Imports {
if stdlib[i] {
continue
}
if i == importpath {
local = append(local, []string{})
} else if strings.HasPrefix(i, importpath) {
local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/"))
} else {
foreign = append(foreign, i)
}
}
prefix := strings.TrimPrefix(source, root+"/")
locator := []string{}
if len(prefix) != len(source) {
locator = strings.Split(prefix, "/")
} else {
// Otherwise, the locator is empty since its the root package and
// no prefix should be added to files.
prefix = ""
}
files := []string{}
for _, f := range p.GoFiles {
files = append(files, path.Join(prefix, f))
}
sfiles := []string{}
for _, f := range p.SFiles {
sfiles = append(sfiles, path.Join(prefix, f))
}
return pkg{
Name: path.Join(importpath, prefix),
Locator: locator,
Files: files,
SFiles: sfiles,
LocalDeps: local,
ForeignDeps: foreign,
IsCommand: p.IsCommand(),
}, nil
}
func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) {
f, err := ioutil.ReadFile(from)
if err != nil {
return
}
err = json.Unmarshal(f, &pkgs)
return
}
func main() {
source := flag.String("source", "", "path to directory with sources to process")
path := flag.String("path", "", "import path for the package")
flag.Parse()
if *source == "" {
log.Fatalf("-source flag must be specified")
}
stdlibPkgs, err := loadStdlibPkgs(stdlibList)
if err != nil {
log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err)
}
goDirs, err := findGoDirs(*source)
if err != nil {
log.Fatalf("failed to walk source directory '%s': %s\n", source, err)
}
all := []pkg{}
for _, d := range goDirs {
analysed, err := analysePackage(*source, d, *path, stdlibPkgs)
// If the Go source analysis returned "no buildable Go files",
// that directory should be skipped.
//
// This might be due to `+build` flags on the platform and other
// reasons (such as test files).
if _, ok := err.(*build.NoGoError); ok {
continue
}
if err != nil {
log.Fatalf("failed to analyse package at %q: %s", d, err)
}
all = append(all, analysed)
}
j, _ := json.Marshal(all)
fmt.Println(string(j))
}

84
overrides/buildGo/proto.nix Normal file
View file

@ -0,0 +1,84 @@
# Copyright 2019 Google LLC.
# SPDX-License-Identifier: Apache-2.0
#
# This file provides derivations for the dependencies of a gRPC
# service in Go.
{ external }:
let
inherit (builtins) fetchGit map;
in rec {
goProto = external {
path = "github.com/golang/protobuf";
src = fetchGit {
url = "https://github.com/golang/protobuf";
rev = "ed6926b37a637426117ccab59282c3839528a700";
};
};
xnet = external {
path = "golang.org/x/net";
src = fetchGit {
url = "https://go.googlesource.com/net";
rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d";
};
deps = map (p: p.gopkg) [
xtext.secure.bidirule
xtext.unicode.bidi
xtext.unicode.norm
];
};
xsys = external {
path = "golang.org/x/sys";
src = fetchGit {
url = "https://go.googlesource.com/sys";
rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd";
};
};
xtext = external {
path = "golang.org/x/text";
src = fetchGit {
url = "https://go.googlesource.com/text";
rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686";
};
};
genproto = external {
path = "google.golang.org/genproto";
src = fetchGit {
url = "https://github.com/google/go-genproto";
rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6";
};
deps = with goProto; map (p: p.gopkg) [
proto
ptypes.any
];
};
goGrpc = external {
path = "google.golang.org/grpc";
deps = map (p: p.gopkg) ([
xnet.trace
xnet.http2
xsys.unix
xnet.http2.hpack
genproto.googleapis.rpc.status
] ++ (with goProto; [
proto
ptypes
ptypes.duration
ptypes.timestamp
]));
src = fetchGit {
url = "https://github.com/grpc/grpc-go";
rev = "d8e3da36ac481ef00e510ca119f6b68177713689";
};
};
}