feat(tvix/build): init

This adds the tvix-build crate, currently only containing a
`tvix_build::proto` module, exposing the data structures defined in
tvix/build/protos.

Change-Id: I75f5d9196969ed0877b1fe640cacfecba0fb2e03
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10235
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Tested-by: BuildkiteCI

tvix/Cargo.lock

@@ -3050,6 +3050,18 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
 
+[[package]]
+name = "tvix-build"
+version = "0.1.0"
+dependencies = [
+ "prost",
+ "prost-build",
+ "tonic",
+ "tonic-build",
+ "tonic-reflection",
+ "tvix-castore",
+]
+
 [[package]]
 name = "tvix-castore"
 version = "0.1.0"

tvix/Cargo.nix

@@ -43,6 +43,16 @@ rec {
       # File a bug if you depend on any for non-debug work!
       debug = internal.debugCrate { inherit packageId; };
     };
+    "tvix-build" = rec {
+      packageId = "tvix-build";
+      build = internal.buildRustCrateWithFeatures {
+        packageId = "tvix-build";
+      };
+
+      # Debug support which might change between releases.
+      # File a bug if you depend on any for non-debug work!
+      debug = internal.debugCrate { inherit packageId; };
+    };
     "tvix-castore" = rec {
       packageId = "tvix-castore";
       build = internal.buildRustCrateWithFeatures {
@@ -9348,6 +9358,50 @@ rec {
         ];
 
       };
+      "tvix-build" = rec {
+        crateName = "tvix-build";
+        version = "0.1.0";
+        edition = "2021";
+        # We can't filter paths with references in Nix 2.4
+        # See https://github.com/NixOS/nix/issues/5410
+        src =
+          if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion))
+          then lib.cleanSourceWith { filter = sourceFilter; src = ./build; }
+          else ./build;
+        dependencies = [
+          {
+            name = "prost";
+            packageId = "prost";
+          }
+          {
+            name = "tonic";
+            packageId = "tonic";
+          }
+          {
+            name = "tonic-reflection";
+            packageId = "tonic-reflection";
+            optional = true;
+          }
+          {
+            name = "tvix-castore";
+            packageId = "tvix-castore";
+          }
+        ];
+        buildDependencies = [
+          {
+            name = "prost-build";
+            packageId = "prost-build";
+          }
+          {
+            name = "tonic-build";
+            packageId = "tonic-build";
+          }
+        ];
+        features = {
+          "tonic-reflection" = [ "dep:tonic-reflection" ];
+        };
+        resolvedDefaultFeatures = [ "default" "tonic-reflection" ];
+      };
       "tvix-castore" = rec {
         crateName = "tvix-castore";
         version = "0.1.0";

tvix/Cargo.toml

@@ -19,6 +19,7 @@
 resolver = "2"
 
 members = [
+  "build",
   "castore",
   "cli",
   "eval",

tvix/build/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "tvix-build"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+prost = "0.12.1"
+tonic = "0.10.2"
+tvix-castore = { path = "../castore" }
+
+[dependencies.tonic-reflection]
+optional = true
+version = "0.10.2"
+
+[build-dependencies]
+prost-build = "0.12.1"
+tonic-build = "0.10.2"
+
+[features]
+default = []
+tonic-reflection = ["dep:tonic-reflection"]

tvix/build/build.rs

@@ -0,0 +1,38 @@
+use std::io::Result;
+
+fn main() -> Result<()> {
+    #[allow(unused_mut)]
+    let mut builder = tonic_build::configure();
+
+    #[cfg(feature = "tonic-reflection")]
+    {
+        let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap());
+        let descriptor_path = out_dir.join("tvix.build.v1.bin");
+
+        builder = builder.file_descriptor_set_path(descriptor_path);
+    };
+
+    // https://github.com/hyperium/tonic/issues/908
+    let mut config = prost_build::Config::new();
+    config.bytes(["."]);
+    config.extern_path(".tvix.castore.v1", "::tvix_castore::proto");
+
+    builder
+        .build_server(true)
+        .build_client(true)
+        .compile_with_config(
+            config,
+            &[
+                "tvix/build/protos/build.proto",
+                "tvix/build/protos/rpc_build.proto",
+            ],
+            // If we are in running `cargo build` manually, using `../..` works fine,
+            // but in case we run inside a nix build, we need to instead point PROTO_ROOT
+            // to a sparseTree containing that structure.
+            &[match std::env::var_os("PROTO_ROOT") {
+                Some(proto_root) => proto_root.to_str().unwrap().to_owned(),
+                None => "../..".to_string(),
+            }],
+        )?;
+    Ok(())
+}

tvix/build/default.nix

@@ -0,0 +1,5 @@
+{ depot, pkgs, ... }:
+
+depot.tvix.crates.workspaceMembers.tvix-build.build.override {
+  runTests = true;
+}

tvix/build/protos/LICENSE

@@ -0,0 +1,21 @@
+Copyright © The Tvix Authors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+“Software”), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

tvix/build/protos/build.proto

@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: MIT
+// Copyright © 2022 The Tvix Authors
+
+syntax = "proto3";
+
+package tvix.build.v1;
+
+import "tvix/castore/protos/castore.proto";
+
+option go_package = "code.tvl.fyi/tvix/build-go;buildv1";
+
+// A BuildRequest describes the request of something to be run on the builder.
+// It is distinct from an actual [Build] that has already happened, or might be
+// currently ongoing.
+//
+// A BuildRequest can be seen as a more normalized version of a Derivation
+// (parsed from A-Term), "writing out" some of the Nix-internal details about
+// how e.g. environment variables in the build are set.
+//
+// Nix has some impurities when building a Derivation, for example the --cores option
+// ends up as an environment variable in the build, that's not part of the ATerm.
+//
+// As of now, we serialize this into the BuildRequest, so builders can stay dumb.
+// This might change in the future.
+//
+// There's also a big difference when it comes to how inputs are modelled:
+//  - Nix only uses store path (strings) to describe the inputs.
+//    As store paths can be input-addressed, a certain store path can contain
+//    different contents (as not all store paths are binary reproducible).
+//    This requires that for every input-addressed input, the builder has access
+//    to either the input's deriver (and needs to build it) or else a trusted
+//    source for the built input.
+//    to upload input-addressed paths, requiring the trusted users concept.
+//  - tvix-build records a list of tvix.castore.v1.Node as inputs.
+//    These map from the store path base name to their contents, relieving the
+//    builder from having to "trust" any input-addressed paths, contrary to Nix.
+//
+// While this approach gives a better hermeticity, it has one downside:
+// A BuildRequest can only be sent once the contents of all its inputs are known.
+//
+// As of now, we're okay to accept this, but it prevents uploading an
+// entirely-non-IFD subgraph of BuildRequests eagerly.
+//
+// FUTUREWORK: We might be introducing another way to refer to inputs, to
+// support "send all BuildRequest for a nixpkgs eval to a remote builder and put
+// the laptop to sleep" usecases later.
+message BuildRequest {
+  // The command (and its args) executed as the build script.
+  // In the case of a Nix derivation, this is usually
+  // ["/path/to/some-bash/bin/bash", "-e", "/path/to/some/builder.sh"].
+  repeated string command_args = 1;
+
+  // The list of outputs the build is expected to produce.
+  // These are basenames inside /nix/store.
+  // If the path is not produced, the build is considered to have failed.
+  // Outputs are sorted.
+  repeated string outputs = 2;
+
+  // The list of environment variables and their values that should be set
+  // inside the build environment.
+  // This includes both environment vars set inside the derivation, as well as
+  // more "ephemeral" ones like NIX_BUILD_CORES, controlled by the `--cores`
+  // CLI option of `nix-build`.
+  // For now, we consume this as an option when turning a Derivation into a BuildRequest,
+  // similar to how Nix has a `--cores` option.
+  // We don't want to bleed these very nix-specific sandbox impl details into
+  // (dumber) builders if we don't have to.
+  // Environment variables are sorted by their keys.
+  repeated EnvVar environment_vars = 3;
+
+  message EnvVar {
+    string key = 1;
+    bytes value = 2;
+  }
+
+  // The list of all root nodes that should be visible in /nix/store at the
+  // time of the build.
+  // As root nodes are content-addressed, no additional signatures are needed
+  // to substitute / make these available in the build environment.
+  // Inputs are sorted by their names.
+  repeated tvix.castore.v1.Node inputs = 4;
+
+  // A set of constraints that need to be satisfied on a build host before a
+  // Build can be started.
+  BuildConstraints constraints = 5;
+
+  // BuildConstraints represents certain conditions that must be fulfilled
+  // inside the build environment to be able to build this.
+  // Constraints can be things like required architecture and minimum amount of memory.
+  // The required input paths are *not* represented in here, because it
+  // wouldn't be hermetic enough - see the comment around inputs too.
+  message BuildConstraints {
+    // The system that's needed to execute the build.
+    string system = 1;
+
+    // The amount of memory required to be available for the build, in bytes.
+    uint64 min_memory = 2;
+
+    // A list of (absolute) paths that need to be available in the build
+    // environment.
+    // TBD, This is probably things like /dev/kvm, but no nix store paths.
+    repeated string available_ro_paths = 3;
+  }
+
+  // TODO: allow describing something like "preferLocal", to influence composition?
+}
+
+// A Build is (one possible) outcome of executing a [BuildRequest].
+message Build {
+  // The orginal build request producing the build.
+  BuildRequest build_request = 1; // <- TODO: define hashing scheme for BuildRequest, refer to it by hash?
+
+  // The outputs that were produced after successfully building.
+  // They are sorted by their names.
+  repeated tvix.castore.v1.Node outputs = 2;
+
+  // TODO: where did this run, how long, logs, …
+}
+
+/// TODO: check remarkable notes on constraints again
+/// TODO: https://github.com/adisbladis/go-nix/commit/603df5db86ab97ba29f6f94d74f4e51642c56834

tvix/build/protos/default.nix

@@ -0,0 +1,35 @@
+{ depot, pkgs, ... }: {
+  # Produces the golang bindings.
+  go-bindings = pkgs.stdenv.mkDerivation {
+    name = "go-bindings";
+
+    src = depot.nix.sparseTree {
+      name = "build-protos";
+      root = depot.path.origSrc;
+      paths = [
+        # We need to include castore.proto (only), as it's referred.
+        ../../castore/protos/castore.proto
+        ./build.proto
+        ./rpc_build.proto
+        ../../../buf.yaml
+        ../../../buf.gen.yaml
+      ];
+    };
+
+    nativeBuildInputs = [
+      pkgs.buf
+      pkgs.protoc-gen-go
+      pkgs.protoc-gen-go-grpc
+    ];
+
+    buildPhase = ''
+      export HOME=$TMPDIR
+      buf lint
+      buf format -d --exit-code
+      buf generate
+
+      mkdir -p $out
+      cp tvix/build/protos/*.pb.go $out/
+    '';
+  };
+}

tvix/build/protos/rpc_build.proto

@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: MIT
+// Copyright © 2022 The Tvix Authors
+syntax = "proto3";
+
+package tvix.build.v1;
+
+import "tvix/build/protos/build.proto";
+
+option go_package = "code.tvl.fyi/tvix/build-go;buildv1";
+
+service BuildService {
+  rpc DoBuild(BuildRequest) returns (Build);
+}

tvix/build/src/lib.rs

@@ -0,0 +1 @@
+pub mod proto;

tvix/build/src/proto/mod.rs

@@ -0,0 +1 @@
+tonic::include_proto!("tvix.build.v1");

tvix/default.nix

@@ -40,6 +40,11 @@ let
         nativeBuildInputs = protobufDep prev;
       };
 
+      tvix-build = prev: {
+        PROTO_ROOT = depot.tvix.proto;
+        nativeBuildInputs = protobufDep prev;
+      };
+
       tvix-castore = prev: {
         PROTO_ROOT = depot.tvix.proto;
         nativeBuildInputs = protobufDep prev;

tvix/proto/default.nix

@@ -6,6 +6,8 @@ depot.nix.sparseTree {
   name = "tvix-protos";
   root = depot.path.origSrc;
   paths = [
+    ../build/protos/build.proto
+    ../build/protos/rpc_build.proto
     ../castore/protos/castore.proto
     ../castore/protos/rpc_blobstore.proto
     ../castore/protos/rpc_directory.proto