feat(tvix/build): init
This adds the tvix-build crate, currently only containing a `tvix_build::proto` module, exposing the data structures defined in tvix/build/protos. Change-Id: I75f5d9196969ed0877b1fe640cacfecba0fb2e03 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10235 Autosubmit: flokli <flokli@flokli.de> Reviewed-by: raitobezarius <tvl@lahfa.xyz> Tested-by: BuildkiteCI
This commit is contained in:
parent
3c0a9a949a
commit
fd27d8ddc3
14 changed files with 330 additions and 0 deletions
12
tvix/Cargo.lock
generated
12
tvix/Cargo.lock
generated
|
@ -3050,6 +3050,18 @@ version = "0.2.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
|
checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tvix-build"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"prost",
|
||||||
|
"prost-build",
|
||||||
|
"tonic",
|
||||||
|
"tonic-build",
|
||||||
|
"tonic-reflection",
|
||||||
|
"tvix-castore",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tvix-castore"
|
name = "tvix-castore"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
|
|
|
@ -43,6 +43,16 @@ rec {
|
||||||
# File a bug if you depend on any for non-debug work!
|
# File a bug if you depend on any for non-debug work!
|
||||||
debug = internal.debugCrate { inherit packageId; };
|
debug = internal.debugCrate { inherit packageId; };
|
||||||
};
|
};
|
||||||
|
"tvix-build" = rec {
|
||||||
|
packageId = "tvix-build";
|
||||||
|
build = internal.buildRustCrateWithFeatures {
|
||||||
|
packageId = "tvix-build";
|
||||||
|
};
|
||||||
|
|
||||||
|
# Debug support which might change between releases.
|
||||||
|
# File a bug if you depend on any for non-debug work!
|
||||||
|
debug = internal.debugCrate { inherit packageId; };
|
||||||
|
};
|
||||||
"tvix-castore" = rec {
|
"tvix-castore" = rec {
|
||||||
packageId = "tvix-castore";
|
packageId = "tvix-castore";
|
||||||
build = internal.buildRustCrateWithFeatures {
|
build = internal.buildRustCrateWithFeatures {
|
||||||
|
@ -9348,6 +9358,50 @@ rec {
|
||||||
];
|
];
|
||||||
|
|
||||||
};
|
};
|
||||||
|
"tvix-build" = rec {
|
||||||
|
crateName = "tvix-build";
|
||||||
|
version = "0.1.0";
|
||||||
|
edition = "2021";
|
||||||
|
# We can't filter paths with references in Nix 2.4
|
||||||
|
# See https://github.com/NixOS/nix/issues/5410
|
||||||
|
src =
|
||||||
|
if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion))
|
||||||
|
then lib.cleanSourceWith { filter = sourceFilter; src = ./build; }
|
||||||
|
else ./build;
|
||||||
|
dependencies = [
|
||||||
|
{
|
||||||
|
name = "prost";
|
||||||
|
packageId = "prost";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "tonic";
|
||||||
|
packageId = "tonic";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "tonic-reflection";
|
||||||
|
packageId = "tonic-reflection";
|
||||||
|
optional = true;
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "tvix-castore";
|
||||||
|
packageId = "tvix-castore";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
buildDependencies = [
|
||||||
|
{
|
||||||
|
name = "prost-build";
|
||||||
|
packageId = "prost-build";
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "tonic-build";
|
||||||
|
packageId = "tonic-build";
|
||||||
|
}
|
||||||
|
];
|
||||||
|
features = {
|
||||||
|
"tonic-reflection" = [ "dep:tonic-reflection" ];
|
||||||
|
};
|
||||||
|
resolvedDefaultFeatures = [ "default" "tonic-reflection" ];
|
||||||
|
};
|
||||||
"tvix-castore" = rec {
|
"tvix-castore" = rec {
|
||||||
crateName = "tvix-castore";
|
crateName = "tvix-castore";
|
||||||
version = "0.1.0";
|
version = "0.1.0";
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
resolver = "2"
|
resolver = "2"
|
||||||
|
|
||||||
members = [
|
members = [
|
||||||
|
"build",
|
||||||
"castore",
|
"castore",
|
||||||
"cli",
|
"cli",
|
||||||
"eval",
|
"eval",
|
||||||
|
|
21
tvix/build/Cargo.toml
Normal file
21
tvix/build/Cargo.toml
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
[package]
|
||||||
|
name = "tvix-build"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition = "2021"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
prost = "0.12.1"
|
||||||
|
tonic = "0.10.2"
|
||||||
|
tvix-castore = { path = "../castore" }
|
||||||
|
|
||||||
|
[dependencies.tonic-reflection]
|
||||||
|
optional = true
|
||||||
|
version = "0.10.2"
|
||||||
|
|
||||||
|
[build-dependencies]
|
||||||
|
prost-build = "0.12.1"
|
||||||
|
tonic-build = "0.10.2"
|
||||||
|
|
||||||
|
[features]
|
||||||
|
default = []
|
||||||
|
tonic-reflection = ["dep:tonic-reflection"]
|
38
tvix/build/build.rs
Normal file
38
tvix/build/build.rs
Normal file
|
@ -0,0 +1,38 @@
|
||||||
|
use std::io::Result;
|
||||||
|
|
||||||
|
fn main() -> Result<()> {
|
||||||
|
#[allow(unused_mut)]
|
||||||
|
let mut builder = tonic_build::configure();
|
||||||
|
|
||||||
|
#[cfg(feature = "tonic-reflection")]
|
||||||
|
{
|
||||||
|
let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap());
|
||||||
|
let descriptor_path = out_dir.join("tvix.build.v1.bin");
|
||||||
|
|
||||||
|
builder = builder.file_descriptor_set_path(descriptor_path);
|
||||||
|
};
|
||||||
|
|
||||||
|
// https://github.com/hyperium/tonic/issues/908
|
||||||
|
let mut config = prost_build::Config::new();
|
||||||
|
config.bytes(["."]);
|
||||||
|
config.extern_path(".tvix.castore.v1", "::tvix_castore::proto");
|
||||||
|
|
||||||
|
builder
|
||||||
|
.build_server(true)
|
||||||
|
.build_client(true)
|
||||||
|
.compile_with_config(
|
||||||
|
config,
|
||||||
|
&[
|
||||||
|
"tvix/build/protos/build.proto",
|
||||||
|
"tvix/build/protos/rpc_build.proto",
|
||||||
|
],
|
||||||
|
// If we are in running `cargo build` manually, using `../..` works fine,
|
||||||
|
// but in case we run inside a nix build, we need to instead point PROTO_ROOT
|
||||||
|
// to a sparseTree containing that structure.
|
||||||
|
&[match std::env::var_os("PROTO_ROOT") {
|
||||||
|
Some(proto_root) => proto_root.to_str().unwrap().to_owned(),
|
||||||
|
None => "../..".to_string(),
|
||||||
|
}],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
5
tvix/build/default.nix
Normal file
5
tvix/build/default.nix
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
{ depot, pkgs, ... }:
|
||||||
|
|
||||||
|
depot.tvix.crates.workspaceMembers.tvix-build.build.override {
|
||||||
|
runTests = true;
|
||||||
|
}
|
21
tvix/build/protos/LICENSE
Normal file
21
tvix/build/protos/LICENSE
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
Copyright © The Tvix Authors
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining
|
||||||
|
a copy of this software and associated documentation files (the
|
||||||
|
“Software”), to deal in the Software without restriction, including
|
||||||
|
without limitation the rights to use, copy, modify, merge, publish,
|
||||||
|
distribute, sublicense, and/or sell copies of the Software, and to
|
||||||
|
permit persons to whom the Software is furnished to do so, subject to
|
||||||
|
the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be
|
||||||
|
included in all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||||
|
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||||
|
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||||
|
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||||
|
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
121
tvix/build/protos/build.proto
Normal file
121
tvix/build/protos/build.proto
Normal file
|
@ -0,0 +1,121 @@
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
// Copyright © 2022 The Tvix Authors
|
||||||
|
|
||||||
|
syntax = "proto3";
|
||||||
|
|
||||||
|
package tvix.build.v1;
|
||||||
|
|
||||||
|
import "tvix/castore/protos/castore.proto";
|
||||||
|
|
||||||
|
option go_package = "code.tvl.fyi/tvix/build-go;buildv1";
|
||||||
|
|
||||||
|
// A BuildRequest describes the request of something to be run on the builder.
|
||||||
|
// It is distinct from an actual [Build] that has already happened, or might be
|
||||||
|
// currently ongoing.
|
||||||
|
//
|
||||||
|
// A BuildRequest can be seen as a more normalized version of a Derivation
|
||||||
|
// (parsed from A-Term), "writing out" some of the Nix-internal details about
|
||||||
|
// how e.g. environment variables in the build are set.
|
||||||
|
//
|
||||||
|
// Nix has some impurities when building a Derivation, for example the --cores option
|
||||||
|
// ends up as an environment variable in the build, that's not part of the ATerm.
|
||||||
|
//
|
||||||
|
// As of now, we serialize this into the BuildRequest, so builders can stay dumb.
|
||||||
|
// This might change in the future.
|
||||||
|
//
|
||||||
|
// There's also a big difference when it comes to how inputs are modelled:
|
||||||
|
// - Nix only uses store path (strings) to describe the inputs.
|
||||||
|
// As store paths can be input-addressed, a certain store path can contain
|
||||||
|
// different contents (as not all store paths are binary reproducible).
|
||||||
|
// This requires that for every input-addressed input, the builder has access
|
||||||
|
// to either the input's deriver (and needs to build it) or else a trusted
|
||||||
|
// source for the built input.
|
||||||
|
// to upload input-addressed paths, requiring the trusted users concept.
|
||||||
|
// - tvix-build records a list of tvix.castore.v1.Node as inputs.
|
||||||
|
// These map from the store path base name to their contents, relieving the
|
||||||
|
// builder from having to "trust" any input-addressed paths, contrary to Nix.
|
||||||
|
//
|
||||||
|
// While this approach gives a better hermeticity, it has one downside:
|
||||||
|
// A BuildRequest can only be sent once the contents of all its inputs are known.
|
||||||
|
//
|
||||||
|
// As of now, we're okay to accept this, but it prevents uploading an
|
||||||
|
// entirely-non-IFD subgraph of BuildRequests eagerly.
|
||||||
|
//
|
||||||
|
// FUTUREWORK: We might be introducing another way to refer to inputs, to
|
||||||
|
// support "send all BuildRequest for a nixpkgs eval to a remote builder and put
|
||||||
|
// the laptop to sleep" usecases later.
|
||||||
|
message BuildRequest {
|
||||||
|
// The command (and its args) executed as the build script.
|
||||||
|
// In the case of a Nix derivation, this is usually
|
||||||
|
// ["/path/to/some-bash/bin/bash", "-e", "/path/to/some/builder.sh"].
|
||||||
|
repeated string command_args = 1;
|
||||||
|
|
||||||
|
// The list of outputs the build is expected to produce.
|
||||||
|
// These are basenames inside /nix/store.
|
||||||
|
// If the path is not produced, the build is considered to have failed.
|
||||||
|
// Outputs are sorted.
|
||||||
|
repeated string outputs = 2;
|
||||||
|
|
||||||
|
// The list of environment variables and their values that should be set
|
||||||
|
// inside the build environment.
|
||||||
|
// This includes both environment vars set inside the derivation, as well as
|
||||||
|
// more "ephemeral" ones like NIX_BUILD_CORES, controlled by the `--cores`
|
||||||
|
// CLI option of `nix-build`.
|
||||||
|
// For now, we consume this as an option when turning a Derivation into a BuildRequest,
|
||||||
|
// similar to how Nix has a `--cores` option.
|
||||||
|
// We don't want to bleed these very nix-specific sandbox impl details into
|
||||||
|
// (dumber) builders if we don't have to.
|
||||||
|
// Environment variables are sorted by their keys.
|
||||||
|
repeated EnvVar environment_vars = 3;
|
||||||
|
|
||||||
|
message EnvVar {
|
||||||
|
string key = 1;
|
||||||
|
bytes value = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The list of all root nodes that should be visible in /nix/store at the
|
||||||
|
// time of the build.
|
||||||
|
// As root nodes are content-addressed, no additional signatures are needed
|
||||||
|
// to substitute / make these available in the build environment.
|
||||||
|
// Inputs are sorted by their names.
|
||||||
|
repeated tvix.castore.v1.Node inputs = 4;
|
||||||
|
|
||||||
|
// A set of constraints that need to be satisfied on a build host before a
|
||||||
|
// Build can be started.
|
||||||
|
BuildConstraints constraints = 5;
|
||||||
|
|
||||||
|
// BuildConstraints represents certain conditions that must be fulfilled
|
||||||
|
// inside the build environment to be able to build this.
|
||||||
|
// Constraints can be things like required architecture and minimum amount of memory.
|
||||||
|
// The required input paths are *not* represented in here, because it
|
||||||
|
// wouldn't be hermetic enough - see the comment around inputs too.
|
||||||
|
message BuildConstraints {
|
||||||
|
// The system that's needed to execute the build.
|
||||||
|
string system = 1;
|
||||||
|
|
||||||
|
// The amount of memory required to be available for the build, in bytes.
|
||||||
|
uint64 min_memory = 2;
|
||||||
|
|
||||||
|
// A list of (absolute) paths that need to be available in the build
|
||||||
|
// environment.
|
||||||
|
// TBD, This is probably things like /dev/kvm, but no nix store paths.
|
||||||
|
repeated string available_ro_paths = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: allow describing something like "preferLocal", to influence composition?
|
||||||
|
}
|
||||||
|
|
||||||
|
// A Build is (one possible) outcome of executing a [BuildRequest].
|
||||||
|
message Build {
|
||||||
|
// The orginal build request producing the build.
|
||||||
|
BuildRequest build_request = 1; // <- TODO: define hashing scheme for BuildRequest, refer to it by hash?
|
||||||
|
|
||||||
|
// The outputs that were produced after successfully building.
|
||||||
|
// They are sorted by their names.
|
||||||
|
repeated tvix.castore.v1.Node outputs = 2;
|
||||||
|
|
||||||
|
// TODO: where did this run, how long, logs, …
|
||||||
|
}
|
||||||
|
|
||||||
|
/// TODO: check remarkable notes on constraints again
|
||||||
|
/// TODO: https://github.com/adisbladis/go-nix/commit/603df5db86ab97ba29f6f94d74f4e51642c56834
|
35
tvix/build/protos/default.nix
Normal file
35
tvix/build/protos/default.nix
Normal file
|
@ -0,0 +1,35 @@
|
||||||
|
{ depot, pkgs, ... }: {
|
||||||
|
# Produces the golang bindings.
|
||||||
|
go-bindings = pkgs.stdenv.mkDerivation {
|
||||||
|
name = "go-bindings";
|
||||||
|
|
||||||
|
src = depot.nix.sparseTree {
|
||||||
|
name = "build-protos";
|
||||||
|
root = depot.path.origSrc;
|
||||||
|
paths = [
|
||||||
|
# We need to include castore.proto (only), as it's referred.
|
||||||
|
../../castore/protos/castore.proto
|
||||||
|
./build.proto
|
||||||
|
./rpc_build.proto
|
||||||
|
../../../buf.yaml
|
||||||
|
../../../buf.gen.yaml
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
nativeBuildInputs = [
|
||||||
|
pkgs.buf
|
||||||
|
pkgs.protoc-gen-go
|
||||||
|
pkgs.protoc-gen-go-grpc
|
||||||
|
];
|
||||||
|
|
||||||
|
buildPhase = ''
|
||||||
|
export HOME=$TMPDIR
|
||||||
|
buf lint
|
||||||
|
buf format -d --exit-code
|
||||||
|
buf generate
|
||||||
|
|
||||||
|
mkdir -p $out
|
||||||
|
cp tvix/build/protos/*.pb.go $out/
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
}
|
13
tvix/build/protos/rpc_build.proto
Normal file
13
tvix/build/protos/rpc_build.proto
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
// Copyright © 2022 The Tvix Authors
|
||||||
|
syntax = "proto3";
|
||||||
|
|
||||||
|
package tvix.build.v1;
|
||||||
|
|
||||||
|
import "tvix/build/protos/build.proto";
|
||||||
|
|
||||||
|
option go_package = "code.tvl.fyi/tvix/build-go;buildv1";
|
||||||
|
|
||||||
|
service BuildService {
|
||||||
|
rpc DoBuild(BuildRequest) returns (Build);
|
||||||
|
}
|
1
tvix/build/src/lib.rs
Normal file
1
tvix/build/src/lib.rs
Normal file
|
@ -0,0 +1 @@
|
||||||
|
pub mod proto;
|
1
tvix/build/src/proto/mod.rs
Normal file
1
tvix/build/src/proto/mod.rs
Normal file
|
@ -0,0 +1 @@
|
||||||
|
tonic::include_proto!("tvix.build.v1");
|
|
@ -40,6 +40,11 @@ let
|
||||||
nativeBuildInputs = protobufDep prev;
|
nativeBuildInputs = protobufDep prev;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
tvix-build = prev: {
|
||||||
|
PROTO_ROOT = depot.tvix.proto;
|
||||||
|
nativeBuildInputs = protobufDep prev;
|
||||||
|
};
|
||||||
|
|
||||||
tvix-castore = prev: {
|
tvix-castore = prev: {
|
||||||
PROTO_ROOT = depot.tvix.proto;
|
PROTO_ROOT = depot.tvix.proto;
|
||||||
nativeBuildInputs = protobufDep prev;
|
nativeBuildInputs = protobufDep prev;
|
||||||
|
|
|
@ -6,6 +6,8 @@ depot.nix.sparseTree {
|
||||||
name = "tvix-protos";
|
name = "tvix-protos";
|
||||||
root = depot.path.origSrc;
|
root = depot.path.origSrc;
|
||||||
paths = [
|
paths = [
|
||||||
|
../build/protos/build.proto
|
||||||
|
../build/protos/rpc_build.proto
|
||||||
../castore/protos/castore.proto
|
../castore/protos/castore.proto
|
||||||
../castore/protos/rpc_blobstore.proto
|
../castore/protos/rpc_blobstore.proto
|
||||||
../castore/protos/rpc_directory.proto
|
../castore/protos/rpc_directory.proto
|
||||||
|
|
Loading…
Reference in a new issue