From f6c94430c8d71b95660ffff2ef621d2747a08cad Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Sat, 23 Dec 2023 23:50:29 +0200 Subject: [PATCH] feat(tvix/build/protos): add some missing fields - directory in which the castore input nodes are mounted - working directory for the build command - scratch paths - network access y/n - whether a (static) /bin/sh should be provided Populate these fields appropriately, and extend the tests in tvix-glue with a FOD example. Change-Id: I4f9de1483d6696d74694a09784910c407acb0be0 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10412 Autosubmit: flokli Tested-by: BuildkiteCI Reviewed-by: sterni --- tvix/build-go/build.pb.go | 184 ++++++++++++------ tvix/build/protos/build.proto | 49 +++-- .../0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv | 1 + tvix/glue/src/tvix_build.rs | 107 ++++++++-- 4 files changed, 252 insertions(+), 89 deletions(-) create mode 100644 tvix/glue/src/tests/0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv diff --git a/tvix/build-go/build.pb.go b/tvix/build-go/build.pb.go index fe996cb40..8c65b04b7 100644 --- a/tvix/build-go/build.pb.go +++ b/tvix/build-go/build.pb.go @@ -64,15 +64,31 @@ type BuildRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // The list of all root nodes that should be visible in STORE_DIR at the time + // of the build. + // As root nodes are content-addressed, no additional signatures are needed + // to substitute / make these available in the build environment. + // Inputs are sorted by their names. + Inputs []*castore_go.Node `protobuf:"bytes,1,rep,name=inputs,proto3" json:"inputs,omitempty"` // The command (and its args) executed as the build script. // In the case of a Nix derivation, this is usually // ["/path/to/some-bash/bin/bash", "-e", "/path/to/some/builder.sh"]. - CommandArgs []string `protobuf:"bytes,1,rep,name=command_args,json=commandArgs,proto3" json:"command_args,omitempty"` - // The list of outputs the build is expected to produce. - // These are basenames inside /nix/store. + CommandArgs []string `protobuf:"bytes,2,rep,name=command_args,json=commandArgs,proto3" json:"command_args,omitempty"` + // The working dir of the command, relative to the build root. + // "build", in the case of Nix. + WorkingDir string `protobuf:"bytes,3,opt,name=working_dir,json=workingDir,proto3" json:"working_dir,omitempty"` + // A list of "scratch" paths, relative to the build root. + // These will be write-able during the build. + // [build] in the case of Nix. + ScratchPaths []string `protobuf:"bytes,4,rep,name=scratch_paths,json=scratchPaths,proto3" json:"scratch_paths,omitempty"` + // The path where the castore input nodes will be located at, + // "/nix/store" in case of Nix. + StoreDir string `protobuf:"bytes,5,opt,name=store_dir,json=storeDir,proto3" json:"store_dir,omitempty"` + // The list of output nodes the build is expected to produce. + // These are basenames inside store_dir. // If the path is not produced, the build is considered to have failed. // Outputs are sorted. - Outputs []string `protobuf:"bytes,2,rep,name=outputs,proto3" json:"outputs,omitempty"` + Outputs []string `protobuf:"bytes,6,rep,name=outputs,proto3" json:"outputs,omitempty"` // The list of environment variables and their values that should be set // inside the build environment. // This includes both environment vars set inside the derivation, as well as @@ -83,16 +99,10 @@ type BuildRequest struct { // We don't want to bleed these very nix-specific sandbox impl details into // (dumber) builders if we don't have to. // Environment variables are sorted by their keys. - EnvironmentVars []*BuildRequest_EnvVar `protobuf:"bytes,3,rep,name=environment_vars,json=environmentVars,proto3" json:"environment_vars,omitempty"` - // The list of all root nodes that should be visible in /nix/store at the - // time of the build. - // As root nodes are content-addressed, no additional signatures are needed - // to substitute / make these available in the build environment. - // Inputs are sorted by their names. - Inputs []*castore_go.Node `protobuf:"bytes,4,rep,name=inputs,proto3" json:"inputs,omitempty"` + EnvironmentVars []*BuildRequest_EnvVar `protobuf:"bytes,7,rep,name=environment_vars,json=environmentVars,proto3" json:"environment_vars,omitempty"` // A set of constraints that need to be satisfied on a build host before a // Build can be started. - Constraints *BuildRequest_BuildConstraints `protobuf:"bytes,5,opt,name=constraints,proto3" json:"constraints,omitempty"` + Constraints *BuildRequest_BuildConstraints `protobuf:"bytes,8,opt,name=constraints,proto3" json:"constraints,omitempty"` } func (x *BuildRequest) Reset() { @@ -127,6 +137,13 @@ func (*BuildRequest) Descriptor() ([]byte, []int) { return file_tvix_build_protos_build_proto_rawDescGZIP(), []int{0} } +func (x *BuildRequest) GetInputs() []*castore_go.Node { + if x != nil { + return x.Inputs + } + return nil +} + func (x *BuildRequest) GetCommandArgs() []string { if x != nil { return x.CommandArgs @@ -134,6 +151,27 @@ func (x *BuildRequest) GetCommandArgs() []string { return nil } +func (x *BuildRequest) GetWorkingDir() string { + if x != nil { + return x.WorkingDir + } + return "" +} + +func (x *BuildRequest) GetScratchPaths() []string { + if x != nil { + return x.ScratchPaths + } + return nil +} + +func (x *BuildRequest) GetStoreDir() string { + if x != nil { + return x.StoreDir + } + return "" +} + func (x *BuildRequest) GetOutputs() []string { if x != nil { return x.Outputs @@ -148,13 +186,6 @@ func (x *BuildRequest) GetEnvironmentVars() []*BuildRequest_EnvVar { return nil } -func (x *BuildRequest) GetInputs() []*castore_go.Node { - if x != nil { - return x.Inputs - } - return nil -} - func (x *BuildRequest) GetConstraints() *BuildRequest_BuildConstraints { if x != nil { return x.Constraints @@ -291,9 +322,13 @@ type BuildRequest_BuildConstraints struct { // The amount of memory required to be available for the build, in bytes. MinMemory uint64 `protobuf:"varint,2,opt,name=min_memory,json=minMemory,proto3" json:"min_memory,omitempty"` // A list of (absolute) paths that need to be available in the build - // environment. - // TBD, This is probably things like /dev/kvm, but no nix store paths. + // environment, like `/dev/kvm`. + // This is distinct from the castore nodes in inputs. AvailableRoPaths []string `protobuf:"bytes,3,rep,name=available_ro_paths,json=availableRoPaths,proto3" json:"available_ro_paths,omitempty"` + // Whether the build should be able to access the network, + NetworkAccess bool `protobuf:"varint,4,opt,name=network_access,json=networkAccess,proto3" json:"network_access,omitempty"` + // Whether to provide a /bin/sh inside the build environment, usually a static bash. + ProvideBinSh bool `protobuf:"varint,5,opt,name=provide_bin_sh,json=provideBinSh,proto3" json:"provide_bin_sh,omitempty"` } func (x *BuildRequest_BuildConstraints) Reset() { @@ -349,6 +384,20 @@ func (x *BuildRequest_BuildConstraints) GetAvailableRoPaths() []string { return nil } +func (x *BuildRequest_BuildConstraints) GetNetworkAccess() bool { + if x != nil { + return x.NetworkAccess + } + return false +} + +func (x *BuildRequest_BuildConstraints) GetProvideBinSh() bool { + if x != nil { + return x.ProvideBinSh + } + return false +} + var File_tvix_build_protos_build_proto protoreflect.FileDescriptor var file_tvix_build_protos_build_proto_rawDesc = []byte{ @@ -357,46 +406,57 @@ var file_tvix_build_protos_build_proto_rawDesc = []byte{ 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0xc4, 0x03, 0x0a, 0x0c, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x61, 0x72, - 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, - 0x64, 0x41, 0x72, 0x67, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x12, - 0x4d, 0x0a, 0x10, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x76, - 0x61, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x76, 0x69, 0x78, + 0x6f, 0x22, 0xf5, 0x04, 0x0a, 0x0c, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, + 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x61, 0x72, 0x67, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x41, 0x72, 0x67, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x5f, + 0x64, 0x69, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x69, + 0x6e, 0x67, 0x44, 0x69, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x63, 0x72, 0x61, 0x74, 0x63, 0x68, + 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x63, + 0x72, 0x61, 0x74, 0x63, 0x68, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x44, 0x69, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, + 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, + 0x73, 0x12, 0x4d, 0x0a, 0x10, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x5f, 0x76, 0x61, 0x72, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x74, 0x76, + 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, + 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x76, 0x56, 0x61, 0x72, 0x52, + 0x0f, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x56, 0x61, 0x72, 0x73, + 0x12, 0x4e, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, + 0x6e, 0x74, 0x73, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, + 0x1a, 0x30, 0x0a, 0x06, 0x45, 0x6e, 0x76, 0x56, 0x61, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x1a, 0xc4, 0x01, 0x0a, 0x10, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x73, + 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x79, 0x73, 0x74, 0x65, + 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, + 0x1d, 0x0a, 0x0a, 0x6d, 0x69, 0x6e, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x09, 0x6d, 0x69, 0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x2c, + 0x0a, 0x12, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x6f, 0x5f, 0x70, + 0x61, 0x74, 0x68, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x76, 0x61, 0x69, + 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x25, 0x0a, 0x0e, + 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x5f, 0x62, + 0x69, 0x6e, 0x5f, 0x73, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x70, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x42, 0x69, 0x6e, 0x53, 0x68, 0x22, 0x7a, 0x0a, 0x05, 0x42, 0x75, 0x69, + 0x6c, 0x64, 0x12, 0x40, 0x0a, 0x0d, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x76, 0x56, 0x61, 0x72, 0x52, 0x0f, 0x65, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x56, 0x61, 0x72, 0x73, 0x12, 0x2d, - 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, - 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x4e, 0x0a, - 0x0b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, - 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x42, 0x75, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, - 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x1a, 0x30, 0x0a, - 0x06, 0x45, 0x6e, 0x76, 0x56, 0x61, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, - 0x77, 0x0a, 0x10, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, - 0x6e, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, - 0x69, 0x6e, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x09, 0x6d, 0x69, 0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x76, - 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, - 0x65, 0x52, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x73, 0x22, 0x7a, 0x0a, 0x05, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x12, 0x40, 0x0a, 0x0d, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, - 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x07, 0x6f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x73, 0x42, 0x24, 0x5a, 0x22, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, - 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2d, - 0x67, 0x6f, 0x3b, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x07, 0x6f, 0x75, + 0x74, 0x70, 0x75, 0x74, 0x73, 0x42, 0x24, 0x5a, 0x22, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, + 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x2d, 0x67, 0x6f, 0x3b, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -420,8 +480,8 @@ var file_tvix_build_protos_build_proto_goTypes = []interface{}{ (*castore_go.Node)(nil), // 4: tvix.castore.v1.Node } var file_tvix_build_protos_build_proto_depIdxs = []int32{ - 2, // 0: tvix.build.v1.BuildRequest.environment_vars:type_name -> tvix.build.v1.BuildRequest.EnvVar - 4, // 1: tvix.build.v1.BuildRequest.inputs:type_name -> tvix.castore.v1.Node + 4, // 0: tvix.build.v1.BuildRequest.inputs:type_name -> tvix.castore.v1.Node + 2, // 1: tvix.build.v1.BuildRequest.environment_vars:type_name -> tvix.build.v1.BuildRequest.EnvVar 3, // 2: tvix.build.v1.BuildRequest.constraints:type_name -> tvix.build.v1.BuildRequest.BuildConstraints 0, // 3: tvix.build.v1.Build.build_request:type_name -> tvix.build.v1.BuildRequest 4, // 4: tvix.build.v1.Build.outputs:type_name -> tvix.castore.v1.Node diff --git a/tvix/build/protos/build.proto b/tvix/build/protos/build.proto index 75146a8bd..bdabb037a 100644 --- a/tvix/build/protos/build.proto +++ b/tvix/build/protos/build.proto @@ -45,16 +45,36 @@ option go_package = "code.tvl.fyi/tvix/build-go;buildv1"; // support "send all BuildRequest for a nixpkgs eval to a remote builder and put // the laptop to sleep" usecases later. message BuildRequest { + // The list of all root nodes that should be visible in STORE_DIR at the time + // of the build. + // As root nodes are content-addressed, no additional signatures are needed + // to substitute / make these available in the build environment. + // Inputs are sorted by their names. + repeated tvix.castore.v1.Node inputs = 1; + // The command (and its args) executed as the build script. // In the case of a Nix derivation, this is usually // ["/path/to/some-bash/bin/bash", "-e", "/path/to/some/builder.sh"]. - repeated string command_args = 1; + repeated string command_args = 2; - // The list of outputs the build is expected to produce. - // These are basenames inside /nix/store. + // The working dir of the command, relative to the build root. + // "build", in the case of Nix. + string working_dir = 3; + + // A list of "scratch" paths, relative to the build root. + // These will be write-able during the build. + // [build] in the case of Nix. + repeated string scratch_paths = 4; + + // The path where the castore input nodes will be located at, + // "/nix/store" in case of Nix. + string store_dir = 5; + + // The list of output nodes the build is expected to produce. + // These are basenames inside store_dir. // If the path is not produced, the build is considered to have failed. // Outputs are sorted. - repeated string outputs = 2; + repeated string outputs = 6; // The list of environment variables and their values that should be set // inside the build environment. @@ -66,23 +86,16 @@ message BuildRequest { // We don't want to bleed these very nix-specific sandbox impl details into // (dumber) builders if we don't have to. // Environment variables are sorted by their keys. - repeated EnvVar environment_vars = 3; + repeated EnvVar environment_vars = 7; message EnvVar { string key = 1; bytes value = 2; } - // The list of all root nodes that should be visible in /nix/store at the - // time of the build. - // As root nodes are content-addressed, no additional signatures are needed - // to substitute / make these available in the build environment. - // Inputs are sorted by their names. - repeated tvix.castore.v1.Node inputs = 4; - // A set of constraints that need to be satisfied on a build host before a // Build can be started. - BuildConstraints constraints = 5; + BuildConstraints constraints = 8; // BuildConstraints represents certain conditions that must be fulfilled // inside the build environment to be able to build this. @@ -97,9 +110,15 @@ message BuildRequest { uint64 min_memory = 2; // A list of (absolute) paths that need to be available in the build - // environment. - // TBD, This is probably things like /dev/kvm, but no nix store paths. + // environment, like `/dev/kvm`. + // This is distinct from the castore nodes in inputs. repeated string available_ro_paths = 3; + + // Whether the build should be able to access the network, + bool network_access = 4; + + // Whether to provide a /bin/sh inside the build environment, usually a static bash. + bool provide_bin_sh = 5; } // TODO: allow describing something like "preferLocal", to influence composition? diff --git a/tvix/glue/src/tests/0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv b/tvix/glue/src/tests/0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv new file mode 100644 index 000000000..a4fea3c5f --- /dev/null +++ b/tvix/glue/src/tests/0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/4q0pg5zpfmznxscq3avycvf9xdvx50n3-bar","r:sha256","08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba")],[],[],":",":",[],[("builder",":"),("name","bar"),("out","/nix/store/4q0pg5zpfmznxscq3avycvf9xdvx50n3-bar"),("outputHash","08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba"),("outputHashAlgo","sha256"),("outputHashMode","recursive"),("system",":")]) \ No newline at end of file diff --git a/tvix/glue/src/tvix_build.rs b/tvix/glue/src/tvix_build.rs index e939abf3c..a0ebf49e1 100644 --- a/tvix/glue/src/tvix_build.rs +++ b/tvix/glue/src/tvix_build.rs @@ -134,12 +134,19 @@ where .cmp(b.node.as_ref().unwrap().get_name()) }); - // Produce constraints. We currently only put platform in here. - // Maybe more things need to be added here in the future. + // Produce constraints. let constraints = Some(BuildConstraints { system: derivation.system.clone(), min_memory: 0, available_ro_paths: vec![], + // in case this is a fixed-output derivation, allow network access. + network_access: derivation.outputs.len() == 1 + && derivation + .outputs + .get("out") + .expect("invalid derivation") + .is_fixed(), + provide_bin_sh: true, }); BuildRequest { @@ -148,6 +155,9 @@ where environment_vars, inputs, constraints, + working_dir: "build".into(), + scratch_paths: vec!["build".into()], + store_dir: nix_compat::store_path::STORE_DIR.into(), } } @@ -205,24 +215,24 @@ mod test { let mut expected_environment_vars = vec![ EnvVar { - key: "bar".to_string(), - value: Bytes::from("/nix/store/mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar"), + key: "bar".into(), + value: "/nix/store/mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar".into(), }, EnvVar { - key: "builder".to_string(), - value: Bytes::from(":"), + key: "builder".into(), + value: ":".into(), }, EnvVar { - key: "name".to_string(), - value: Bytes::from("foo"), + key: "name".into(), + value: "foo".into(), }, EnvVar { - key: "out".to_string(), - value: Bytes::from("/nix/store/fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo"), + key: "out".into(), + value: "/nix/store/fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo".into(), }, EnvVar { - key: "system".to_string(), - value: Bytes::from(":"), + key: "system".into(), + value: ":".into(), }, ]; @@ -242,8 +252,81 @@ mod test { constraints: Some(BuildConstraints { system: derivation.system.clone(), min_memory: 0, + network_access: false, available_ro_paths: vec![], + provide_bin_sh: true, }), + working_dir: "build".to_string(), + scratch_paths: vec!["build".to_string()], + store_dir: nix_compat::store_path::STORE_DIR.to_string(), + }, + build_request + ); + } + + #[test] + fn test_fod_to_build_request() { + let aterm_bytes = include_bytes!("tests/0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv"); + + let derivation = Derivation::from_aterm_bytes(aterm_bytes).expect("must parse"); + + let build_request = + derivation_to_build_request(&derivation, |_| unreachable!(), |_, _| unreachable!()); + + let mut expected_environment_vars = vec![ + EnvVar { + key: "builder".into(), + value: ":".into(), + }, + EnvVar { + key: "name".into(), + value: "bar".into(), + }, + EnvVar { + key: "out".into(), + value: "/nix/store/4q0pg5zpfmznxscq3avycvf9xdvx50n3-bar".into(), + }, + EnvVar { + key: "outputHash".into(), + value: "08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba".into(), + }, + EnvVar { + key: "outputHashAlgo".into(), + value: "sha256".into(), + }, + EnvVar { + key: "outputHashMode".into(), + value: "recursive".into(), + }, + EnvVar { + key: "system".into(), + value: ":".into(), + }, + ]; + + expected_environment_vars.extend(NIX_ENVIRONMENT_VARS.iter().map(|(k, v)| EnvVar { + key: k.to_string(), + value: Bytes::from_static(v.as_bytes()), + })); + + expected_environment_vars.sort_unstable_by_key(|e| e.key.to_owned()); + + assert_eq!( + BuildRequest { + command_args: vec![":".to_string()], + outputs: vec!["4q0pg5zpfmznxscq3avycvf9xdvx50n3-bar".to_string()], + environment_vars: expected_environment_vars, + inputs: vec![], + constraints: Some(BuildConstraints { + system: derivation.system.clone(), + min_memory: 0, + network_access: true, + available_ro_paths: vec![], + provide_bin_sh: true, + }), + working_dir: "build".to_string(), + scratch_paths: vec!["build".to_string()], + store_dir: nix_compat::store_path::STORE_DIR.to_string(), }, build_request );