mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/build/protos): add additional_files field

Browse source 
Nix builds also have some other inputs, outside of inputs_dir, used in
derivations setting `passAsFile` and `__structuredAttrs`.

This extends our `BuildRequest` structure to accomodate these usecases.

Change-Id: I8de6516eb467fa48a4961d88933ebd548f131049
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10519
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink 2024-01-01 17:07:42 +02:00 committed by clbot
parent 6b136dfd23
commit e6782bb7c4
3 changed files with 142 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

167
tvix/build-go/build.pb.go
View file

@ -104,6 +104,10 @@ type BuildRequest struct {
// A set of constraints that need to be satisfied on a build host before a
// Build can be started.
Constraints *BuildRequest_BuildConstraints `protobuf:"bytes,8,opt,name=constraints,proto3" json:"constraints,omitempty"`
// Additional (small) files and their contents that should be placed into the
// build environment, but outside inputs_dir.
// Used for passAsFile and structuredAttrs in Nix.
AdditionalFiles []*BuildRequest_AdditionalFile `protobuf:"bytes,9,rep,name=additional_files,json=additionalFiles,proto3" json:"additional_files,omitempty"`
}
func (x *BuildRequest) Reset() {
@ -194,6 +198,13 @@ func (x *BuildRequest) GetConstraints() *BuildRequest_BuildConstraints {
return nil
}
func (x *BuildRequest) GetAdditionalFiles() []*BuildRequest_AdditionalFile {
if x != nil {
return x.AdditionalFiles
}
return nil
}
// A Build is (one possible) outcome of executing a [BuildRequest].
type Build struct {
state protoimpl.MessageState
@ -399,6 +410,61 @@ func (x *BuildRequest_BuildConstraints) GetProvideBinSh() bool {
return false
}
type BuildRequest_AdditionalFile struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
Contents []byte `protobuf:"bytes,2,opt,name=contents,proto3" json:"contents,omitempty"`
}
func (x *BuildRequest_AdditionalFile) Reset() {
*x = BuildRequest_AdditionalFile{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_build_protos_build_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *BuildRequest_AdditionalFile) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*BuildRequest_AdditionalFile) ProtoMessage() {}
func (x *BuildRequest_AdditionalFile) ProtoReflect() protoreflect.Message {
mi := &file_tvix_build_protos_build_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use BuildRequest_AdditionalFile.ProtoReflect.Descriptor instead.
func (*BuildRequest_AdditionalFile) Descriptor() ([]byte, []int) {
return file_tvix_build_protos_build_proto_rawDescGZIP(), []int{0, 2}
}
func (x *BuildRequest_AdditionalFile) GetPath() string {
if x != nil {
return x.Path
}
return ""
}
func (x *BuildRequest_AdditionalFile) GetContents() []byte {
if x != nil {
return x.Contents
}
return nil
}
var File_tvix_build_protos_build_proto protoreflect.FileDescriptor
var file_tvix_build_protos_build_proto_rawDesc = []byte{
@ -407,7 +473,7 @@ var file_tvix_build_protos_build_proto_rawDesc = []byte{
0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x1a, 0x21,
0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x22, 0xf7, 0x04, 0x0a, 0x0c, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65,
0x6f, 0x22, 0x90, 0x06, 0x0a, 0x0c, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x12, 0x2d, 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03,
0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74,
@ -431,33 +497,42 @@ var file_tvix_build_protos_build_proto_rawDesc = []byte{
0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72,
0x61, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e,
0x74, 0x73, 0x1a, 0x30, 0x0a, 0x06, 0x45, 0x6e, 0x76, 0x56, 0x61, 0x72, 0x12, 0x10, 0x0a, 0x03,
0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14,
0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76,
0x61, 0x6c, 0x75, 0x65, 0x1a, 0xc4, 0x01, 0x0a, 0x10, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x43, 0x6f,
0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x79, 0x73,
0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x79, 0x73, 0x74, 0x65,
0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x69, 0x6e, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18,
0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x6d, 0x69, 0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79,
0x12, 0x2c, 0x0a, 0x12, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x6f,
0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x76,
0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x25,
0x0a, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73,
0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41,
0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65,
0x5f, 0x62, 0x69, 0x6e, 0x5f, 0x73, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x70,
0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x42, 0x69, 0x6e, 0x53, 0x68, 0x22, 0x7a, 0x0a, 0x05, 0x42,
0x75, 0x69, 0x6c, 0x64, 0x12, 0x40, 0x0a, 0x0d, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x72, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x76,
0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c,
0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74,
0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63,
0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x07,
0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x42, 0x24, 0x5a, 0x22, 0x63, 0x6f, 0x64, 0x65, 0x2e,
0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x62, 0x75, 0x69,
0x6c, 0x64, 0x2d, 0x67, 0x6f, 0x3b, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x76, 0x31, 0x62, 0x06, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x33,
0x74, 0x73, 0x12, 0x55, 0x0a, 0x10, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c,
0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x74,
0x76, 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69,
0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69,
0x6f, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69,
0x6f, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x1a, 0x30, 0x0a, 0x06, 0x45, 0x6e, 0x76,
0x56, 0x61, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0xc4, 0x01, 0x0a, 0x10,
0x42, 0x75, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73,
0x12, 0x16, 0x0a, 0x06, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x06, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x69, 0x6e, 0x5f,
0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x6d, 0x69,
0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x76, 0x61, 0x69, 0x6c,
0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x03, 0x20,
0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x6f,
0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b,
0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e,
0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x24, 0x0a, 0x0e,
0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x5f, 0x62, 0x69, 0x6e, 0x5f, 0x73, 0x68, 0x18, 0x05,
0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x42, 0x69, 0x6e,
0x53, 0x68, 0x1a, 0x40, 0x0a, 0x0e, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c,
0x46, 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74,
0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74,
0x65, 0x6e, 0x74, 0x73, 0x22, 0x7a, 0x0a, 0x05, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x12, 0x40, 0x0a,
0x0d, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c,
0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x52, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x2f, 0x0a, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b,
0x32, 0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e,
0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73,
0x42, 0x24, 0x5a, 0x22, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69,
0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2d, 0x67, 0x6f, 0x3b, 0x62,
0x75, 0x69, 0x6c, 0x64, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@ -472,25 +547,27 @@ func file_tvix_build_protos_build_proto_rawDescGZIP() []byte {
return file_tvix_build_protos_build_proto_rawDescData
}
var file_tvix_build_protos_build_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
var file_tvix_build_protos_build_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
var file_tvix_build_protos_build_proto_goTypes = []interface{}{
(*BuildRequest)(nil), // 0: tvix.build.v1.BuildRequest
(*Build)(nil), // 1: tvix.build.v1.Build
(*BuildRequest_EnvVar)(nil), // 2: tvix.build.v1.BuildRequest.EnvVar
(*BuildRequest_BuildConstraints)(nil), // 3: tvix.build.v1.BuildRequest.BuildConstraints
(*castore_go.Node)(nil), // 4: tvix.castore.v1.Node
(*BuildRequest_AdditionalFile)(nil), // 4: tvix.build.v1.BuildRequest.AdditionalFile
(*castore_go.Node)(nil), // 5: tvix.castore.v1.Node
}
var file_tvix_build_protos_build_proto_depIdxs = []int32{
4, // 0: tvix.build.v1.BuildRequest.inputs:type_name -> tvix.castore.v1.Node
5, // 0: tvix.build.v1.BuildRequest.inputs:type_name -> tvix.castore.v1.Node
2, // 1: tvix.build.v1.BuildRequest.environment_vars:type_name -> tvix.build.v1.BuildRequest.EnvVar
3, // 2: tvix.build.v1.BuildRequest.constraints:type_name -> tvix.build.v1.BuildRequest.BuildConstraints
0, // 3: tvix.build.v1.Build.build_request:type_name -> tvix.build.v1.BuildRequest
4, // 4: tvix.build.v1.Build.outputs:type_name -> tvix.castore.v1.Node
5, // [5:5] is the sub-list for method output_type
5, // [5:5] is the sub-list for method input_type
5, // [5:5] is the sub-list for extension type_name
5, // [5:5] is the sub-list for extension extendee
0, // [0:5] is the sub-list for field type_name
4, // 3: tvix.build.v1.BuildRequest.additional_files:type_name -> tvix.build.v1.BuildRequest.AdditionalFile
0, // 4: tvix.build.v1.Build.build_request:type_name -> tvix.build.v1.BuildRequest
5, // 5: tvix.build.v1.Build.outputs:type_name -> tvix.castore.v1.Node
6, // [6:6] is the sub-list for method output_type
6, // [6:6] is the sub-list for method input_type
6, // [6:6] is the sub-list for extension type_name
6, // [6:6] is the sub-list for extension extendee
0, // [0:6] is the sub-list for field type_name
}
func init() { file_tvix_build_protos_build_proto_init() }
@ -547,6 +624,18 @@ func file_tvix_build_protos_build_proto_init() {
return nil
}
}
file_tvix_build_protos_build_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*BuildRequest_AdditionalFile); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
@ -554,7 +643,7 @@ func file_tvix_build_protos_build_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_tvix_build_protos_build_proto_rawDesc,
NumEnums: 0,
NumMessages: 4,
NumMessages: 5,
NumExtensions: 0,
NumServices: 0,
},

10
tvix/build/protos/build.proto
View file

@ -122,6 +122,16 @@ message BuildRequest {
bool provide_bin_sh = 5;
}
// Additional (small) files and their contents that should be placed into the
// build environment, but outside inputs_dir.
// Used for passAsFile and structuredAttrs in Nix.
repeated AdditionalFile additional_files = 9;
message AdditionalFile {
string path = 1;
bytes contents = 2;
}
// TODO: allow describing something like "preferLocal", to influence composition?
}

4
tvix/glue/src/tvix_build.rs
View file

@ -151,6 +151,8 @@ where
constraints,
working_dir: "build".into(),
scratch_paths: vec!["build".into(), "nix/store".into()],
// TODO: handle passAsFile, structuredAttrs.
additional_files: vec![],
}
}
@ -250,6 +252,7 @@ mod test {
available_ro_paths: vec![],
provide_bin_sh: true,
}),
additional_files: vec![],
working_dir: "build".into(),
scratch_paths: vec!["build".into(), "nix/store".into()],
},
@ -318,6 +321,7 @@ mod test {
available_ro_paths: vec![],
provide_bin_sh: true,
}),
additional_files: vec![],
working_dir: "build".into(),
scratch_paths: vec!["build".into(), "nix/store".into()],
},