From e6782bb7c4ffad9d0ecf0bca5acb16f2b7698594 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Mon, 1 Jan 2024 17:07:42 +0200 Subject: [PATCH] feat(tvix/build/protos): add additional_files field Nix builds also have some other inputs, outside of inputs_dir, used in derivations setting `passAsFile` and `__structuredAttrs`. This extends our `BuildRequest` structure to accomodate these usecases. Change-Id: I8de6516eb467fa48a4961d88933ebd548f131049 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10519 Tested-by: BuildkiteCI Reviewed-by: tazjin Autosubmit: flokli --- tvix/build-go/build.pb.go | 167 ++++++++++++++++++++++++++-------- tvix/build/protos/build.proto | 10 ++ tvix/glue/src/tvix_build.rs | 4 + 3 files changed, 142 insertions(+), 39 deletions(-) diff --git a/tvix/build-go/build.pb.go b/tvix/build-go/build.pb.go index fd387c1c3..acd58b373 100644 --- a/tvix/build-go/build.pb.go +++ b/tvix/build-go/build.pb.go @@ -104,6 +104,10 @@ type BuildRequest struct { // A set of constraints that need to be satisfied on a build host before a // Build can be started. Constraints *BuildRequest_BuildConstraints `protobuf:"bytes,8,opt,name=constraints,proto3" json:"constraints,omitempty"` + // Additional (small) files and their contents that should be placed into the + // build environment, but outside inputs_dir. + // Used for passAsFile and structuredAttrs in Nix. + AdditionalFiles []*BuildRequest_AdditionalFile `protobuf:"bytes,9,rep,name=additional_files,json=additionalFiles,proto3" json:"additional_files,omitempty"` } func (x *BuildRequest) Reset() { @@ -194,6 +198,13 @@ func (x *BuildRequest) GetConstraints() *BuildRequest_BuildConstraints { return nil } +func (x *BuildRequest) GetAdditionalFiles() []*BuildRequest_AdditionalFile { + if x != nil { + return x.AdditionalFiles + } + return nil +} + // A Build is (one possible) outcome of executing a [BuildRequest]. type Build struct { state protoimpl.MessageState @@ -399,6 +410,61 @@ func (x *BuildRequest_BuildConstraints) GetProvideBinSh() bool { return false } +type BuildRequest_AdditionalFile struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + Contents []byte `protobuf:"bytes,2,opt,name=contents,proto3" json:"contents,omitempty"` +} + +func (x *BuildRequest_AdditionalFile) Reset() { + *x = BuildRequest_AdditionalFile{} + if protoimpl.UnsafeEnabled { + mi := &file_tvix_build_protos_build_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BuildRequest_AdditionalFile) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BuildRequest_AdditionalFile) ProtoMessage() {} + +func (x *BuildRequest_AdditionalFile) ProtoReflect() protoreflect.Message { + mi := &file_tvix_build_protos_build_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BuildRequest_AdditionalFile.ProtoReflect.Descriptor instead. +func (*BuildRequest_AdditionalFile) Descriptor() ([]byte, []int) { + return file_tvix_build_protos_build_proto_rawDescGZIP(), []int{0, 2} +} + +func (x *BuildRequest_AdditionalFile) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *BuildRequest_AdditionalFile) GetContents() []byte { + if x != nil { + return x.Contents + } + return nil +} + var File_tvix_build_protos_build_proto protoreflect.FileDescriptor var file_tvix_build_protos_build_proto_rawDesc = []byte{ @@ -407,7 +473,7 @@ var file_tvix_build_protos_build_proto_rawDesc = []byte{ 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0xf7, 0x04, 0x0a, 0x0c, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x6f, 0x22, 0x90, 0x06, 0x0a, 0x0c, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, @@ -431,33 +497,42 @@ var file_tvix_build_protos_build_proto_rawDesc = []byte{ 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, - 0x74, 0x73, 0x1a, 0x30, 0x0a, 0x06, 0x45, 0x6e, 0x76, 0x56, 0x61, 0x72, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x1a, 0xc4, 0x01, 0x0a, 0x10, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x43, 0x6f, - 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x79, 0x73, - 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x69, 0x6e, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x6d, 0x69, 0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, - 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x6f, - 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x76, - 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x25, - 0x0a, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, - 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x5f, 0x62, 0x69, 0x6e, 0x5f, 0x73, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x42, 0x69, 0x6e, 0x53, 0x68, 0x22, 0x7a, 0x0a, 0x05, 0x42, - 0x75, 0x69, 0x6c, 0x64, 0x12, 0x40, 0x0a, 0x0d, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x72, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x76, - 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, - 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x07, - 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x42, 0x24, 0x5a, 0x22, 0x63, 0x6f, 0x64, 0x65, 0x2e, - 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x2d, 0x67, 0x6f, 0x3b, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x76, 0x31, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x73, 0x12, 0x55, 0x0a, 0x10, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, + 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x74, + 0x76, 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, + 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x0f, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, + 0x6f, 0x6e, 0x61, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x1a, 0x30, 0x0a, 0x06, 0x45, 0x6e, 0x76, + 0x56, 0x61, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0xc4, 0x01, 0x0a, 0x10, + 0x42, 0x75, 0x69, 0x6c, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x74, 0x73, + 0x12, 0x16, 0x0a, 0x06, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x69, 0x6e, 0x5f, + 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x6d, 0x69, + 0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x2c, 0x0a, 0x12, 0x61, 0x76, 0x61, 0x69, 0x6c, + 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x10, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x6f, + 0x50, 0x61, 0x74, 0x68, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x24, 0x0a, 0x0e, + 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x5f, 0x62, 0x69, 0x6e, 0x5f, 0x73, 0x68, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x42, 0x69, 0x6e, + 0x53, 0x68, 0x1a, 0x40, 0x0a, 0x0e, 0x41, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, + 0x46, 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x73, 0x22, 0x7a, 0x0a, 0x05, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x12, 0x40, 0x0a, + 0x0d, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x52, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x2f, 0x0a, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, + 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x07, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, + 0x42, 0x24, 0x5a, 0x22, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, + 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x2d, 0x67, 0x6f, 0x3b, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -472,25 +547,27 @@ func file_tvix_build_protos_build_proto_rawDescGZIP() []byte { return file_tvix_build_protos_build_proto_rawDescData } -var file_tvix_build_protos_build_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_tvix_build_protos_build_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_tvix_build_protos_build_proto_goTypes = []interface{}{ (*BuildRequest)(nil), // 0: tvix.build.v1.BuildRequest (*Build)(nil), // 1: tvix.build.v1.Build (*BuildRequest_EnvVar)(nil), // 2: tvix.build.v1.BuildRequest.EnvVar (*BuildRequest_BuildConstraints)(nil), // 3: tvix.build.v1.BuildRequest.BuildConstraints - (*castore_go.Node)(nil), // 4: tvix.castore.v1.Node + (*BuildRequest_AdditionalFile)(nil), // 4: tvix.build.v1.BuildRequest.AdditionalFile + (*castore_go.Node)(nil), // 5: tvix.castore.v1.Node } var file_tvix_build_protos_build_proto_depIdxs = []int32{ - 4, // 0: tvix.build.v1.BuildRequest.inputs:type_name -> tvix.castore.v1.Node + 5, // 0: tvix.build.v1.BuildRequest.inputs:type_name -> tvix.castore.v1.Node 2, // 1: tvix.build.v1.BuildRequest.environment_vars:type_name -> tvix.build.v1.BuildRequest.EnvVar 3, // 2: tvix.build.v1.BuildRequest.constraints:type_name -> tvix.build.v1.BuildRequest.BuildConstraints - 0, // 3: tvix.build.v1.Build.build_request:type_name -> tvix.build.v1.BuildRequest - 4, // 4: tvix.build.v1.Build.outputs:type_name -> tvix.castore.v1.Node - 5, // [5:5] is the sub-list for method output_type - 5, // [5:5] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 4, // 3: tvix.build.v1.BuildRequest.additional_files:type_name -> tvix.build.v1.BuildRequest.AdditionalFile + 0, // 4: tvix.build.v1.Build.build_request:type_name -> tvix.build.v1.BuildRequest + 5, // 5: tvix.build.v1.Build.outputs:type_name -> tvix.castore.v1.Node + 6, // [6:6] is the sub-list for method output_type + 6, // [6:6] is the sub-list for method input_type + 6, // [6:6] is the sub-list for extension type_name + 6, // [6:6] is the sub-list for extension extendee + 0, // [0:6] is the sub-list for field type_name } func init() { file_tvix_build_protos_build_proto_init() } @@ -547,6 +624,18 @@ func file_tvix_build_protos_build_proto_init() { return nil } } + file_tvix_build_protos_build_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BuildRequest_AdditionalFile); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -554,7 +643,7 @@ func file_tvix_build_protos_build_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_tvix_build_protos_build_proto_rawDesc, NumEnums: 0, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 0, }, diff --git a/tvix/build/protos/build.proto b/tvix/build/protos/build.proto index e007ce968..f6c62b877 100644 --- a/tvix/build/protos/build.proto +++ b/tvix/build/protos/build.proto @@ -122,6 +122,16 @@ message BuildRequest { bool provide_bin_sh = 5; } + // Additional (small) files and their contents that should be placed into the + // build environment, but outside inputs_dir. + // Used for passAsFile and structuredAttrs in Nix. + repeated AdditionalFile additional_files = 9; + + message AdditionalFile { + string path = 1; + bytes contents = 2; + } + // TODO: allow describing something like "preferLocal", to influence composition? } diff --git a/tvix/glue/src/tvix_build.rs b/tvix/glue/src/tvix_build.rs index b647c7d40..85a9ba40a 100644 --- a/tvix/glue/src/tvix_build.rs +++ b/tvix/glue/src/tvix_build.rs @@ -151,6 +151,8 @@ where constraints, working_dir: "build".into(), scratch_paths: vec!["build".into(), "nix/store".into()], + // TODO: handle passAsFile, structuredAttrs. + additional_files: vec![], } } @@ -250,6 +252,7 @@ mod test { available_ro_paths: vec![], provide_bin_sh: true, }), + additional_files: vec![], working_dir: "build".into(), scratch_paths: vec!["build".into(), "nix/store".into()], }, @@ -318,6 +321,7 @@ mod test { available_ro_paths: vec![], provide_bin_sh: true, }), + additional_files: vec![], working_dir: "build".into(), scratch_paths: vec!["build".into(), "nix/store".into()], },