mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/store/protos): add Deriver field to PathInfo

Browse source 
This uses the newly introduced StorePath message type to add a Deriver
field to the PathInfo message.

Support for validation is added to both the golang and rust
implementation. This includes extending unit tests.

Change-Id: Ifc3eb3263fa25b9eec260db354cd74234c40af7e
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9647
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2023-10-11 00:22:40 +02:00 committed by flokli
parent 5f8eb4eeaa
commit 2d2c4322d9
10 changed files with 161 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
tvix/cli/src/tvix_store_io.rs
View file

@ -346,6 +346,7 @@ async fn import_path_with_pathinfo(
// TODO: narinfo for talosctl.src contains `CA: fixed:r:sha256:1x13j5hy75221bf6kz7cpgld9vgic6bqx07w5xjs4pxnksj6lxb6`
// do we need this anywhere?
}),
deriver: None,
};
// put into [PathInfoService], and return the [PathInfo] that we get

16
tvix/nix-compat/src/store_path/mod.rs
View file

@ -116,6 +116,14 @@ impl StorePath {
}
}
/// Construct a [StorePath] from a name and digest.
pub fn from_name_and_digest(name: String, digest: &[u8]) -> Result<StorePath, Error> {
Ok(Self {
name: validate_name(name.as_bytes())?,
digest: digest.try_into().map_err(|_| Error::InvalidLength())?,
})
}
/// Decompose a string into a [StorePath] and a [PathBuf] containing the
/// rest of the path, or an error.
#[cfg(target_family = "unix")]
@ -179,6 +187,14 @@ pub(crate) fn validate_name(s: &[u8]) -> Result<String, Error> {
Ok(String::from_utf8(s.to_vec()).unwrap())
}
/// Ensures the StorePath fulfils the requirements for store paths.
/// Useful when populating the struct manually instead of parsing.
pub fn validate(s: &StorePath) -> Result<(), Error> {
validate_name(s.name.as_bytes())?;
Ok(())
}
impl fmt::Display for StorePath {
/// The string representation of a store path starts with a digest (20
/// bytes), [crate::nixbase32]-encoded, followed by a `-`,

31
tvix/store/protos/pathinfo.go
View file

@ -60,42 +60,55 @@ func (p *PathInfo) Validate() (*storepath.StorePath, error) {
// for all three node types, ensure the name properly parses to a store path,
// and in case it refers to a digest, ensure it has the right length.
var storePath *storepath.StorePath
var err error
if node := rootNode.GetDirectory(); node != nil {
if len(node.Digest) != 32 {
return nil, fmt.Errorf("invalid digest size for %s, expected %d, got %d", node.Name, 32, len(node.Digest))
}
storePath, err := storepath.FromString(string(node.GetName()))
storePath, err = storepath.FromString(string(node.GetName()))
if err != nil {
return nil, fmt.Errorf("unable to parse %s as StorePath: %w", node.Name, err)
}
return storePath, nil
} else if node := rootNode.GetFile(); node != nil {
if len(node.Digest) != 32 {
return nil, fmt.Errorf("invalid digest size for %s, expected %d, got %d", node.Name, 32, len(node.Digest))
}
storePath, err := storepath.FromString(string(node.GetName()))
storePath, err = storepath.FromString(string(node.GetName()))
if err != nil {
return nil, fmt.Errorf("unable to parse %s as StorePath: %w", node.Name, err)
}
return storePath, nil
} else if node := rootNode.GetSymlink(); node != nil {
storePath, err := storepath.FromString(string(node.GetName()))
storePath, err = storepath.FromString(string(node.GetName()))
if err != nil {
return nil, fmt.Errorf("unable to parse %s as StorePath: %w", node.Name, err)
}
return storePath, nil
} else {
// this would only happen if we introduced a new type
panic("unreachable")
}
// If the Deriver field is populated, ensure it parses to a StorePath.
// We can't check for it to *not* end with .drv, as the .drv files produced by
// recursive Nix end with multiple .drv suffixes, and only one is popped when
// converting to this field.
if p.Deriver != nil {
storePath := storepath.StorePath{
Name: string(p.Deriver.GetName()),
Digest: p.Deriver.GetDigest(),
}
if err := storePath.Validate(); err != nil {
return nil, fmt.Errorf("invalid deriver field: %w", err)
}
}
return storePath, nil
}

73
tvix/store/protos/pathinfo.pb.go
View file

@ -39,6 +39,9 @@ type PathInfo struct {
References [][]byte `protobuf:"bytes,2,rep,name=references,proto3" json:"references,omitempty"`
// see below.
Narinfo *NARInfo `protobuf:"bytes,3,opt,name=narinfo,proto3" json:"narinfo,omitempty"`
// The StorePath of the .drv file producing this output.
// The .drv suffix is omitted in its `name` field.
Deriver *StorePath `protobuf:"bytes,4,opt,name=deriver,proto3" json:"deriver,omitempty"`
}
func (x *PathInfo) Reset() {
@ -94,6 +97,13 @@ func (x *PathInfo) GetNarinfo() *NARInfo {
return nil
}
func (x *PathInfo) GetDeriver() *StorePath {
if x != nil {
return x.Deriver
}
return nil
}
// Represents a path in the Nix store (a direct child of STORE_DIR).
// It is commonly formatted by a nixbase32-encoding the digest, and
// concatenating the name, separated by a `-`.
@ -306,7 +316,7 @@ var file_tvix_store_protos_pathinfo_proto_rawDesc = []byte{
0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76,
0x31, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x22, 0x87, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66,
0x72, 0x6f, 0x74, 0x6f, 0x22, 0xbb, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66,
0x6f, 0x12, 0x29, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76,
0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a,
@ -314,28 +324,32 @@ var file_tvix_store_protos_pathinfo_proto_rawDesc = []byte{
0x52, 0x0a, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x30, 0x0a, 0x07,
0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e,
0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41,
0x52, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0x37,
0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6e,
0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12,
0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52,
0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0xe3, 0x01, 0x0a, 0x07, 0x4e, 0x41, 0x52, 0x49,
0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18,
0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d,
0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01,
0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35, 0x36, 0x12, 0x40, 0x0a,
0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28,
0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76,
0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74,
0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12,
0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d,
0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65,
0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x1a, 0x33, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e,
0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74,
0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x42, 0x28, 0x5a,
0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76,
0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b,
0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x52, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x32,
0x0a, 0x07, 0x64, 0x65, 0x72, 0x69, 0x76, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e,
0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x52, 0x07, 0x64, 0x65, 0x72, 0x69, 0x76,
0x65, 0x72, 0x22, 0x37, 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12,
0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20,
0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0xe3, 0x01, 0x0a, 0x07,
0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73,
0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69,
0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36,
0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35,
0x36, 0x12, 0x40, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18,
0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f,
0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x53, 0x69,
0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75,
0x72, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65,
0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65,
0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x1a, 0x33, 0x0a, 0x09,
0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d,
0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a,
0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74,
0x61, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79,
0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x33,
}
var (
@ -361,12 +375,13 @@ var file_tvix_store_protos_pathinfo_proto_goTypes = []interface{}{
var file_tvix_store_protos_pathinfo_proto_depIdxs = []int32{
4, // 0: tvix.store.v1.PathInfo.node:type_name -> tvix.castore.v1.Node
2, // 1: tvix.store.v1.PathInfo.narinfo:type_name -> tvix.store.v1.NARInfo
3, // 2: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature
3, // [3:3] is the sub-list for method output_type
3, // [3:3] is the sub-list for method input_type
3, // [3:3] is the sub-list for extension type_name
3, // [3:3] is the sub-list for extension extendee
0, // [0:3] is the sub-list for field type_name
1, // 2: tvix.store.v1.PathInfo.deriver:type_name -> tvix.store.v1.StorePath
3, // 3: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature
4, // [4:4] is the sub-list for method output_type
4, // [4:4] is the sub-list for method input_type
4, // [4:4] is the sub-list for extension type_name
4, // [4:4] is the sub-list for extension extendee
0, // [0:4] is the sub-list for field type_name
}
func init() { file_tvix_store_protos_pathinfo_proto_init() }

4
tvix/store/protos/pathinfo.proto
View file

@ -21,6 +21,10 @@ message PathInfo {
// see below.
NARInfo narinfo = 3;
// The StorePath of the .drv file producing this output.
// The .drv suffix is omitted in its `name` field.
StorePath deriver = 4;
}
// Represents a path in the Nix store (a direct child of STORE_DIR).

26
tvix/store/protos/pathinfo_test.go
View file

@ -120,4 +120,30 @@ func TestValidate(t *testing.T) {
_, err := pi.Validate()
assert.Error(t, err, "must not validate")
})
t.Run("happy deriver", func(t *testing.T) {
pi := genPathInfoSymlink()
// add the Deriver Field.
pi.Deriver = &storev1pb.StorePath{
Digest: exampleStorePathDigest,
Name: "foo",
}
_, err := pi.Validate()
assert.NoError(t, err, "must validate")
})
t.Run("invalid deriver", func(t *testing.T) {
pi := genPathInfoSymlink()
// add the Deriver Field, with a broken digest
pi.Deriver = &storev1pb.StorePath{
Digest: []byte{},
Name: "foo2",
}
_, err := pi.Validate()
assert.Error(t, err, "must not validate")
})
}

1
tvix/store/src/bin/tvix-store.rs
View file

@ -296,6 +296,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
signatures: vec![],
reference_names: vec![],
}),
deriver: None,
};
// put into [PathInfoService], and return the PathInfo that we get back

16
tvix/store/src/proto/mod.rs
View file

@ -48,7 +48,7 @@ pub enum ValidatePathInfoError {
#[error("Inconsistent Number of References: {0} (references) vs {1} (narinfo)")]
InconsistentNumberOfReferences(usize, usize),
/// A string in narinfo.reference_names does not parse to a StorePath.
/// A string in narinfo.reference_names does not parse to a [store_path::StorePath].
#[error("Invalid reference_name at position {0}: {1}")]
InvalidNarinfoReferenceName(usize, String),
@ -60,6 +60,10 @@ pub enum ValidatePathInfoError {
[u8; store_path::DIGEST_SIZE],
[u8; store_path::DIGEST_SIZE],
),
/// The deriver field is invalid.
#[error("deriver field is invalid: {0}")]
InvalidDeriverField(store_path::Error),
}
/// Parses a root node name.
@ -152,6 +156,16 @@ impl PathInfo {
}
};
// If the Deriver field is populated, ensure it parses to a
// [store_path::StorePath].
// We can't check for it to *not* end with .drv, as the .drv files produced by
// recursive Nix end with multiple .drv suffixes, and only one is popped when
// converting to this field.
if let Some(deriver) = &self.deriver {
store_path::StorePath::from_name_and_digest(deriver.name.clone(), &deriver.digest)
.map_err(ValidatePathInfoError::InvalidDeriverField)?;
}
// return the root nix path
Ok(root_nix_path)
}

31
tvix/store/src/proto/tests/pathinfo.rs
View file

@ -262,3 +262,34 @@ fn validate_symlink_target_null_byte_invalid() {
node.validate().expect_err("must fail validation");
}
/// Create a PathInfo with a correct deriver field and ensure it succeeds.
#[test]
fn validate_valid_deriver() {
let mut path_info = PATH_INFO_WITHOUT_NARINFO.clone();
// add a valid deriver
path_info.deriver = Some(crate::proto::StorePath {
name: "foo".to_string(),
digest: DUMMY_OUTPUT_HASH.clone(),
});
path_info.validate().expect("must validate");
}
/// Create a PathInfo with a broken deriver field and ensure it fails.
#[test]
fn validate_invalid_deriver() {
let mut path_info = PATH_INFO_WITHOUT_NARINFO.clone();
// add a broken deriver (invalid digest)
path_info.deriver = Some(crate::proto::StorePath {
name: "foo".to_string(),
digest: vec![].into(),
});
match path_info.validate().expect_err("must fail validation") {
ValidatePathInfoError::InvalidDeriverField(_) => {}
e => panic!("unexpected error: {:?}", e),
}
}

1
tvix/store/src/tests/fixtures.rs
View file

@ -109,6 +109,7 @@ lazy_static! {
}),
references: vec![DUMMY_OUTPUT_HASH.clone()],
narinfo: None,
deriver: None,
};
/// A PathInfo message with .narinfo populated.