mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix/nix-compat): make NixHash an enum with fixed-len bytes

Browse source 
Less Vec<u8> passed around.

Change-Id: Ie153a6bfaa084d7490ffa38634efdf5f3c31a768
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9722
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2023-10-14 17:48:16 +01:00 committed by clbot
parent 786b0324a9
commit 4ae0f428bd
8 changed files with 226 additions and 220 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
tvix/nix-compat/src/derivation/mod.rs
View file

@ -168,7 +168,8 @@ impl Derivation {
// This is not the [NixHash::to_nix_hash_string], but without the sha256: prefix). // This is not the [NixHash::to_nix_hash_string], but without the sha256: prefix).
for (drv_path, output_names) in &self.input_derivations { for (drv_path, output_names) in &self.input_derivations {
replaced_input_derivations.insert( replaced_input_derivations.insert(
data_encoding::HEXLOWER.encode(&fn_get_derivation_or_fod_hash(drv_path).digest), data_encoding::HEXLOWER
.encode(fn_get_derivation_or_fod_hash(drv_path).digest_as_bytes()),
output_names.clone(), output_names.clone(),
); );
} }
@ -186,12 +187,7 @@ impl Derivation {
hasher.finalize().to_vec() hasher.finalize().to_vec()
}); });
// We populate the struct directly, as we know the sha256 digest has the NixHash::Sha256(digest.try_into().unwrap())
// right size.
NixHash {
algo: crate::nixhash::HashAlgo::Sha256,
digest: digest.to_vec(),
}
} }
/// This calculates all output paths of a Derivation and updates the struct. /// This calculates all output paths of a Derivation and updates the struct.

4
tvix/nix-compat/src/derivation/output.rs
View file

@ -44,8 +44,8 @@ impl Output {
if let Some(hash) = &self.hash_with_mode { if let Some(hash) = &self.hash_with_mode {
match hash { match hash {
NixHashWithMode::Flat(h) | NixHashWithMode::Recursive(h) => { NixHashWithMode::Flat(h) | NixHashWithMode::Recursive(h) => {
if h.algo != HashAlgo::Sha1 || h.algo != HashAlgo::Sha256 { if h.algo() != HashAlgo::Sha1 || h.algo() != HashAlgo::Sha256 {
return Err(OutputError::InvalidHashAlgo(h.algo.to_string())); return Err(OutputError::InvalidHashAlgo(h.algo().to_string()));
} }
} }
} }

7
tvix/nix-compat/src/derivation/tests/mod.rs
View file

@ -238,17 +238,16 @@ fn output_path_construction() {
Output { Output {
path: "".to_string(), // will be calculated path: "".to_string(), // will be calculated
hash_with_mode: Some(crate::nixhash::NixHashWithMode::Recursive( hash_with_mode: Some(crate::nixhash::NixHashWithMode::Recursive(
( crate::nixhash::from_algo_and_digest(
crate::nixhash::HashAlgo::Sha256, crate::nixhash::HashAlgo::Sha256,
data_encoding::HEXLOWER &data_encoding::HEXLOWER
.decode( .decode(
"08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba" "08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba"
.as_bytes(), .as_bytes(),
) )
.unwrap(), .unwrap(),
) )
.try_into() .unwrap(),
.unwrap(),
)), )),
}, },
); );

24
tvix/nix-compat/src/derivation/write.rs
View file

@ -79,22 +79,20 @@ pub fn write_outputs(
let mut elements: Vec<&str> = vec![output_name, &output.path]; let mut elements: Vec<&str> = vec![output_name, &output.path];
let (e2, e3) = match &output.hash_with_mode { let (mode_and_algo, digest) = match &output.hash_with_mode {
Some(hash) => match hash { Some(crate::nixhash::NixHashWithMode::Flat(h)) => (
crate::nixhash::NixHashWithMode::Flat(h) => ( h.algo().to_string(),
h.algo.to_string(), data_encoding::HEXLOWER.encode(h.digest_as_bytes()),
data_encoding::HEXLOWER.encode(&h.digest), ),
), Some(crate::nixhash::NixHashWithMode::Recursive(h)) => (
crate::nixhash::NixHashWithMode::Recursive(h) => ( format!("r:{}", h.algo()),
format!("r:{}", h.algo), data_encoding::HEXLOWER.encode(h.digest_as_bytes()),
data_encoding::HEXLOWER.encode(&h.digest), ),
),
},
None => ("".to_string(), "".to_string()), None => ("".to_string(), "".to_string()),
}; };
elements.push(&e2); elements.push(&mode_and_algo);
elements.push(&e3); elements.push(&digest);
write_array_elements(writer, &elements)?; write_array_elements(writer, &elements)?;

12
tvix/nix-compat/src/nixhash/algos.rs
View file

@ -13,6 +13,18 @@ pub enum HashAlgo {
Sha512, Sha512,
} }
impl HashAlgo {
// return the number of bytes in the digest of the given hash algo.
pub fn digest_length(&self) -> usize {
match self {
HashAlgo::Sha1 => 20,
HashAlgo::Sha256 => 32,
HashAlgo::Sha512 => 64,
HashAlgo::Md5 => 16,
}
}
}
impl Display for HashAlgo { impl Display for HashAlgo {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match &self { match &self {

276
tvix/nix-compat/src/nixhash/mod.rs
View file

@ -8,38 +8,75 @@ mod with_mode;
pub use algos::HashAlgo; pub use algos::HashAlgo;
pub use with_mode::NixHashWithMode; pub use with_mode::NixHashWithMode;
/// Nix allows specifying hashes in various encodings, and magically just /// NixHash represents hashes known by Nix.
/// derives the encoding.
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
pub struct NixHash { pub enum NixHash {
pub digest: Vec<u8>, Md5([u8; 16]),
Sha1([u8; 20]),
pub algo: HashAlgo, Sha256([u8; 32]),
Sha512([u8; 64]),
} }
impl NixHash { impl NixHash {
/// returns the algo as [HashAlgo].
pub fn algo(&self) -> HashAlgo {
match self {
NixHash::Md5(_) => HashAlgo::Md5,
NixHash::Sha1(_) => HashAlgo::Sha1,
NixHash::Sha256(_) => HashAlgo::Sha256,
NixHash::Sha512(_) => HashAlgo::Sha512,
}
}
/// returns the digest as variable-length byte slice.
pub fn digest_as_bytes(&self) -> &[u8] {
match self {
NixHash::Md5(digest) => digest,
NixHash::Sha1(digest) => digest,
NixHash::Sha256(digest) => digest,
NixHash::Sha512(digest) => digest,
}
}
/// Formats a [NixHash] in the Nix default hash format, /// Formats a [NixHash] in the Nix default hash format,
/// which is the algo, followed by a colon, then the lower hex encoded digest. /// which is the algo, followed by a colon, then the lower hex encoded digest.
pub fn to_nix_hash_string(&self) -> String { pub fn to_nix_hash_string(&self) -> String {
format!("{}:{}", self.algo, HEXLOWER.encode(&self.digest)) format!(
"{}:{}",
self.algo(),
HEXLOWER.encode(self.digest_as_bytes())
)
} }
} }
impl TryFrom<(HashAlgo, Vec<u8>)> for NixHash { impl TryFrom<(HashAlgo, &[u8])> for NixHash {
type Error = Error; type Error = Error;
/// Constructs a new [NixHash] by specifying [HashAlgo] and digest. /// Constructs a new [NixHash] by specifying [HashAlgo] and digest.
// It can fail if the passed digest length doesn't match what's expected for /// It can fail if the passed digest length doesn't match what's expected for
// the passed algo. /// the passed algo.
fn try_from(value: (HashAlgo, Vec<u8>)) -> Result<Self, Self::Error> { fn try_from(value: (HashAlgo, &[u8])) -> Result<Self, Self::Error> {
let (algo, digest) = value; let (algo, digest) = value;
if digest.len() != hash_algo_length(&algo) { from_algo_and_digest(algo, digest)
return Err(Error::InvalidEncodedDigestLength(digest.len(), algo));
}
Ok(Self { algo, digest })
} }
} }
/// Constructs a new [NixHash] by specifying [HashAlgo] and digest.
// It can fail if the passed digest length doesn't match what's expected for
// the passed algo.
pub fn from_algo_and_digest(algo: HashAlgo, digest: &[u8]) -> Result<NixHash, Error> {
if digest.len() != algo.digest_length() {
return Err(Error::InvalidEncodedDigestLength(digest.len(), algo));
}
Ok(match algo {
HashAlgo::Md5 => NixHash::Md5(digest.try_into().unwrap()),
HashAlgo::Sha1 => NixHash::Sha1(digest.try_into().unwrap()),
HashAlgo::Sha256 => NixHash::Sha256(digest.try_into().unwrap()),
HashAlgo::Sha512 => NixHash::Sha512(digest.try_into().unwrap()),
})
}
/// Errors related to NixHash construction. /// Errors related to NixHash construction.
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub enum Error { pub enum Error {
@ -56,12 +93,14 @@ pub enum Error {
#[error("invalid base64 encoding: {0}")] #[error("invalid base64 encoding: {0}")]
InvalidBase64Encoding(data_encoding::DecodeError), InvalidBase64Encoding(data_encoding::DecodeError),
#[error("conflicting hash algo: {0} (hash_algo) vs {1} (inline)")] #[error("conflicting hash algo: {0} (hash_algo) vs {1} (inline)")]
ConflictingHashAlgos(String, String), ConflictingHashAlgos(HashAlgo, HashAlgo),
#[error("missing inline hash algo, but no externally-specified algo: {0}")] #[error("missing inline hash algo, but no externally-specified algo: {0}")]
MissingInlineHashAlgo(String), MissingInlineHashAlgo(String),
} }
/// parses a string to a nix hash. /// Nix allows specifying hashes in various encodings, and magically just
/// derives the encoding.
/// This function parses strings to a NixHash.
/// ///
/// Hashes can be: /// Hashes can be:
/// - Nix hash strings /// - Nix hash strings
@ -79,14 +118,11 @@ pub enum Error {
/// case of a nix hash string or SRI), in which it needs to be consistent with the /// case of a nix hash string or SRI), in which it needs to be consistent with the
/// one communicated out-of-band. /// one communicated out-of-band.
pub fn from_str(s: &str, algo_str: Option<&str>) -> Result<NixHash, Error> { pub fn from_str(s: &str, algo_str: Option<&str>) -> Result<NixHash, Error> {
// validate algo_str, construct hash_algo // if algo_str is some, parse or bail out
let algo: Option<HashAlgo> = match &algo_str { let algo: Option<HashAlgo> = if let Some(algo_str) = algo_str {
Some("sha1") => Some(HashAlgo::Sha1), Some(algo_str.try_into()?)
Some("sha256") => Some(HashAlgo::Sha256), } else {
Some("sha512") => Some(HashAlgo::Sha512), None
Some("md5") => Some(HashAlgo::Md5),
Some(e) => return Err(Error::InvalidAlgo(e.to_string())),
None => None,
}; };
// Peek at the beginning of the string to detect SRI hashes. // Peek at the beginning of the string to detect SRI hashes.
@ -96,13 +132,11 @@ pub fn from_str(s: &str, algo_str: Option<&str>) -> Result<NixHash, Error> {
|| s.starts_with("md5-") || s.starts_with("md5-")
{ {
let parsed_nixhash = from_sri_str(s)?; let parsed_nixhash = from_sri_str(s)?;
// ensure the algo matches with what has been passed externally, if so. // ensure the algo matches with what has been passed externally, if so.
if let Some(algo) = algo { if let Some(algo) = algo {
if algo != parsed_nixhash.algo { if algo != parsed_nixhash.algo() {
return Err(Error::ConflictingHashAlgos( return Err(Error::ConflictingHashAlgos(algo, parsed_nixhash.algo()));
algo.to_string(),
parsed_nixhash.algo.to_string(),
));
} }
} }
return Ok(parsed_nixhash); return Ok(parsed_nixhash);
@ -117,11 +151,8 @@ pub fn from_str(s: &str, algo_str: Option<&str>) -> Result<NixHash, Error> {
let parsed_nixhash = from_nix_str(s)?; let parsed_nixhash = from_nix_str(s)?;
// ensure the algo matches with what has been passed externally, if so. // ensure the algo matches with what has been passed externally, if so.
if let Some(algo) = algo { if let Some(algo) = algo {
if algo != parsed_nixhash.algo { if algo != parsed_nixhash.algo() {
return Err(Error::ConflictingHashAlgos( return Err(Error::ConflictingHashAlgos(algo, parsed_nixhash.algo()));
algo.to_string(),
parsed_nixhash.algo.to_string(),
));
} }
} }
return Ok(parsed_nixhash); return Ok(parsed_nixhash);
@ -131,20 +162,20 @@ pub fn from_str(s: &str, algo_str: Option<&str>) -> Result<NixHash, Error> {
match algo { match algo {
// Fail if there isn't. // Fail if there isn't.
None => Err(Error::MissingInlineHashAlgo(s.to_string())), None => Err(Error::MissingInlineHashAlgo(s.to_string())),
Some(algo) => decode_digest(s, algo), Some(algo) => decode_digest(s.as_bytes(), algo),
} }
} }
/// Parses a Nix hash string ($algo:$digest) to a NixHash. /// Parses a Nix hash string ($algo:$digest) to a NixHash.
pub fn from_nix_str(s: &str) -> Result<NixHash, Error> { pub fn from_nix_str(s: &str) -> Result<NixHash, Error> {
if let Some(rest) = s.strip_prefix("sha1:") { if let Some(rest) = s.strip_prefix("sha1:") {
decode_digest(rest, HashAlgo::Sha1) decode_digest(rest.as_bytes(), HashAlgo::Sha1)
} else if let Some(rest) = s.strip_prefix("sha256:") { } else if let Some(rest) = s.strip_prefix("sha256:") {
decode_digest(rest, HashAlgo::Sha256) decode_digest(rest.as_bytes(), HashAlgo::Sha256)
} else if let Some(rest) = s.strip_prefix("sha512:") { } else if let Some(rest) = s.strip_prefix("sha512:") {
decode_digest(rest, HashAlgo::Sha512) decode_digest(rest.as_bytes(), HashAlgo::Sha512)
} else if let Some(rest) = s.strip_prefix("md5:") { } else if let Some(rest) = s.strip_prefix("md5:") {
decode_digest(rest, HashAlgo::Md5) decode_digest(rest.as_bytes(), HashAlgo::Md5)
} else { } else {
Err(Error::InvalidAlgo(s.to_string())) Err(Error::InvalidAlgo(s.to_string()))
} }
@ -170,68 +201,52 @@ pub fn from_sri_str(s: &str) -> Result<NixHash, Error> {
// the rest should be the digest (as Nix doesn't support more than one hash in an SRI string). // the rest should be the digest (as Nix doesn't support more than one hash in an SRI string).
let encoded_digest = &s[idx + 1..]; let encoded_digest = &s[idx + 1..];
let actual_len = encoded_digest.as_bytes().len();
// verify the digest length matches what we'd expect from the hash function, // decode the digest and algo into a [NixHash]
// and then either try decoding as BASE64 or BASE64_NOPAD. match decode_digest(encoded_digest.as_bytes(), algo) {
// This will also reject SRI strings with more than one hash, because the length won't match // If decoding was successful, pass along
if actual_len == BASE64.encode_len(hash_algo_length(&algo)) { Ok(nixhash) => Ok(nixhash),
let digest: Vec<u8> = BASE64 // For SRI hashes (only), BASE64_NOPAD is also tolerated,
.decode(encoded_digest.as_bytes()) // so try to parse for this, too.
.map_err(Error::InvalidBase64Encoding)?;
Ok(NixHash { digest, algo })
} else if actual_len == BASE64_NOPAD.encode_len(hash_algo_length(&algo)) {
let digest: Vec<u8> = BASE64_NOPAD
.decode(encoded_digest.as_bytes())
.map_err(Error::InvalidBase64Encoding)?;
Ok(NixHash { digest, algo })
} else {
// NOTE: As of now, we reject SRI hashes containing additional // NOTE: As of now, we reject SRI hashes containing additional
// characters (which upstream Nix seems to simply truncate), as // characters (which upstream Nix seems to simply truncate), as
// there's no occurence of this is in nixpkgs. // there's no occurence of this is in nixpkgs.
// It most likely should also be a bug in Nix. // It most likely should also be a bug in Nix.
Err(Error::InvalidEncodedDigestLength( Err(Error::InvalidEncodedDigestLength(digest_len, hash_algo)) => {
encoded_digest.as_bytes().len(), if encoded_digest.len() == BASE64_NOPAD.encode_len(algo.digest_length()) {
algo, let digest = BASE64_NOPAD
)) .decode(encoded_digest.as_bytes())
.map_err(Error::InvalidBase64Encoding)?;
Ok(from_algo_and_digest(algo, &digest).unwrap())
} else {
Err(Error::InvalidEncodedDigestLength(digest_len, hash_algo))?
}
}
Err(e) => Err(e)?,
} }
} }
/// Decode a plain digest depending on the hash algo specified externally. /// Decode a plain digest depending on the hash algo specified externally.
/// hexlower, nixbase32 and base64 encodings are supported - the encoding is /// hexlower, nixbase32 and base64 encodings are supported - the encoding is
/// inferred from the input length. /// inferred from the input length.
fn decode_digest(s: &str, algo: HashAlgo) -> Result<NixHash, Error> { fn decode_digest(s: &[u8], algo: HashAlgo) -> Result<NixHash, Error> {
// for the chosen hash algo, calculate the expected (decoded) digest length // for the chosen hash algo, calculate the expected (decoded) digest length
// (as bytes) // (as bytes)
let expected_digest_len = hash_algo_length(&algo); let digest = if s.len() == HEXLOWER.encode_len(algo.digest_length()) {
HEXLOWER
.decode(s.as_ref())
.map_err(Error::InvalidBase16Encoding)?
} else if s.len() == nixbase32::encode_len(algo.digest_length()) {
nixbase32::decode(s).map_err(Error::InvalidBase32Encoding)?
} else if s.len() == BASE64.encode_len(algo.digest_length()) {
BASE64
.decode(s.as_ref())
.map_err(Error::InvalidBase64Encoding)?
} else {
Err(Error::InvalidEncodedDigestLength(s.len(), algo))?
};
Ok(NixHash { Ok(from_algo_and_digest(algo, &digest).unwrap())
digest: match s.len() {
n if n == data_encoding::HEXLOWER.encode_len(expected_digest_len) => {
data_encoding::HEXLOWER
.decode(s.as_ref())
.map_err(Error::InvalidBase16Encoding)
}
n if n == nixbase32::encode_len(expected_digest_len) => {
nixbase32::decode(s.as_ref()).map_err(Error::InvalidBase32Encoding)
}
n if n == BASE64.encode_len(expected_digest_len) => BASE64
.decode(s.as_ref())
.map_err(Error::InvalidBase64Encoding),
_ => return Err(Error::InvalidEncodedDigestLength(s.len(), algo)),
}?,
algo,
})
}
// return the number of bytes in the digest of the given hash algo.
fn hash_algo_length(hash_algo: &HashAlgo) -> usize {
match hash_algo {
HashAlgo::Sha1 => 20,
HashAlgo::Sha256 => 32,
HashAlgo::Sha512 => 64,
HashAlgo::Md5 => 16,
}
} }
#[cfg(test)] #[cfg(test)]
@ -240,32 +255,33 @@ mod tests {
nixbase32, nixbase32,
nixhash::{self, HashAlgo, NixHash}, nixhash::{self, HashAlgo, NixHash},
}; };
use data_encoding::{BASE64, BASE64_NOPAD, HEXLOWER};
use test_case::test_case; use test_case::test_case;
const DIGEST_SHA1: &[u8] = &[ const DIGEST_SHA1: [u8; 20] = [
0x60, 0x16, 0x77, 0x79, 0x97, 0xc3, 0x0a, 0xb0, 0x24, 0x13, 0xcf, 0x50, 0x95, 0x62, 0x2c, 0x60, 0x16, 0x77, 0x79, 0x97, 0xc3, 0x0a, 0xb0, 0x24, 0x13, 0xcf, 0x50, 0x95, 0x62, 0x2c,
0xd7, 0x92, 0x42, 0x83, 0xac, 0xd7, 0x92, 0x42, 0x83, 0xac,
]; ];
const DIGEST_SHA256: &[u8] = &[ const DIGEST_SHA256: [u8; 32] = [
0xa5, 0xce, 0x9c, 0x15, 0x5e, 0xd0, 0x93, 0x97, 0x61, 0x46, 0x46, 0xc9, 0x71, 0x7f, 0xc7, 0xa5, 0xce, 0x9c, 0x15, 0x5e, 0xd0, 0x93, 0x97, 0x61, 0x46, 0x46, 0xc9, 0x71, 0x7f, 0xc7,
0xcd, 0x94, 0xb1, 0x02, 0x3d, 0x7b, 0x76, 0xb6, 0x18, 0xd4, 0x09, 0xe4, 0xfe, 0xfd, 0x6e, 0xcd, 0x94, 0xb1, 0x02, 0x3d, 0x7b, 0x76, 0xb6, 0x18, 0xd4, 0x09, 0xe4, 0xfe, 0xfd, 0x6e,
0x9d, 0x39, 0x9d, 0x39,
]; ];
const DIGEST_SHA512: &[u8] = &[ const DIGEST_SHA512: [u8; 64] = [
0xab, 0x40, 0xd0, 0xbe, 0x35, 0x41, 0xf0, 0x77, 0x4b, 0xba, 0x78, 0x15, 0xd1, 0x3d, 0x10, 0xab, 0x40, 0xd0, 0xbe, 0x35, 0x41, 0xf0, 0x77, 0x4b, 0xba, 0x78, 0x15, 0xd1, 0x3d, 0x10,
0xb0, 0x32, 0x52, 0xe9, 0x6e, 0x95, 0xf7, 0xdb, 0xb4, 0xee, 0x99, 0xa3, 0xb4, 0x31, 0xc2, 0xb0, 0x32, 0x52, 0xe9, 0x6e, 0x95, 0xf7, 0xdb, 0xb4, 0xee, 0x99, 0xa3, 0xb4, 0x31, 0xc2,
0x16, 0x62, 0xfd, 0x69, 0x71, 0xa0, 0x20, 0x16, 0x0e, 0x39, 0x84, 0x8a, 0xa5, 0xf3, 0x05, 0x16, 0x62, 0xfd, 0x69, 0x71, 0xa0, 0x20, 0x16, 0x0e, 0x39, 0x84, 0x8a, 0xa5, 0xf3, 0x05,
0xb9, 0xbe, 0x0f, 0x78, 0x72, 0x7b, 0x2b, 0x07, 0x89, 0xe3, 0x9f, 0x12, 0x4d, 0x21, 0xe9, 0xb9, 0xbe, 0x0f, 0x78, 0x72, 0x7b, 0x2b, 0x07, 0x89, 0xe3, 0x9f, 0x12, 0x4d, 0x21, 0xe9,
0x2b, 0x8f, 0x39, 0xef, 0x2b, 0x8f, 0x39, 0xef,
]; ];
const DIGEST_MD5: &[u8] = &[ const DIGEST_MD5: [u8; 16] = [
0xc4, 0x87, 0x4a, 0x88, 0x97, 0x44, 0x0b, 0x39, 0x3d, 0x86, 0x2d, 0x8f, 0xd4, 0x59, 0x07, 0xc4, 0x87, 0x4a, 0x88, 0x97, 0x44, 0x0b, 0x39, 0x3d, 0x86, 0x2d, 0x8f, 0xd4, 0x59, 0x07,
0x3f, 0x3f,
]; ];
fn to_base16(digest: &[u8]) -> String { fn to_base16(digest: &[u8]) -> String {
data_encoding::HEXLOWER.encode(digest) HEXLOWER.encode(digest)
} }
fn to_nixbase32(digest: &[u8]) -> String { fn to_nixbase32(digest: &[u8]) -> String {
@ -273,11 +289,11 @@ mod tests {
} }
fn to_base64(digest: &[u8]) -> String { fn to_base64(digest: &[u8]) -> String {
data_encoding::BASE64.encode(digest) BASE64.encode(digest)
} }
fn to_base64_nopad(digest: &[u8]) -> String { fn to_base64_nopad(digest: &[u8]) -> String {
data_encoding::BASE64_NOPAD.encode(digest) BASE64_NOPAD.encode(digest)
} }
// TODO // TODO
@ -289,36 +305,35 @@ mod tests {
} }
/// Test parsing a hash string in various formats, and also when/how the out-of-band algo is needed. /// Test parsing a hash string in various formats, and also when/how the out-of-band algo is needed.
#[test_case(DIGEST_SHA1, HashAlgo::Sha1; "sha1")] #[test_case(&NixHash::Sha1(DIGEST_SHA1); "sha1")]
#[test_case(DIGEST_SHA256, HashAlgo::Sha256; "sha256")] #[test_case(&NixHash::Sha256(DIGEST_SHA256); "sha256")]
#[test_case(DIGEST_SHA512, HashAlgo::Sha512; "sha512")] #[test_case(&NixHash::Sha512(DIGEST_SHA512); "sha512")]
#[test_case(DIGEST_MD5, HashAlgo::Md5; "md5")] #[test_case(&NixHash::Md5(DIGEST_MD5); "md5")]
fn from_str(digest: &[u8], algo: HashAlgo) { fn from_str(expected_hash: &NixHash) {
let expected_hash = NixHash { let algo = &expected_hash.algo();
digest: digest.to_vec(), let digest = expected_hash.digest_as_bytes();
algo,
};
// parse SRI // parse SRI
{ {
// base64 without out-of-band algo // base64 without out-of-band algo
let s = make_sri_string(&algo, to_base64(digest)); let s = make_sri_string(algo, to_base64(digest));
let h = nixhash::from_str(&s, None).expect("must succeed"); let h = nixhash::from_str(&s, None).expect("must succeed");
assert_eq!(expected_hash, h); assert_eq!(expected_hash, &h);
// base64 with out-of-band-algo // base64 with out-of-band-algo
let s = make_sri_string(&algo, to_base64(digest)); let s = make_sri_string(algo, to_base64(digest));
let h = nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed"); let h = nixhash::from_str(&s, Some(&expected_hash.algo().to_string()))
assert_eq!(expected_hash, h); .expect("must succeed");
assert_eq!(expected_hash, &h);
// base64_nopad without out-of-band algo // base64_nopad without out-of-band algo
let s = make_sri_string(&algo, to_base64_nopad(digest)); let s = make_sri_string(algo, to_base64_nopad(digest));
let h = nixhash::from_str(&s, None).expect("must succeed"); let h = nixhash::from_str(&s, None).expect("must succeed");
assert_eq!(expected_hash, h); assert_eq!(expected_hash, &h);
// base64_nopad with out-of-band-algo // base64_nopad with out-of-band-algo
let s = make_sri_string(&algo, to_base64_nopad(digest)); let s = make_sri_string(algo, to_base64_nopad(digest));
let h = nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed"); let h = nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed");
assert_eq!(expected_hash, h); assert_eq!(expected_hash, &h);
} }
// parse plain base16. should succeed with algo out-of-band, but fail without. // parse plain base16. should succeed with algo out-of-band, but fail without.
@ -326,7 +341,7 @@ mod tests {
let s = to_base16(digest); let s = to_base16(digest);
nixhash::from_str(&s, None).expect_err("must fail"); nixhash::from_str(&s, None).expect_err("must fail");
let h = nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed"); let h = nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed");
assert_eq!(expected_hash, h); assert_eq!(expected_hash, &h);
} }
// parse plain nixbase32. should succeed with algo out-of-band, but fail without. // parse plain nixbase32. should succeed with algo out-of-band, but fail without.
@ -334,7 +349,7 @@ mod tests {
let s = to_nixbase32(digest); let s = to_nixbase32(digest);
nixhash::from_str(&s, None).expect_err("must fail"); nixhash::from_str(&s, None).expect_err("must fail");
let h = nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed"); let h = nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed");
assert_eq!(expected_hash, h); assert_eq!(expected_hash, &h);
} }
// parse plain base64. should succeed with algo out-of-band, but fail without. // parse plain base64. should succeed with algo out-of-band, but fail without.
@ -342,45 +357,45 @@ mod tests {
let s = to_base64(digest); let s = to_base64(digest);
nixhash::from_str(&s, None).expect_err("must fail"); nixhash::from_str(&s, None).expect_err("must fail");
let h = nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed"); let h = nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed");
assert_eq!(expected_hash, h); assert_eq!(expected_hash, &h);
} }
// parse Nix hash strings // parse Nix hash strings
{ {
// base16. should succeed with both algo out-of-band and in-band. // base16. should succeed with both algo out-of-band and in-band.
{ {
let s = make_nixhash(&algo, to_base16(digest)); let s = make_nixhash(algo, to_base16(digest));
assert_eq!( assert_eq!(
expected_hash, expected_hash,
nixhash::from_str(&s, None).expect("must succeed") &nixhash::from_str(&s, None).expect("must succeed")
); );
assert_eq!( assert_eq!(
expected_hash, expected_hash,
nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed") &nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed")
); );
} }
// nixbase32. should succeed with both algo out-of-band and in-band. // nixbase32. should succeed with both algo out-of-band and in-band.
{ {
let s = make_nixhash(&algo, to_nixbase32(digest)); let s = make_nixhash(algo, to_nixbase32(digest));
assert_eq!( assert_eq!(
expected_hash, expected_hash,
nixhash::from_str(&s, None).expect("must succeed") &nixhash::from_str(&s, None).expect("must succeed")
); );
assert_eq!( assert_eq!(
expected_hash, expected_hash,
nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed") &nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed")
); );
} }
// base64. should succeed with both algo out-of-band and in-band. // base64. should succeed with both algo out-of-band and in-band.
{ {
let s = make_nixhash(&algo, to_base64(digest)); let s = make_nixhash(algo, to_base64(digest));
assert_eq!( assert_eq!(
expected_hash, expected_hash,
nixhash::from_str(&s, None).expect("must succeed") &nixhash::from_str(&s, None).expect("must succeed")
); );
assert_eq!( assert_eq!(
expected_hash, expected_hash,
nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed") &nixhash::from_str(&s, Some(&algo.to_string())).expect("must succeed")
); );
} }
} }
@ -392,14 +407,15 @@ mod tests {
let nix_hash = nixhash::from_sri_str("sha256-pc6cFV7Qk5dhRkbJcX/HzZSxAj17drYY1Ank/v1unTk=") let nix_hash = nixhash::from_sri_str("sha256-pc6cFV7Qk5dhRkbJcX/HzZSxAj17drYY1Ank/v1unTk=")
.expect("must succeed"); .expect("must succeed");
assert_eq!(HashAlgo::Sha256, nix_hash.algo); assert_eq!(HashAlgo::Sha256, nix_hash.algo());
assert_eq!( assert_eq!(
vec![ vec![
0xa5, 0xce, 0x9c, 0x15, 0x5e, 0xd0, 0x93, 0x97, 0x61, 0x46, 0x46, 0xc9, 0x71, 0x7f, 0xa5, 0xce, 0x9c, 0x15, 0x5e, 0xd0, 0x93, 0x97, 0x61, 0x46, 0x46, 0xc9, 0x71, 0x7f,
0xc7, 0xcd, 0x94, 0xb1, 0x02, 0x3d, 0x7b, 0x76, 0xb6, 0x18, 0xd4, 0x09, 0xe4, 0xfe, 0xc7, 0xcd, 0x94, 0xb1, 0x02, 0x3d, 0x7b, 0x76, 0xb6, 0x18, 0xd4, 0x09, 0xe4, 0xfe,
0xfd, 0x6e, 0x9d, 0x39 0xfd, 0x6e, 0x9d, 0x39
], ]
nix_hash.digest .as_slice(),
nix_hash.digest_as_bytes()
) )
} }
@ -453,12 +469,12 @@ mod tests {
// passing hash algo out of band should succeed // passing hash algo out of band should succeed
let nix_hash = nixhash::from_str(&format!("sha256-{}", &broken_base64), Some("sha256")) let nix_hash = nixhash::from_str(&format!("sha256-{}", &broken_base64), Some("sha256"))
.expect("must succeed"); .expect("must succeed");
assert_eq!(&expected_digest, &nix_hash.digest); assert_eq!(&expected_digest, &nix_hash.digest_as_bytes());
// not passing hash algo out of band should succeed // not passing hash algo out of band should succeed
let nix_hash = let nix_hash =
nixhash::from_str(&format!("sha256-{}", &broken_base64), None).expect("must succeed"); nixhash::from_str(&format!("sha256-{}", &broken_base64), None).expect("must succeed");
assert_eq!(&expected_digest, &nix_hash.digest); assert_eq!(&expected_digest, &nix_hash.digest_as_bytes());
// not passing SRI, but hash algo out of band should fail // not passing SRI, but hash algo out of band should fail
nixhash::from_str(broken_base64, Some("sha256")).expect_err("must fail"); nixhash::from_str(broken_base64, Some("sha256")).expect_err("must fail");

43
tvix/nix-compat/src/nixhash/with_mode.rs
View file

@ -6,6 +6,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::{Map, Value}; use serde_json::{Map, Value};
use super::algos::SUPPORTED_ALGOS; use super::algos::SUPPORTED_ALGOS;
use super::from_algo_and_digest;
pub enum NixHashMode { pub enum NixHashMode {
Flat, Flat,
@ -103,40 +104,38 @@ impl NixHashWithMode {
if let Some(s) = v.as_str() { if let Some(s) = v.as_str() {
match s.strip_prefix("r:") { match s.strip_prefix("r:") {
Some(rest) => Ok(Some(Self::Recursive( Some(rest) => Ok(Some(Self::Recursive(
( from_algo_and_digest(
HashAlgo::try_from(rest).map_err(|e| { HashAlgo::try_from(rest).map_err(|e| {
serde::de::Error::invalid_value( serde::de::Error::invalid_value(
Unexpected::Other(&e.to_string()), Unexpected::Other(&e.to_string()),
&format!("one of {}", SUPPORTED_ALGOS.join(",")).as_str(), &format!("one of {}", SUPPORTED_ALGOS.join(",")).as_str(),
) )
})?, })?,
digest, &digest,
) )
.try_into() .map_err(|e: nixhash::Error| {
.map_err(|e: nixhash::Error| { serde::de::Error::invalid_value(
serde::de::Error::invalid_value( Unexpected::Other(&e.to_string()),
Unexpected::Other(&e.to_string()), &"a digest with right length",
&"a digest with right length", )
) })?,
})?,
))), ))),
None => Ok(Some(Self::Flat( None => Ok(Some(Self::Flat(
( from_algo_and_digest(
HashAlgo::try_from(s).map_err(|e| { HashAlgo::try_from(s).map_err(|e| {
serde::de::Error::invalid_value( serde::de::Error::invalid_value(
Unexpected::Other(&e.to_string()), Unexpected::Other(&e.to_string()),
&format!("one of {}", SUPPORTED_ALGOS.join(",")).as_str(), &format!("one of {}", SUPPORTED_ALGOS.join(",")).as_str(),
) )
})?, })?,
digest, &digest,
) )
.try_into() .map_err(|e: nixhash::Error| {
.map_err(|e: nixhash::Error| { serde::de::Error::invalid_value(
serde::de::Error::invalid_value( Unexpected::Other(&e.to_string()),
Unexpected::Other(&e.to_string()), &"a digest with right length",
&"a digest with right length", )
) })?,
})?,
))), ))),
} }
} else { } else {
@ -162,12 +161,12 @@ impl Serialize for NixHashWithMode {
let mut map = serializer.serialize_map(Some(2))?; let mut map = serializer.serialize_map(Some(2))?;
match self { match self {
NixHashWithMode::Flat(h) => { NixHashWithMode::Flat(h) => {
map.serialize_entry("hash", &nixbase32::encode(&h.digest))?; map.serialize_entry("hash", &nixbase32::encode(h.digest_as_bytes()))?;
map.serialize_entry("hashAlgo", &h.algo.to_string())?; map.serialize_entry("hashAlgo", &h.algo())?;
} }
NixHashWithMode::Recursive(h) => { NixHashWithMode::Recursive(h) => {
map.serialize_entry("hash", &nixbase32::encode(&h.digest))?; map.serialize_entry("hash", &nixbase32::encode(h.digest_as_bytes()))?;
map.serialize_entry("hashAlgo", &format!("r:{}", &h.algo.to_string()))?; map.serialize_entry("hashAlgo", &format!("r:{}", &h.algo()))?;
} }
}; };
map.end() map.end()

70
tvix/nix-compat/src/store_path/utils.rs
View file

@ -1,6 +1,6 @@
use super::{Error, STORE_DIR}; use super::{Error, STORE_DIR};
use crate::nixbase32; use crate::nixbase32;
use crate::nixhash::{HashAlgo, NixHash, NixHashWithMode}; use crate::nixhash::{NixHash, NixHashWithMode};
use crate::store_path::StorePath; use crate::store_path::StorePath;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use thiserror; use thiserror;
@ -55,12 +55,7 @@ pub fn build_text_path<S: AsRef<str>, I: IntoIterator<Item = S>, C: AsRef<[u8]>>
hasher.finalize() hasher.finalize()
}; };
// We populate the struct directly, as we know the sha256 digest has the NixHash::Sha256(content_digest.into())
// right size.
NixHash {
algo: crate::nixhash::HashAlgo::Sha256,
digest: content_digest.to_vec(),
}
}, },
name, name,
) )
@ -74,17 +69,14 @@ pub fn build_regular_ca_path<S: AsRef<str>, I: IntoIterator<Item = S>>(
self_reference: bool, self_reference: bool,
) -> Result<StorePath, BuildStorePathError> { ) -> Result<StorePath, BuildStorePathError> {
match &hash_with_mode { match &hash_with_mode {
NixHashWithMode::Recursive( NixHashWithMode::Recursive(ref hash @ NixHash::Sha256(_)) => {
ref hash @ NixHash { build_store_path_from_fingerprint_parts(
algo: HashAlgo::Sha256, &make_type("source", references, self_reference),
.. hash,
}, name,
) => build_store_path_from_fingerprint_parts( )
&make_type("source", references, self_reference), .map_err(BuildStorePathError::InvalidStorePath)
hash, }
name,
)
.map_err(BuildStorePathError::InvalidStorePath),
_ => { _ => {
if references.into_iter().next().is_some() { if references.into_iter().next().is_some() {
return Err(BuildStorePathError::InvalidReference()); return Err(BuildStorePathError::InvalidReference());
@ -98,21 +90,17 @@ pub fn build_regular_ca_path<S: AsRef<str>, I: IntoIterator<Item = S>>(
let content_digest = { let content_digest = {
let mut hasher = Sha256::new_with_prefix("fixed:out:"); let mut hasher = Sha256::new_with_prefix("fixed:out:");
hasher.update(hash_with_mode.mode().prefix()); hasher.update(hash_with_mode.mode().prefix());
hasher.update(hash_with_mode.digest().algo.to_string()); hasher.update(hash_with_mode.digest().algo().to_string());
hasher.update(":"); hasher.update(":");
hasher.update( hasher.update(
&data_encoding::HEXLOWER.encode(&hash_with_mode.digest().digest), &data_encoding::HEXLOWER
.encode(hash_with_mode.digest().digest_as_bytes()),
); );
hasher.update(":"); hasher.update(":");
hasher.finalize() hasher.finalize()
}; };
// We don't use [NixHash::from_algo_and_digest], as we know [Sha256] has NixHash::Sha256(content_digest.into())
// the right digest size.
NixHash {
algo: crate::nixhash::HashAlgo::Sha256,
digest: content_digest.to_vec(),
}
}, },
name, name,
) )
@ -123,12 +111,8 @@ pub fn build_regular_ca_path<S: AsRef<str>, I: IntoIterator<Item = S>>(
/// For given NAR sha256 digest and name, return the new [StorePath] this would have. /// For given NAR sha256 digest and name, return the new [StorePath] this would have.
pub fn build_nar_based_store_path(nar_sha256_digest: &[u8; 32], name: &str) -> StorePath { pub fn build_nar_based_store_path(nar_sha256_digest: &[u8; 32], name: &str) -> StorePath {
// We populate the struct directly, as we know the sha256 digest has the let nar_hash_with_mode =
// right size. NixHashWithMode::Recursive(NixHash::Sha256(nar_sha256_digest.to_owned()));
let nar_hash_with_mode = NixHashWithMode::Recursive(NixHash {
algo: HashAlgo::Sha256,
digest: nar_sha256_digest.to_vec(),
});
build_regular_ca_path(name, &nar_hash_with_mode, Vec::<String>::new(), false).unwrap() build_regular_ca_path(name, &nar_hash_with_mode, Vec::<String>::new(), false).unwrap()
} }
@ -266,12 +250,13 @@ mod test {
fn build_sha1_path() { fn build_sha1_path() {
let outer = build_regular_ca_path( let outer = build_regular_ca_path(
"bar", "bar",
&NixHashWithMode::Recursive(NixHash { &NixHashWithMode::Recursive(NixHash::Sha1(
algo: HashAlgo::Sha1, data_encoding::HEXLOWER
digest: data_encoding::HEXLOWER
.decode(b"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") .decode(b"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33")
.expect("hex should decode"), .expect("hex should decode")
}), .try_into()
.expect("should have right len"),
)),
Vec::<String>::new(), Vec::<String>::new(),
false, false,
) )
@ -294,11 +279,12 @@ mod test {
// rewrote '/nix/store/5xd714cbfnkz02h2vbsj4fm03x3f15nf-baz' to '/nix/store/s89y431zzhmdn3k8r96rvakryddkpv2v-baz' // rewrote '/nix/store/5xd714cbfnkz02h2vbsj4fm03x3f15nf-baz' to '/nix/store/s89y431zzhmdn3k8r96rvakryddkpv2v-baz'
let outer = build_regular_ca_path( let outer = build_regular_ca_path(
"baz", "baz",
&NixHashWithMode::Recursive(NixHash { &NixHashWithMode::Recursive(NixHash::Sha256(
algo: HashAlgo::Sha256, nixbase32::decode(b"1xqkzcb3909fp07qngljr4wcdnrh1gdam1m2n29i6hhrxlmkgkv1")
digest: nixbase32::decode(b"1xqkzcb3909fp07qngljr4wcdnrh1gdam1m2n29i6hhrxlmkgkv1") .expect("hex should decode")
.expect("hex should decode"), .try_into()
}), .expect("should have right len"),
)),
vec!["/nix/store/dxwkwjzdaq7ka55pkk252gh32bgpmql4-foo"], vec!["/nix/store/dxwkwjzdaq7ka55pkk252gh32bgpmql4-foo"],
false, false,
) )