2023-03-30 01:08:09 +02:00
|
|
|
use crate::nixbase32;
|
|
|
|
use crate::nixhash::{HashAlgo, NixHash};
|
feat(tvix/nix-compat): don't swallow hash validation errors
Previously, Output deserialization would silence validation errors and
provide `None` for `hash_with_mode` as soon as a validation error would
happen inside of the `NixHashWithMode` deserialization, e.g. invalid
hash length would not provide an validation error but a silent `None`
value.
This is problematic, we workaround a serde limitation here by writing
our own Deserializer.
As you can see, we write some boilerplate code unfortunately, as, for
example:
- `#[serde(fail_if_unparsed_as="Option::is_none")]` is not a thing,
otherwise, we could have been able to just bubble up errors in case of
"not fully parsed" (and not missing) values.
- `From<&serde_json::Value> for serde::de::Unexpected` is not a thing,
otherwise, we could just map invalid type errors and reuse the
existing types instead of doing extremely bizarre things with
`serde::de::Unexpected::Other`, note: this is a not problem for
expected, we know what we expect, we don't know what we received in
practice.
I decided to write a `NixHashWithMode::from_map` which will eat a map
deserialized via `serde_json`, so our serde magic is not totally "data
model" agnostic.
I wanted to go for data model agnosticity and enable maximal
performance, e.g. building the structure as the map values are streamed
in the Visitor, this is needlessly painful because `Output` and
`NixHashWithMode` are in different files and this really makes sense
only if we write the full implementation in one file, indeed, otherwise,
you end up duplicating the work or having strange coupling.
So, for now, we will allocate a full map of the fields inside the
`Output`, i.e. if any "unknown field" is in that map, we will
deserialize it for no reason.
Doing it properly, as I repeat it in the code and to flokli at C3Camp
2023, requires to patch serde upstream IMHO.
Change-Id: I46fe6ccb8c390c48d6934fd3e3f02a0dfe59557b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9107
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
2023-08-19 18:36:27 +02:00
|
|
|
use serde::de::Unexpected;
|
2023-03-30 01:08:09 +02:00
|
|
|
use serde::ser::SerializeMap;
|
|
|
|
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
feat(tvix/nix-compat): don't swallow hash validation errors
Previously, Output deserialization would silence validation errors and
provide `None` for `hash_with_mode` as soon as a validation error would
happen inside of the `NixHashWithMode` deserialization, e.g. invalid
hash length would not provide an validation error but a silent `None`
value.
This is problematic, we workaround a serde limitation here by writing
our own Deserializer.
As you can see, we write some boilerplate code unfortunately, as, for
example:
- `#[serde(fail_if_unparsed_as="Option::is_none")]` is not a thing,
otherwise, we could have been able to just bubble up errors in case of
"not fully parsed" (and not missing) values.
- `From<&serde_json::Value> for serde::de::Unexpected` is not a thing,
otherwise, we could just map invalid type errors and reuse the
existing types instead of doing extremely bizarre things with
`serde::de::Unexpected::Other`, note: this is a not problem for
expected, we know what we expect, we don't know what we received in
practice.
I decided to write a `NixHashWithMode::from_map` which will eat a map
deserialized via `serde_json`, so our serde magic is not totally "data
model" agnostic.
I wanted to go for data model agnosticity and enable maximal
performance, e.g. building the structure as the map values are streamed
in the Visitor, this is needlessly painful because `Output` and
`NixHashWithMode` are in different files and this really makes sense
only if we write the full implementation in one file, indeed, otherwise,
you end up duplicating the work or having strange coupling.
So, for now, we will allocate a full map of the fields inside the
`Output`, i.e. if any "unknown field" is in that map, we will
deserialize it for no reason.
Doing it properly, as I repeat it in the code and to flokli at C3Camp
2023, requires to patch serde upstream IMHO.
Change-Id: I46fe6ccb8c390c48d6934fd3e3f02a0dfe59557b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9107
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
2023-08-19 18:36:27 +02:00
|
|
|
use serde_json::{Map, Value};
|
|
|
|
|
|
|
|
use super::algos::SUPPORTED_ALGOS;
|
2023-03-30 01:08:09 +02:00
|
|
|
|
2023-03-31 16:20:04 +02:00
|
|
|
pub enum NixHashMode {
|
|
|
|
Flat,
|
|
|
|
Recursive,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl NixHashMode {
|
|
|
|
pub fn prefix(self) -> &'static str {
|
|
|
|
match self {
|
|
|
|
Self::Flat => "",
|
|
|
|
Self::Recursive => "r:",
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-30 01:08:09 +02:00
|
|
|
/// A Nix Hash can either be flat or recursive.
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
|
|
pub enum NixHashWithMode {
|
|
|
|
Flat(NixHash),
|
|
|
|
Recursive(NixHash),
|
|
|
|
}
|
|
|
|
|
|
|
|
impl NixHashWithMode {
|
2023-03-31 16:20:04 +02:00
|
|
|
pub fn mode(&self) -> NixHashMode {
|
|
|
|
match self {
|
|
|
|
Self::Flat(_) => NixHashMode::Flat,
|
|
|
|
Self::Recursive(_) => NixHashMode::Recursive,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn digest(&self) -> &NixHash {
|
|
|
|
match self {
|
|
|
|
Self::Flat(ref h) => h,
|
|
|
|
Self::Recursive(ref h) => h,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-30 01:08:09 +02:00
|
|
|
/// Formats a [NixHashWithMode] in the Nix default hash format,
|
|
|
|
/// which is the algo, followed by a colon, then the lower hex encoded digest.
|
|
|
|
/// In case the hash itself is recursive, a `r:` is added as prefix
|
|
|
|
pub fn to_nix_hash_string(&self) -> String {
|
2023-03-31 16:20:04 +02:00
|
|
|
String::from(self.mode().prefix()) + &self.digest().to_nix_hash_string()
|
2023-03-30 01:08:09 +02:00
|
|
|
}
|
feat(tvix/nix-compat): don't swallow hash validation errors
Previously, Output deserialization would silence validation errors and
provide `None` for `hash_with_mode` as soon as a validation error would
happen inside of the `NixHashWithMode` deserialization, e.g. invalid
hash length would not provide an validation error but a silent `None`
value.
This is problematic, we workaround a serde limitation here by writing
our own Deserializer.
As you can see, we write some boilerplate code unfortunately, as, for
example:
- `#[serde(fail_if_unparsed_as="Option::is_none")]` is not a thing,
otherwise, we could have been able to just bubble up errors in case of
"not fully parsed" (and not missing) values.
- `From<&serde_json::Value> for serde::de::Unexpected` is not a thing,
otherwise, we could just map invalid type errors and reuse the
existing types instead of doing extremely bizarre things with
`serde::de::Unexpected::Other`, note: this is a not problem for
expected, we know what we expect, we don't know what we received in
practice.
I decided to write a `NixHashWithMode::from_map` which will eat a map
deserialized via `serde_json`, so our serde magic is not totally "data
model" agnostic.
I wanted to go for data model agnosticity and enable maximal
performance, e.g. building the structure as the map values are streamed
in the Visitor, this is needlessly painful because `Output` and
`NixHashWithMode` are in different files and this really makes sense
only if we write the full implementation in one file, indeed, otherwise,
you end up duplicating the work or having strange coupling.
So, for now, we will allocate a full map of the fields inside the
`Output`, i.e. if any "unknown field" is in that map, we will
deserialize it for no reason.
Doing it properly, as I repeat it in the code and to flokli at C3Camp
2023, requires to patch serde upstream IMHO.
Change-Id: I46fe6ccb8c390c48d6934fd3e3f02a0dfe59557b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9107
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
2023-08-19 18:36:27 +02:00
|
|
|
|
|
|
|
/// This takes a serde_json::Map and turns it into this structure. This is necessary to do such
|
|
|
|
/// shenigans because we have external consumers, like the Derivation parser, who would like to
|
|
|
|
/// know whether we have a invalid or a missing NixHashWithMode structure in another structure,
|
|
|
|
/// e.g. Output.
|
|
|
|
/// This means we have this combinatorial situation:
|
|
|
|
/// - no hash, no hashAlgo: no NixHashWithMode so we return Ok(None).
|
|
|
|
/// - present hash, missing hashAlgo: invalid, we will return missing_field
|
|
|
|
/// - missing hash, present hashAlgo: same
|
|
|
|
/// - present hash, present hashAlgo: either we return ourselves or a type/value validation
|
|
|
|
/// error.
|
|
|
|
/// This function is for internal consumption regarding those needs until we have a better
|
|
|
|
/// solution. Now this is said, let's explain how this works.
|
|
|
|
///
|
|
|
|
/// We want to map the serde data model into a NixHashWithMode.
|
|
|
|
///
|
|
|
|
/// The serde data model has a `hash` field (containing a digest in nixbase32),
|
|
|
|
/// and a `hashAlgo` field, containing the stringified hash algo.
|
|
|
|
/// In case the hash is recursive, hashAlgo also has a `r:` prefix.
|
|
|
|
///
|
|
|
|
/// This is to match how `nix show-derivation` command shows them in JSON
|
|
|
|
/// representation.
|
|
|
|
pub(crate) fn from_map<'de, D>(map: &Map<String, Value>) -> Result<Option<Self>, D::Error>
|
|
|
|
where
|
|
|
|
D: Deserializer<'de>,
|
|
|
|
{
|
|
|
|
// If we don't have hash neither hashAlgo, let's just return None.
|
|
|
|
if !map.contains_key("hash") && !map.contains_key("hashAlgo") {
|
|
|
|
return Ok(None);
|
|
|
|
}
|
|
|
|
|
|
|
|
let digest: Vec<u8> = {
|
|
|
|
if let Some(v) = map.get("hash") {
|
|
|
|
if let Some(s) = v.as_str() {
|
|
|
|
data_encoding::HEXLOWER
|
|
|
|
.decode(s.as_bytes())
|
|
|
|
.map_err(|e| serde::de::Error::custom(e.to_string()))?
|
|
|
|
} else {
|
|
|
|
return Err(serde::de::Error::invalid_type(
|
|
|
|
Unexpected::Other(&v.to_string()),
|
|
|
|
&"a string",
|
|
|
|
));
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
return Err(serde::de::Error::missing_field(
|
|
|
|
"couldn't extract `hash` key but `hashAlgo` key present",
|
|
|
|
));
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
if let Some(v) = map.get("hashAlgo") {
|
|
|
|
if let Some(s) = v.as_str() {
|
|
|
|
match s.strip_prefix("r:") {
|
|
|
|
Some(rest) => Ok(Some(Self::Recursive(NixHash::new(
|
|
|
|
HashAlgo::try_from(rest).map_err(|e| {
|
|
|
|
serde::de::Error::invalid_value(
|
|
|
|
Unexpected::Other(&e.to_string()),
|
|
|
|
&format!("one of {}", SUPPORTED_ALGOS.join(",")).as_str(),
|
|
|
|
)
|
|
|
|
})?,
|
|
|
|
digest,
|
|
|
|
)))),
|
|
|
|
None => Ok(Some(Self::Flat(NixHash::new(
|
|
|
|
HashAlgo::try_from(s).map_err(|e| {
|
|
|
|
serde::de::Error::invalid_value(
|
|
|
|
Unexpected::Other(&e.to_string()),
|
|
|
|
&format!("one of {}", SUPPORTED_ALGOS.join(",")).as_str(),
|
|
|
|
)
|
|
|
|
})?,
|
|
|
|
digest,
|
|
|
|
)))),
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
Err(serde::de::Error::invalid_type(
|
|
|
|
Unexpected::Other(&v.to_string()),
|
|
|
|
&"a string",
|
|
|
|
))
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
Err(serde::de::Error::missing_field(
|
|
|
|
"couldn't extract `hashAlgo` key, but `hash` key present",
|
|
|
|
))
|
|
|
|
}
|
|
|
|
}
|
2023-03-30 01:08:09 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
impl Serialize for NixHashWithMode {
|
|
|
|
/// map a NixHashWithMode into the serde data model.
|
|
|
|
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
|
|
|
where
|
|
|
|
S: Serializer,
|
|
|
|
{
|
|
|
|
let mut map = serializer.serialize_map(Some(2))?;
|
|
|
|
match self {
|
|
|
|
NixHashWithMode::Flat(h) => {
|
|
|
|
map.serialize_entry("hash", &nixbase32::encode(&h.digest))?;
|
|
|
|
map.serialize_entry("hashAlgo", &h.algo.to_string())?;
|
|
|
|
}
|
|
|
|
NixHashWithMode::Recursive(h) => {
|
|
|
|
map.serialize_entry("hash", &nixbase32::encode(&h.digest))?;
|
|
|
|
map.serialize_entry("hashAlgo", &format!("r:{}", &h.algo.to_string()))?;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
map.end()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<'de> Deserialize<'de> for NixHashWithMode {
|
|
|
|
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
|
|
|
where
|
|
|
|
D: Deserializer<'de>,
|
|
|
|
{
|
feat(tvix/nix-compat): don't swallow hash validation errors
Previously, Output deserialization would silence validation errors and
provide `None` for `hash_with_mode` as soon as a validation error would
happen inside of the `NixHashWithMode` deserialization, e.g. invalid
hash length would not provide an validation error but a silent `None`
value.
This is problematic, we workaround a serde limitation here by writing
our own Deserializer.
As you can see, we write some boilerplate code unfortunately, as, for
example:
- `#[serde(fail_if_unparsed_as="Option::is_none")]` is not a thing,
otherwise, we could have been able to just bubble up errors in case of
"not fully parsed" (and not missing) values.
- `From<&serde_json::Value> for serde::de::Unexpected` is not a thing,
otherwise, we could just map invalid type errors and reuse the
existing types instead of doing extremely bizarre things with
`serde::de::Unexpected::Other`, note: this is a not problem for
expected, we know what we expect, we don't know what we received in
practice.
I decided to write a `NixHashWithMode::from_map` which will eat a map
deserialized via `serde_json`, so our serde magic is not totally "data
model" agnostic.
I wanted to go for data model agnosticity and enable maximal
performance, e.g. building the structure as the map values are streamed
in the Visitor, this is needlessly painful because `Output` and
`NixHashWithMode` are in different files and this really makes sense
only if we write the full implementation in one file, indeed, otherwise,
you end up duplicating the work or having strange coupling.
So, for now, we will allocate a full map of the fields inside the
`Output`, i.e. if any "unknown field" is in that map, we will
deserialize it for no reason.
Doing it properly, as I repeat it in the code and to flokli at C3Camp
2023, requires to patch serde upstream IMHO.
Change-Id: I46fe6ccb8c390c48d6934fd3e3f02a0dfe59557b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9107
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
2023-08-19 18:36:27 +02:00
|
|
|
let value = Self::from_map::<D>(&Map::deserialize(deserializer)?)?;
|
2023-03-30 01:08:09 +02:00
|
|
|
|
feat(tvix/nix-compat): don't swallow hash validation errors
Previously, Output deserialization would silence validation errors and
provide `None` for `hash_with_mode` as soon as a validation error would
happen inside of the `NixHashWithMode` deserialization, e.g. invalid
hash length would not provide an validation error but a silent `None`
value.
This is problematic, we workaround a serde limitation here by writing
our own Deserializer.
As you can see, we write some boilerplate code unfortunately, as, for
example:
- `#[serde(fail_if_unparsed_as="Option::is_none")]` is not a thing,
otherwise, we could have been able to just bubble up errors in case of
"not fully parsed" (and not missing) values.
- `From<&serde_json::Value> for serde::de::Unexpected` is not a thing,
otherwise, we could just map invalid type errors and reuse the
existing types instead of doing extremely bizarre things with
`serde::de::Unexpected::Other`, note: this is a not problem for
expected, we know what we expect, we don't know what we received in
practice.
I decided to write a `NixHashWithMode::from_map` which will eat a map
deserialized via `serde_json`, so our serde magic is not totally "data
model" agnostic.
I wanted to go for data model agnosticity and enable maximal
performance, e.g. building the structure as the map values are streamed
in the Visitor, this is needlessly painful because `Output` and
`NixHashWithMode` are in different files and this really makes sense
only if we write the full implementation in one file, indeed, otherwise,
you end up duplicating the work or having strange coupling.
So, for now, we will allocate a full map of the fields inside the
`Output`, i.e. if any "unknown field" is in that map, we will
deserialize it for no reason.
Doing it properly, as I repeat it in the code and to flokli at C3Camp
2023, requires to patch serde upstream IMHO.
Change-Id: I46fe6ccb8c390c48d6934fd3e3f02a0dfe59557b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9107
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
2023-08-19 18:36:27 +02:00
|
|
|
match value {
|
|
|
|
None => Err(serde::de::Error::custom("couldn't parse as map")),
|
|
|
|
Some(v) => Ok(v),
|
2023-03-30 01:08:09 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|