tvl-depot/tvix/glue/src/known_paths.rs

//! This module implements logic required for persisting known paths
//! during an evaluation.
//!
//! Tvix needs to be able to keep track of each Nix store path that it
//! knows about during the scope of a single evaluation and its
//! related builds.
//!
//! This data is required to find the derivation needed to actually trigger the
//! build, if necessary.

use nix_compat::{
    derivation::Derivation,
    store_path::{BuildStorePathError, StorePath, StorePathRef},
};
use std::collections::HashMap;

use crate::fetchers::Fetch;

/// Struct keeping track of all known Derivations in the current evaluation.
/// This keeps both the Derivation struct, as well as the "Hash derivation
/// modulo".
#[derive(Debug, Default)]
pub struct KnownPaths {
    /// All known derivation or FOD hashes.
    ///
    /// Keys are derivation paths, values are a tuple of the "hash derivation
    /// modulo" and the Derivation struct itself.
    derivations: HashMap<StorePath<String>, ([u8; 32], Derivation)>,

    /// A map from output path to (one) drv path.
    /// Note that in the case of FODs, multiple drvs can produce the same output
    /// path. We use one of them.
    outputs_to_drvpath: HashMap<StorePath<String>, StorePath<String>>,

    /// A map from output path to fetches (and their names).
    outputs_to_fetches: HashMap<StorePath<String>, (String, Fetch)>,
}

impl KnownPaths {
    /// Fetch the opaque "hash derivation modulo" for a given derivation path.
    pub fn get_hash_derivation_modulo(&self, drv_path: &StorePath<String>) -> Option<&[u8; 32]> {
        self.derivations
            .get(drv_path)
            .map(|(hash_derivation_modulo, _derivation)| hash_derivation_modulo)
    }

    /// Return a reference to the Derivation for a given drv path.
    pub fn get_drv_by_drvpath(&self, drv_path: &StorePath<String>) -> Option<&Derivation> {
        self.derivations
            .get(drv_path)
            .map(|(_hash_derivation_modulo, derivation)| derivation)
    }

    /// Return the drv path of the derivation producing the passed output path.
    /// Note there can be multiple Derivations producing the same output path in
    /// flight; this function will only return one of them.
    pub fn get_drv_path_for_output_path(
        &self,
        output_path: &StorePath<String>,
    ) -> Option<&StorePath<String>> {
        self.outputs_to_drvpath.get(output_path)
    }

    /// Insert a new [Derivation] into this struct.
    /// The Derivation struct must pass validation, and its output paths need to
    /// be fully calculated.
    /// All input derivations this refers to must also be inserted to this
    /// struct.
    pub fn add_derivation(&mut self, drv_path: StorePath<String>, drv: Derivation) {
        // check input derivations to have been inserted.
        #[cfg(debug_assertions)]
        {
            for input_drv_path in drv.input_derivations.keys() {
                debug_assert!(self.derivations.contains_key(input_drv_path));
            }
        }

        // compute the hash derivation modulo
        let hash_derivation_modulo = drv.hash_derivation_modulo(|drv_path| {
            self.get_hash_derivation_modulo(&drv_path.to_owned())
                .unwrap_or_else(|| panic!("{} not found", drv_path))
                .to_owned()
        });

        // For all output paths, update our lookup table.
        // We only write into the lookup table once.
        for output in drv.outputs.values() {
            self.outputs_to_drvpath
                .entry(output.path.as_ref().expect("missing store path").clone())
                .or_insert(drv_path.to_owned());
        }

        // insert the derivation itself
        #[allow(unused_variables)] // assertions on this only compiled in debug builds
        let old = self
            .derivations
            .insert(drv_path.to_owned(), (hash_derivation_modulo, drv));

        #[cfg(debug_assertions)]
        {
            if let Some(old) = old {
                debug_assert!(
                    old.0 == hash_derivation_modulo,
                    "hash derivation modulo for a given derivation should always be calculated the same"
                );
            }
        }
    }

    /// Insert a new [Fetch] into this struct, which *must* have an expected
    /// hash (otherwise we wouldn't be able to calculate the store path).
    /// Fetches without a known hash need to be fetched inside builtins.
    pub fn add_fetch<'a>(
        &mut self,
        fetch: Fetch,
        name: &'a str,
    ) -> Result<StorePathRef<'a>, BuildStorePathError> {
        let store_path = fetch
            .store_path(name)?
            .expect("Tvix bug: fetch must have an expected hash");
        // insert the fetch.
        self.outputs_to_fetches
            .insert(store_path.to_owned(), (name.to_owned(), fetch));

        Ok(store_path)
    }

    /// Return the name and fetch producing the passed output path.
    /// Note there can also be (multiple) Derivations producing the same output path.
    pub fn get_fetch_for_output_path(
        &self,
        output_path: &StorePath<String>,
    ) -> Option<(String, Fetch)> {
        self.outputs_to_fetches
            .get(output_path)
            .map(|(name, fetch)| (name.to_owned(), fetch.to_owned()))
    }

    /// Returns an iterator over all known derivations and their store path.
    pub fn get_derivations(&self) -> impl Iterator<Item = (&StorePath<String>, &Derivation)> {
        self.derivations.iter().map(|(k, v)| (k, &v.1))
    }
}

#[cfg(test)]
mod tests {
    use nix_compat::{derivation::Derivation, nixbase32, nixhash, store_path::StorePath};
    use url::Url;

    use crate::fetchers::Fetch;

    use super::KnownPaths;
    use hex_literal::hex;
    use lazy_static::lazy_static;

    lazy_static! {
        static ref BAR_DRV: Derivation = Derivation::from_aterm_bytes(include_bytes!(
            "tests/ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv"
        ))
        .expect("must parse");
        static ref FOO_DRV: Derivation = Derivation::from_aterm_bytes(include_bytes!(
            "tests/ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv"
        ))
        .expect("must parse");
        static ref BAR_DRV_PATH: StorePath<String> =
            StorePath::from_bytes(b"ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv").expect("must parse");
        static ref FOO_DRV_PATH: StorePath<String> =
            StorePath::from_bytes(b"ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv").expect("must parse");
        static ref BAR_OUT_PATH: StorePath<String> =
            StorePath::from_bytes(b"mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar").expect("must parse");
        static ref FOO_OUT_PATH: StorePath<String> =
            StorePath::from_bytes(b"fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo").expect("must parse");

        static ref FETCH_URL : Fetch = Fetch::URL{
            url: Url::parse("https://raw.githubusercontent.com/aaptel/notmuch-extract-patch/f732a53e12a7c91a06755ebfab2007adc9b3063b/notmuch-extract-patch").unwrap(),
            exp_hash: Some(nixhash::from_sri_str("sha256-Xa1Jbl2Eq5+L0ww+Ph1osA3Z/Dxe/RkN1/dITQCdXFk=").unwrap())
        };
        static ref FETCH_URL_OUT_PATH: StorePath<String> = StorePath::from_bytes(b"06qi00hylriyfm0nl827crgjvbax84mz-notmuch-extract-patch").unwrap();

        static ref FETCH_TARBALL : Fetch = Fetch::Tarball{
            url: Url::parse("https://github.com/NixOS/nixpkgs/archive/91050ea1e57e50388fa87a3302ba12d188ef723a.tar.gz").unwrap(),
            exp_nar_sha256: Some(nixbase32::decode_fixed("1hf6cgaci1n186kkkjq106ryf8mmlq9vnwgfwh625wa8hfgdn4dm").unwrap())
        };
        static ref FETCH_TARBALL_OUT_PATH: StorePath<String> = StorePath::from_bytes(b"7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source").unwrap();
    }

    /// ensure we don't allow acdding a Derivation that depends on another,
    /// not-yet-added Derivation.
    #[test]
    #[should_panic]
    fn drv_reject_if_missing_input_drv() {
        let mut known_paths = KnownPaths::default();

        // FOO_DRV depends on BAR_DRV, which wasn't added.
        known_paths.add_derivation(FOO_DRV_PATH.clone(), FOO_DRV.clone());
    }

    #[test]
    fn drv_happy_path() {
        let mut known_paths = KnownPaths::default();

        // get_drv_by_drvpath should return None for non-existing Derivations,
        // same as get_hash_derivation_modulo and get_drv_path_for_output_path
        assert_eq!(None, known_paths.get_drv_by_drvpath(&BAR_DRV_PATH));
        assert_eq!(None, known_paths.get_hash_derivation_modulo(&BAR_DRV_PATH));
        assert_eq!(
            None,
            known_paths.get_drv_path_for_output_path(&BAR_OUT_PATH)
        );

        // Add BAR_DRV
        known_paths.add_derivation(BAR_DRV_PATH.clone(), BAR_DRV.clone());

        // We should get it back
        assert_eq!(
            Some(&BAR_DRV.clone()),
            known_paths.get_drv_by_drvpath(&BAR_DRV_PATH)
        );

        // Test get_drv_path_for_output_path
        assert_eq!(
            Some(&BAR_DRV_PATH.clone()),
            known_paths.get_drv_path_for_output_path(&BAR_OUT_PATH)
        );

        // It should be possible to get the hash derivation modulo.
        assert_eq!(
            Some(&hex!(
                "c79aebd0ce3269393d4a1fde2cbd1d975d879b40f0bf40a48f550edc107fd5df"
            )),
            known_paths.get_hash_derivation_modulo(&BAR_DRV_PATH.clone())
        );

        // Now insert FOO_DRV too. It shouldn't panic, as BAR_DRV is already
        // added.
        known_paths.add_derivation(FOO_DRV_PATH.clone(), FOO_DRV.clone());

        assert_eq!(
            Some(&FOO_DRV.clone()),
            known_paths.get_drv_by_drvpath(&FOO_DRV_PATH)
        );
        assert_eq!(
            Some(&hex!(
                "af030d36d63d3d7f56a71adaba26b36f5fa1f9847da5eed953ed62e18192762f"
            )),
            known_paths.get_hash_derivation_modulo(&FOO_DRV_PATH.clone())
        );

        // Test get_drv_path_for_output_path
        assert_eq!(
            Some(&FOO_DRV_PATH.clone()),
            known_paths.get_drv_path_for_output_path(&FOO_OUT_PATH)
        );
    }

    #[test]
    fn fetch_happy_path() {
        let mut known_paths = KnownPaths::default();

        // get_fetch_for_output_path should return None for new fetches.
        assert!(known_paths
            .get_fetch_for_output_path(&FETCH_TARBALL_OUT_PATH)
            .is_none());

        // add_fetch should return the properly calculated store paths.
        assert_eq!(
            *FETCH_TARBALL_OUT_PATH,
            known_paths
                .add_fetch(FETCH_TARBALL.clone(), "source")
                .unwrap()
                .to_owned()
        );

        assert_eq!(
            *FETCH_URL_OUT_PATH,
            known_paths
                .add_fetch(FETCH_URL.clone(), "notmuch-extract-patch")
                .unwrap()
                .to_owned()
        );
    }

    #[test]
    fn get_derivations_working() {
        let mut known_paths = KnownPaths::default();

        // Add BAR_DRV
        known_paths.add_derivation(BAR_DRV_PATH.clone(), BAR_DRV.clone());

        // We should be able to find BAR_DRV_PATH and BAR_DRV as a pair in get_derivations.
        assert_eq!(
            Some((&BAR_DRV_PATH.clone(), &BAR_DRV.clone())),
            known_paths
                .get_derivations()
                .find(|(s, d)| (*s, *d) == (&BAR_DRV_PATH, &BAR_DRV))
        );
    }

    // TODO: add test panicking about missing digest
}