mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix/castore/import): make module, split off fs and error

Browse source 
Move error types and filesystem-specific functions to a separate file,
and keep the fs:: namespace in public exports.

Change-Id: I5e9e83ad78d9aea38553fafc293d3e4f8c31a8c1
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11486
Tested-by: BuildkiteCI
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink 2024-04-20 15:01:49 +03:00 committed by clbot
parent c4cb099823
commit e9db0449e7
7 changed files with 225 additions and 197 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
tvix/castore/src/import/error.rs Normal file
View file

@ -0,0 +1,39 @@
use std::{fs::FileType, path::PathBuf};
use crate::Error as CastoreError;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("failed to upload directory at {0}: {1}")]
UploadDirectoryError(PathBuf, CastoreError),
#[error("invalid encoding encountered for entry {0:?}")]
InvalidEncoding(PathBuf),
#[error("unable to stat {0}: {1}")]
UnableToStat(PathBuf, std::io::Error),
#[error("unable to open {0}: {1}")]
UnableToOpen(PathBuf, std::io::Error),
#[error("unable to read {0}: {1}")]
UnableToRead(PathBuf, std::io::Error),
#[error("unsupported file {0} type: {1:?}")]
UnsupportedFileType(PathBuf, FileType),
}
impl From<CastoreError> for Error {
fn from(value: CastoreError) -> Self {
match value {
CastoreError::InvalidRequest(_) => panic!("tvix bug"),
CastoreError::StorageError(_) => panic!("error"),
}
}
}
impl From<Error> for std::io::Error {
fn from(value: Error) -> Self {
std::io::Error::new(std::io::ErrorKind::Other, value)
}
}

143
tvix/castore/src/import/fs.rs Normal file
View file

@ -0,0 +1,143 @@
use std::os::unix::fs::MetadataExt;
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
use futures::stream::BoxStream;
use tracing::instrument;
use walkdir::DirEntry;
use walkdir::WalkDir;
use crate::blobservice::BlobService;
use crate::directoryservice::DirectoryService;
use crate::proto::node::Node;
use super::ingest_entries;
use super::upload_blob_at_path;
use super::Error;
use super::IngestionEntry;
///! Imports that deal with a real filesystem.
/// Ingests the contents at a given path into the tvix store, interacting with a [BlobService] and
/// [DirectoryService]. It returns the root node or an error.
///
/// It does not follow symlinks at the root, they will be ingested as actual symlinks.
#[instrument(skip(blob_service, directory_service), fields(path), err)]
pub async fn ingest_path<BS, DS, P>(
blob_service: BS,
directory_service: DS,
path: P,
) -> Result<Node, Error>
where
P: AsRef<Path> + std::fmt::Debug,
BS: BlobService + Clone,
DS: AsRef<dyn DirectoryService>,
{
let entry_stream = walk_path_for_ingestion(blob_service, path.as_ref());
ingest_entries(directory_service, entry_stream).await
}
/// Walk the filesystem at a given path and returns a stream of ingestion entries.
///
/// This is how [`ingest_path`] assembles the set of entries to pass on [`ingest_entries`].
/// This low-level function can be used if additional filtering or processing is required on the
/// entries.
///
/// It does not follow symlinks at the root, they will be ingested as actual symlinks.
///
/// This function will walk the filesystem using `walkdir` and will consume
/// `O(#number of entries)` space.
#[instrument(fields(path), skip(blob_service))]
fn walk_path_for_ingestion<'a, BS>(
blob_service: BS,
path: &'a Path,
) -> BoxStream<'a, Result<IngestionEntry<'a>, Error>>
where
BS: BlobService + Clone + 'a,
{
let iter = WalkDir::new(path)
.follow_links(false)
.follow_root_links(false)
.contents_first(true)
.into_iter();
dir_entry_iter_to_ingestion_stream(blob_service, iter, path)
}
/// Converts an iterator of [walkdir::DirEntry]s into a stream of ingestion entries.
/// This can then be fed into [ingest_entries] to ingest all the entries into the castore.
///
/// The root is the [Path] in the filesystem that is being ingested into the castore.
pub fn dir_entry_iter_to_ingestion_stream<'a, BS, I>(
blob_service: BS,
iter: I,
root: &'a Path,
) -> BoxStream<'a, Result<IngestionEntry<'a>, Error>>
where
BS: BlobService + Clone + 'a,
I: Iterator<Item = Result<DirEntry, walkdir::Error>> + Send + 'a,
{
let prefix = root.parent().unwrap_or_else(|| Path::new(""));
let iter = iter.map(move |entry| match entry {
Ok(entry) => dir_entry_to_ingestion_entry(blob_service.clone(), &entry, prefix),
Err(error) => Err(Error::UnableToStat(
root.to_path_buf(),
error.into_io_error().expect("walkdir err must be some"),
)),
});
Box::pin(futures::stream::iter(iter))
}
/// Converts a [walkdir::DirEntry] into an [IngestionEntry], uploading blobs to the
/// provided [BlobService].
///
/// The prefix path is stripped from the path of each entry. This is usually the parent path
/// of the path being ingested so that the last element of the stream only has one component.
fn dir_entry_to_ingestion_entry<'a, BS>(
blob_service: BS,
entry: &DirEntry,
prefix: &Path,
) -> Result<IngestionEntry<'a>, Error>
where
BS: BlobService + 'a,
{
let file_type = entry.file_type();
let path = entry
.path()
.strip_prefix(prefix)
.expect("Tvix bug: failed to strip root path prefix")
.to_path_buf();
if file_type.is_dir() {
Ok(IngestionEntry::Dir { path })
} else if file_type.is_symlink() {
let target = std::fs::read_link(entry.path())
.map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e))?;
Ok(IngestionEntry::Symlink { path, target })
} else if file_type.is_file() {
let metadata = entry
.metadata()
.map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e.into()))?;
// TODO: In the future, for small files, hash right away and upload async.
let digest = Box::pin(upload_blob_at_path(
blob_service,
entry.path().to_path_buf(),
));
Ok(IngestionEntry::Regular {
path,
size: metadata.size(),
// If it's executable by the user, it'll become executable.
// This matches nix's dump() function behaviour.
executable: metadata.permissions().mode() & 64 != 0,
digest,
})
} else {
Ok(IngestionEntry::Unknown { path, file_type })
}
}

228
tvix/castore/src/import.rs → tvix/castore/src/import/mod.rs
View file

@ -1,3 +1,10 @@
//! Deals with ingesting contents into castore.
//! The main library function here is [ingest_entries], receiving a stream of
//! [IngestionEntry].
//!
//! Specific implementations, such as ingesting from the filesystem, live in
//! child modules.
use crate::blobservice::BlobService;
use crate::directoryservice::DirectoryPutter;
use crate::directoryservice::DirectoryService;
@ -7,12 +14,9 @@ use crate::proto::DirectoryNode;
use crate::proto::FileNode;
use crate::proto::SymlinkNode;
use crate::B3Digest;
use crate::Error as CastoreError;
use futures::stream::BoxStream;
use futures::Future;
use futures::{Stream, StreamExt};
use std::fs::FileType;
use std::os::unix::fs::MetadataExt;
use std::pin::Pin;
use tracing::Level;
@ -21,200 +25,17 @@ use std::os::unix::ffi::OsStrExt;
use std::{
collections::HashMap,
fmt::Debug,
os::unix::prelude::PermissionsExt,
path::{Path, PathBuf},
};
use tracing::instrument;
use walkdir::DirEntry;
use walkdir::WalkDir;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("failed to upload directory at {0}: {1}")]
UploadDirectoryError(PathBuf, CastoreError),
mod error;
pub use error::Error;
#[error("invalid encoding encountered for entry {0:?}")]
InvalidEncoding(PathBuf),
pub mod fs;
#[error("unable to stat {0}: {1}")]
UnableToStat(PathBuf, std::io::Error),
#[error("unable to open {0}: {1}")]
UnableToOpen(PathBuf, std::io::Error),
#[error("unable to read {0}: {1}")]
UnableToRead(PathBuf, std::io::Error),
#[error("unsupported file {0} type: {1:?}")]
UnsupportedFileType(PathBuf, FileType),
}
impl From<CastoreError> for Error {
fn from(value: CastoreError) -> Self {
match value {
CastoreError::InvalidRequest(_) => panic!("tvix bug"),
CastoreError::StorageError(_) => panic!("error"),
}
}
}
impl From<Error> for std::io::Error {
fn from(value: Error) -> Self {
std::io::Error::new(std::io::ErrorKind::Other, value)
}
}
/// Walk the filesystem at a given path and returns a stream of ingestion entries.
///
/// This is how [`ingest_path`] assembles the set of entries to pass on [`ingest_entries`].
/// This low-level function can be used if additional filtering or processing is required on the
/// entries.
///
/// It does not follow symlinks at the root, they will be ingested as actual symlinks.
///
/// This function will walk the filesystem using `walkdir` and will consume
/// `O(#number of entries)` space.
#[instrument(fields(path), skip(blob_service))]
fn walk_path_for_ingestion<'a, BS>(
blob_service: BS,
path: &'a Path,
) -> BoxStream<'a, Result<IngestionEntry<'a>, Error>>
where
BS: BlobService + Clone + 'a,
{
let iter = WalkDir::new(path)
.follow_links(false)
.follow_root_links(false)
.contents_first(true)
.into_iter();
dir_entry_iter_to_ingestion_stream(blob_service, iter, path)
}
/// Converts an iterator of [walkdir::DirEntry]s into a stream of ingestion entries.
/// This can then be fed into [ingest_entries] to ingest all the entries into the castore.
///
/// The root is the [Path] in the filesystem that is being ingested into the castore.
pub fn dir_entry_iter_to_ingestion_stream<'a, BS, I>(
blob_service: BS,
iter: I,
root: &'a Path,
) -> BoxStream<'a, Result<IngestionEntry<'a>, Error>>
where
BS: BlobService + Clone + 'a,
I: Iterator<Item = Result<DirEntry, walkdir::Error>> + Send + 'a,
{
let prefix = root.parent().unwrap_or_else(|| Path::new(""));
let iter = iter.map(move |entry| match entry {
Ok(entry) => dir_entry_to_ingestion_entry(blob_service.clone(), &entry, prefix),
Err(error) => Err(Error::UnableToStat(
root.to_path_buf(),
error.into_io_error().expect("walkdir err must be some"),
)),
});
Box::pin(futures::stream::iter(iter))
}
/// Converts a [walkdir::DirEntry] into an [IngestionEntry], uploading blobs to the
/// provided [BlobService].
///
/// The prefix path is stripped from the path of each entry. This is usually the parent path
/// of the path being ingested so that the last element of the stream only has one component.
fn dir_entry_to_ingestion_entry<'a, BS>(
blob_service: BS,
entry: &DirEntry,
prefix: &Path,
) -> Result<IngestionEntry<'a>, Error>
where
BS: BlobService + 'a,
{
let file_type = entry.file_type();
let path = entry
.path()
.strip_prefix(prefix)
.expect("Tvix bug: failed to strip root path prefix")
.to_path_buf();
if file_type.is_dir() {
Ok(IngestionEntry::Dir { path })
} else if file_type.is_symlink() {
let target = std::fs::read_link(entry.path())
.map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e))?;
Ok(IngestionEntry::Symlink { path, target })
} else if file_type.is_file() {
let metadata = entry
.metadata()
.map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e.into()))?;
// TODO: In the future, for small files, hash right away and upload async.
let digest = Box::pin(upload_blob_at_path(
blob_service,
entry.path().to_path_buf(),
));
Ok(IngestionEntry::Regular {
path,
size: metadata.size(),
// If it's executable by the user, it'll become executable.
// This matches nix's dump() function behaviour.
executable: metadata.permissions().mode() & 64 != 0,
digest,
})
} else {
Ok(IngestionEntry::Unknown { path, file_type })
}
}
/// Uploads the file at the provided [Path] the the [BlobService].
#[instrument(skip(blob_service), fields(path), err)]
async fn upload_blob_at_path<BS>(blob_service: BS, path: PathBuf) -> Result<B3Digest, Error>
where
BS: BlobService,
{
let mut file = match tokio::fs::File::open(&path).await {
Ok(file) => file,
Err(e) => return Err(Error::UnableToRead(path, e)),
};
let mut writer = blob_service.open_write().await;
if let Err(e) = tokio::io::copy(&mut file, &mut writer).await {
return Err(Error::UnableToRead(path, e));
};
let digest = writer
.close()
.await
.map_err(|e| Error::UnableToRead(path, e))?;
Ok(digest)
}
/// Ingests the contents at a given path into the tvix store, interacting with a [BlobService] and
/// [DirectoryService]. It returns the root node or an error.
///
/// It does not follow symlinks at the root, they will be ingested as actual symlinks.
#[instrument(skip(blob_service, directory_service), fields(path), err)]
pub async fn ingest_path<BS, DS, P>(
blob_service: BS,
directory_service: DS,
path: P,
) -> Result<Node, Error>
where
P: AsRef<Path> + std::fmt::Debug,
BS: BlobService + Clone,
DS: AsRef<dyn DirectoryService>,
{
let entry_stream = walk_path_for_ingestion(blob_service, path.as_ref());
ingest_entries(directory_service, entry_stream).await
}
/// Ingests elements from the given stream of [IngestionEntry] into a the passed [DirectoryService].
/// Ingests [IngestionEntry] from the given stream into a the passed [DirectoryService].
/// On success, returns the root [Node].
///
/// The stream must have the following invariants:
/// - All children entries must come before their parents.
@ -336,6 +157,31 @@ where
Ok(root_node)
}
/// Uploads the file at the provided [Path] the the [BlobService].
#[instrument(skip(blob_service), fields(path), err)]
async fn upload_blob_at_path<BS>(blob_service: BS, path: PathBuf) -> Result<B3Digest, Error>
where
BS: BlobService,
{
let mut file = match tokio::fs::File::open(&path).await {
Ok(file) => file,
Err(e) => return Err(Error::UnableToRead(path, e)),
};
let mut writer = blob_service.open_write().await;
if let Err(e) = tokio::io::copy(&mut file, &mut writer).await {
return Err(Error::UnableToRead(path, e));
};
let digest = writer
.close()
.await
.map_err(|e| Error::UnableToRead(path, e))?;
Ok(digest)
}
type BlobFut<'a> = Pin<Box<dyn Future<Output = Result<B3Digest, Error>> + Send + 'a>>;
pub enum IngestionEntry<'a> {

2
tvix/castore/src/tests/import.rs
View file

@ -1,7 +1,7 @@
use crate::blobservice::{self, BlobService};
use crate::directoryservice;
use crate::fixtures::*;
use crate::import::ingest_path;
use crate::import::fs::ingest_path;
use crate::proto;
use std::sync::Arc;

2
tvix/glue/src/tvix_store_io.rs
View file

@ -18,7 +18,7 @@ use std::{
use tokio_util::io::SyncIoBridge;
use tracing::{error, instrument, warn, Level};
use tvix_build::buildservice::BuildService;
use tvix_castore::import::dir_entry_iter_to_ingestion_stream;
use tvix_castore::import::fs::dir_entry_iter_to_ingestion_stream;
use tvix_eval::{ErrorKind, EvalIO, FileType, StdIO};
use tvix_store::utils::AsyncIoBridge;
use walkdir::DirEntry;

2
tvix/store/src/bin/tvix-store.rs
View file

@ -17,7 +17,7 @@ use tracing::Level;
use tracing_subscriber::EnvFilter;
use tracing_subscriber::Layer;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
use tvix_castore::import::ingest_path;
use tvix_castore::import::fs::ingest_path;
use tvix_store::proto::NarInfo;
use tvix_store::proto::PathInfo;

6
tvix/store/src/import.rs
View file

@ -1,7 +1,8 @@
use std::path::Path;
use tracing::{debug, instrument};
use tvix_castore::{
blobservice::BlobService, directoryservice::DirectoryService, proto::node::Node, B3Digest,
blobservice::BlobService, directoryservice::DirectoryService, import::fs::ingest_path,
proto::node::Node, B3Digest,
};
use nix_compat::{
@ -116,8 +117,7 @@ where
DS: AsRef<dyn DirectoryService>,
PS: AsRef<dyn PathInfoService>,
{
let root_node =
tvix_castore::import::ingest_path(blob_service, directory_service, path.as_ref()).await?;
let root_node = ingest_path(blob_service, directory_service, path.as_ref()).await?;
// Ask the PathInfoService for the NAR size and sha256
let (nar_size, nar_sha256) = path_info_service.as_ref().calculate_nar(&root_node).await?;