mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix/store/import): use DirectoryPutter in import.rs

Browse source 
This should allow import_path to communicate to a gRPC remote store,
that actually verifies the Directory nodes are interconnected.

Change-Id: Ic5d28c33518f50dedec15f1732d81579a3afaff1
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8357
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2023-03-27 17:24:53 +02:00 committed by clbot
parent 0836450006
commit 8263c024c9
1 changed files with 7 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
tvix/store/src/import.rs
View file

@ -1,4 +1,4 @@
use crate::{chunkservice::read_all_and_chunk, proto}; use crate::{chunkservice::read_all_and_chunk, directoryservice::DirectoryPutter, proto};
use std::{ use std::{
collections::HashMap, collections::HashMap,
fmt::Debug, fmt::Debug,
@ -57,10 +57,10 @@ impl From<super::Error> for Error {
// //
// It assumes the caller adds returned nodes to the directories it assembles. // It assumes the caller adds returned nodes to the directories it assembles.
#[instrument(skip_all, fields(entry.file_type=?&entry.file_type(),entry.path=?entry.path()))] #[instrument(skip_all, fields(entry.file_type=?&entry.file_type(),entry.path=?entry.path()))]
fn process_entry<BS: BlobService, CS: ChunkService + std::marker::Sync, DS: DirectoryService>( fn process_entry<BS: BlobService, CS: ChunkService + std::marker::Sync, DP: DirectoryPutter>(
blob_service: &mut BS, blob_service: &mut BS,
chunk_service: &mut CS, chunk_service: &mut CS,
directory_service: &mut DS, directory_putter: &mut DP,
entry: &walkdir::DirEntry, entry: &walkdir::DirEntry,
maybe_directory: Option<proto::Directory>, maybe_directory: Option<proto::Directory>,
) -> Result<proto::node::Node, Error> { ) -> Result<proto::node::Node, Error> {
@ -75,7 +75,7 @@ fn process_entry<BS: BlobService, CS: ChunkService + std::marker::Sync, DS: Dire
let directory_size = directory.size(); let directory_size = directory.size();
// upload this directory // upload this directory
directory_service directory_putter
.put(directory) .put(directory)
.map_err(|e| Error::UploadDirectoryError(entry.path().to_path_buf(), e))?; .map_err(|e| Error::UploadDirectoryError(entry.path().to_path_buf(), e))?;
@ -188,6 +188,8 @@ pub fn import_path<
let mut directories: HashMap<PathBuf, proto::Directory> = HashMap::default(); let mut directories: HashMap<PathBuf, proto::Directory> = HashMap::default();
let mut directory_putter = directory_service.put_multiple_start();
for entry in WalkDir::new(p) for entry in WalkDir::new(p)
.follow_links(false) .follow_links(false)
.contents_first(true) .contents_first(true)
@ -213,7 +215,7 @@ pub fn import_path<
let node = process_entry( let node = process_entry(
blob_service, blob_service,
chunk_service, chunk_service,
directory_service, &mut directory_putter,
&entry, &entry,
maybe_directory, maybe_directory,
)?; )?;