feat(tvix/store/fuse): allow listing

This provides an additional configuration flag to the tvix-store mount
subcommand, and logic in the fuse module to request listing for the
root of the mountpoint.

Change-Id: I05a8bc11f7991b574696f27a30afe0f4e718a58c
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9217
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: adisbladis <adisbladis@gmail.com>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2023-09-03 17:10:06 +03:00 committed by clbot
parent da9d706e0a
commit f9b5fc49b1
3 changed files with 111 additions and 4 deletions

View file

@ -90,6 +90,10 @@ enum Commands {
#[arg(long, env, default_value = "grpc+http://[::1]:8000")]
path_info_service_addr: String,
/// Whether to list elements at the root of the mount point.
#[clap(long, short, action)]
list_root: bool,
},
}
@ -250,6 +254,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
blob_service_addr,
directory_service_addr,
path_info_service_addr,
list_root,
} => {
let blob_service = blobservice::from_addr(&blob_service_addr)?;
let directory_service = directoryservice::from_addr(&directory_service_addr)?;
@ -260,7 +265,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
)?;
tokio::task::spawn_blocking(move || {
let f = FUSE::new(blob_service, directory_service, path_info_service);
let f = FUSE::new(
blob_service,
directory_service,
path_info_service,
list_root,
);
fuser::mount2(f, &dest, &[])
})
.await??

View file

@ -66,6 +66,9 @@ pub struct FUSE {
directory_service: Arc<dyn DirectoryService>,
path_info_service: Arc<dyn PathInfoService>,
/// Whether to (try) listing elements in the root.
list_root: bool,
/// This maps a given StorePath to the inode we allocated for the root inode.
store_paths: HashMap<StorePath, u64>,
@ -83,12 +86,15 @@ impl FUSE {
blob_service: Arc<dyn BlobService>,
directory_service: Arc<dyn DirectoryService>,
path_info_service: Arc<dyn PathInfoService>,
list_root: bool,
) -> Self {
Self {
blob_service,
directory_service,
path_info_service,
list_root,
store_paths: HashMap::default(),
inode_tracker: Default::default(),
@ -311,8 +317,55 @@ impl fuser::Filesystem for FUSE {
debug!("readdir");
if ino == fuser::FUSE_ROOT_ID {
reply.error(libc::EPERM); // same error code as ipfs/kubo
return;
if !self.list_root {
reply.error(libc::EPERM); // same error code as ipfs/kubo
return;
} else {
for (i, path_info) in self
.path_info_service
.list()
.skip(offset as usize)
.enumerate()
{
let path_info = match path_info {
Err(e) => {
warn!("failed to retrieve pathinfo: {}", e);
reply.error(libc::EPERM);
return;
}
Ok(path_info) => path_info,
};
// We know the root node exists and the store_path can be parsed because clients MUST validate.
let root_node = path_info.node.unwrap().node.unwrap();
let store_path = StorePath::from_bytes(root_node.get_name()).unwrap();
let ino = match self.store_paths.get(&store_path) {
Some(ino) => *ino,
None => {
// insert the (sparse) inode data and register in
// self.store_paths.
let ino = self.inode_tracker.put((&root_node).into());
self.store_paths.insert(store_path.clone(), ino);
ino
}
};
let ty = match root_node {
Node::Directory(_) => fuser::FileType::Directory,
Node::File(_) => fuser::FileType::RegularFile,
Node::Symlink(_) => fuser::FileType::Symlink,
};
let full =
reply.add(ino, offset + i as i64 + 1_i64, ty, store_path.to_string());
if full {
break;
}
}
reply.ok();
return;
}
}
// lookup the inode data.

View file

@ -25,6 +25,17 @@ fn setup_and_mount<P: AsRef<Path>, F>(
mountpoint: P,
setup_fn: F,
) -> Result<fuser::BackgroundSession, std::io::Error>
where
F: Fn(Arc<dyn BlobService>, Arc<dyn DirectoryService>, Arc<dyn PathInfoService>),
{
setup_and_mount_with_listing(mountpoint, setup_fn, false)
}
fn setup_and_mount_with_listing<P: AsRef<Path>, F>(
mountpoint: P,
setup_fn: F,
list_root: bool,
) -> Result<fuser::BackgroundSession, std::io::Error>
where
F: Fn(Arc<dyn BlobService>, Arc<dyn DirectoryService>, Arc<dyn PathInfoService>),
{
@ -38,7 +49,12 @@ where
path_info_service.clone(),
);
let fs = FUSE::new(blob_service, directory_service, path_info_service);
let fs = FUSE::new(
blob_service,
directory_service,
path_info_service,
list_root,
);
fuser::spawn_mount2(fs, mountpoint, &[])
}
@ -280,6 +296,34 @@ fn root() {
fuser_session.join()
}
/// Ensure listing the root is allowed if configured explicitly
#[test]
fn root_with_listing() {
// https://plume.benboeckel.net/~/JustAnotherBlog/skipping-tests-in-rust
if !std::path::Path::new("/dev/fuse").exists() {
eprintln!("skipping test");
return;
}
let tmpdir = TempDir::new().unwrap();
let fuser_session =
setup_and_mount_with_listing(tmpdir.path(), populate_blob_a, true).expect("must succeed");
{
// read_dir succeeds, but getting the first element will fail.
let mut it = fs::read_dir(tmpdir).expect("must succeed");
let e = it.next().expect("must be some").expect("must succeed");
let metadata = e.metadata().expect("must succeed");
assert!(metadata.is_file());
assert!(metadata.permissions().readonly());
assert_eq!(fixtures::BLOB_A.len() as u64, metadata.len());
}
fuser_session.join()
}
/// Ensure we can stat a file at the root
#[test]
fn stat_file_at_root() {