mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore(tvix): move store/fs to castore/fs

Browse source 
With the recent introduction of the RootNodes trait, there's nothing in
the fs module pulling in tvix-store dependencies, so it can live in
tvix-castore.

This allows other crates to make use of TvixStoreFS, without having to
pull in tvix-store.

For example, a tvix-build using a fuse mountpoint at /nix/store doesn't
need a PathInfoService to hold the root nodes that should be present,
but just a list.

tvix-store now has a pathinfoservice/fs module, which contains the
necessary glue logic to implement the RootNodes trait for a
PathInfoService.

To satisfy Rust orphan rules for trait implementations, we had to add a
small wrapper struct. It's mostly hidden away by the make_fs helper
function returning a TvixStoreFs.

It can't be entirely private, as its still leaking into the concrete
type of TvixStoreFS.

tvix-store still has `fuse` and `virtiofs` features, but they now simply
enable these features in the `tvix-castore` crate they depend on.

The tests for the fuse functionality stay in tvix-store for now, as
they populate the root nodes through a PathInfoService.

Once above mentioned "list of root nodes" implementation exists, we
might want to shuffle this around one more time.

Fixes b/341.

Change-Id: I989f664827a5a361b23b34368d242d10c157c756
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10378
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Reviewed-by: sterni <sternenseemann@systemli.org>
This commit is contained in:
Florian Klink 2023-12-17 00:16:22 +02:00 committed by flokli
parent 52cad86195
commit a5167c508c
18 changed files with 257 additions and 199 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
tvix/Cargo.lock generated
View file

@ -3108,9 +3108,12 @@ dependencies = [
"bstr",
"bytes",
"data-encoding",
"fuse-backend-rs",
"futures",
"hex-literal",
"lazy_static",
"libc",
"parking_lot 0.12.1",
"pin-project-lite",
"prost",
"prost-build",
@ -3128,6 +3131,12 @@ dependencies = [
"tower",
"tracing",
"url",
"vhost",
"vhost-user-backend",
"virtio-bindings 0.2.1",
"virtio-queue",
"vm-memory",
"vmm-sys-util",
"walkdir",
]
@ -3233,12 +3242,10 @@ dependencies = [
"clap",
"count-write",
"data-encoding",
"fuse-backend-rs",
"futures",
"lazy_static",
"libc",
"nix-compat",
"parking_lot 0.12.1",
"pin-project-lite",
"prost",
"prost-build",
@ -3261,12 +3268,6 @@ dependencies = [
"tracing-subscriber",
"tvix-castore",
"url",
"vhost",
"vhost-user-backend",
"virtio-bindings 0.2.1",
"virtio-queue",
"vm-memory",
"vmm-sys-util",
"walkdir",
"xz2",
]

104
tvix/Cargo.nix
View file

@ -9608,6 +9608,11 @@ rec {
name = "data-encoding";
packageId = "data-encoding";
}
{
name = "fuse-backend-rs";
packageId = "fuse-backend-rs";
optional = true;
}
{
name = "futures";
packageId = "futures";
@ -9616,6 +9621,15 @@ rec {
name = "lazy_static";
packageId = "lazy_static";
}
{
name = "libc";
packageId = "libc";
optional = true;
}
{
name = "parking_lot";
packageId = "parking_lot 0.12.1";
}
{
name = "pin-project-lite";
packageId = "pin-project-lite";
@ -9668,6 +9682,36 @@ rec {
name = "url";
packageId = "url";
}
{
name = "vhost";
packageId = "vhost";
optional = true;
}
{
name = "vhost-user-backend";
packageId = "vhost-user-backend";
optional = true;
}
{
name = "virtio-bindings";
packageId = "virtio-bindings 0.2.1";
optional = true;
}
{
name = "virtio-queue";
packageId = "virtio-queue";
optional = true;
}
{
name = "vm-memory";
packageId = "vm-memory";
optional = true;
}
{
name = "vmm-sys-util";
packageId = "vmm-sys-util";
optional = true;
}
{
name = "walkdir";
packageId = "walkdir";
@ -9702,9 +9746,12 @@ rec {
}
];
features = {
"fs" = [ "dep:libc" "dep:fuse-backend-rs" ];
"fuse" = [ "fs" ];
"tonic-reflection" = [ "dep:tonic-reflection" ];
"virtiofs" = [ "fs" "dep:vhost" "dep:vhost-user-backend" "dep:virtio-queue" "dep:vm-memory" "dep:vmm-sys-util" "dep:virtio-bindings" "fuse-backend-rs?/vhost-user-fs" "fuse-backend-rs?/virtiofs" ];
};
resolvedDefaultFeatures = [ "default" "tonic-reflection" ];
resolvedDefaultFeatures = [ "default" "fs" "fuse" "tonic-reflection" "virtiofs" ];
};
"tvix-cli" = rec {
crateName = "tvix-cli";
@ -10110,11 +10157,6 @@ rec {
name = "data-encoding";
packageId = "data-encoding";
}
{
name = "fuse-backend-rs";
packageId = "fuse-backend-rs";
optional = true;
}
{
name = "futures";
packageId = "futures";
@ -10123,20 +10165,11 @@ rec {
name = "lazy_static";
packageId = "lazy_static";
}
{
name = "libc";
packageId = "libc";
optional = true;
}
{
name = "nix-compat";
packageId = "nix-compat";
features = [ "async" ];
}
{
name = "parking_lot";
packageId = "parking_lot 0.12.1";
}
{
name = "pin-project-lite";
packageId = "pin-project-lite";
@ -10214,36 +10247,6 @@ rec {
name = "url";
packageId = "url";
}
{
name = "vhost";
packageId = "vhost";
optional = true;
}
{
name = "vhost-user-backend";
packageId = "vhost-user-backend";
optional = true;
}
{
name = "virtio-bindings";
packageId = "virtio-bindings 0.2.1";
optional = true;
}
{
name = "virtio-queue";
packageId = "virtio-queue";
optional = true;
}
{
name = "vm-memory";
packageId = "vm-memory";
optional = true;
}
{
name = "vmm-sys-util";
packageId = "vmm-sys-util";
optional = true;
}
{
name = "walkdir";
packageId = "walkdir";
@ -10264,6 +10267,10 @@ rec {
}
];
devDependencies = [
{
name = "libc";
packageId = "libc";
}
{
name = "tempfile";
packageId = "tempfile";
@ -10279,12 +10286,11 @@ rec {
];
features = {
"default" = [ "fuse" "tonic-reflection" ];
"fs" = [ "dep:libc" "dep:fuse-backend-rs" ];
"fuse" = [ "fs" ];
"fuse" = [ "tvix-castore/fuse" ];
"tonic-reflection" = [ "dep:tonic-reflection" "tvix-castore/tonic-reflection" ];
"virtiofs" = [ "fs" "dep:vhost" "dep:vhost-user-backend" "dep:virtio-queue" "dep:vm-memory" "dep:vmm-sys-util" "dep:virtio-bindings" "fuse-backend-rs?/vhost-user-fs" "fuse-backend-rs?/virtiofs" ];
"virtiofs" = [ "tvix-castore/virtiofs" ];
};
resolvedDefaultFeatures = [ "default" "fs" "fuse" "tonic-reflection" "virtiofs" ];
resolvedDefaultFeatures = [ "default" "fuse" "tonic-reflection" "virtiofs" ];
};
"typenum" = rec {
crateName = "typenum";

46
tvix/castore/Cargo.toml
View file

@ -10,6 +10,7 @@ bytes = "1.4.0"
data-encoding = "2.3.3"
futures = "0.3.28"
lazy_static = "1.4.0"
parking_lot = "0.12.1"
pin-project-lite = "0.2.13"
prost = "0.12.1"
sled = { version = "0.34.7" }
@ -25,10 +26,42 @@ walkdir = "2.4.0"
bstr = "1.6.0"
async-tempfile = "0.4.0"
[dependencies.fuse-backend-rs]
optional = true
version = "0.11.0"
[dependencies.libc]
optional = true
version = "0.2.144"
[dependencies.tonic-reflection]
optional = true
version = "0.10.2"
[dependencies.vhost]
optional = true
version = "0.6"
[dependencies.vhost-user-backend]
optional = true
version = "0.8"
[dependencies.virtio-queue]
optional = true
version = "0.7"
[dependencies.vm-memory]
optional = true
version = "0.10"
[dependencies.vmm-sys-util]
optional = true
version = "0.11"
[dependencies.virtio-bindings]
optional = true
version = "0.2.1"
[build-dependencies]
prost-build = "0.12.1"
tonic-build = "0.10.2"
@ -41,4 +74,17 @@ hex-literal = "0.4.1"
[features]
default = []
fs = ["dep:libc", "dep:fuse-backend-rs"]
virtiofs = [
"fs",
"dep:vhost",
"dep:vhost-user-backend",
"dep:virtio-queue",
"dep:vm-memory",
"dep:vmm-sys-util",
"dep:virtio-bindings",
"fuse-backend-rs?/vhost-user-fs", # impl FsCacheReqHandler for SlaveFsCacheReq
"fuse-backend-rs?/virtiofs",
]
fuse = ["fs"]
tonic-reflection = ["dep:tonic-reflection"]

0
tvix/store/src/fs/file_attr.rs → tvix/castore/src/fs/file_attr.rs
View file

0
tvix/store/src/fs/fuse.rs → tvix/castore/src/fs/fuse.rs
View file

4
tvix/store/src/fs/inode_tracker.rs → tvix/castore/src/fs/inode_tracker.rs
View file

@ -1,7 +1,7 @@
use std::{collections::HashMap, sync::Arc};
use super::inodes::{DirectoryInodeData, InodeData};
use tvix_castore::B3Digest;
use crate::B3Digest;
/// InodeTracker keeps track of inodes, stores data being these inodes and deals
/// with inode allocation.
@ -132,7 +132,7 @@ impl InodeTracker {
#[cfg(test)]
mod tests {
use crate::tests::fixtures;
use crate::fixtures;
use super::InodeData;
use super::InodeTracker;

6
tvix/store/src/fs/inodes.rs → tvix/castore/src/fs/inodes.rs
View file

@ -1,7 +1,7 @@
//! This module contains all the data structures used to track information
//! about inodes, which present tvix-store nodes in a filesystem.
use tvix_castore::proto as castorepb;
use tvix_castore::B3Digest;
//! about inodes, which present tvix-castore nodes in a filesystem.
use crate::proto as castorepb;
use crate::B3Digest;
#[derive(Clone, Debug)]
pub enum InodeData {

20
tvix/store/src/fs/mod.rs → tvix/castore/src/fs/mod.rs
View file

@ -9,9 +9,13 @@ pub mod fuse;
#[cfg(feature = "virtiofs")]
pub mod virtiofs;
#[cfg(test)]
mod tests;
use crate::proto as castorepb;
use crate::{
blobservice::{BlobReader, BlobService},
directoryservice::DirectoryService,
proto::{node::Node, NamedNode},
B3Digest,
};
use fuse_backend_rs::abi::fuse_abi::stat64;
use fuse_backend_rs::api::filesystem::{Context, FileSystem, FsOptions, ROOT_ID};
use futures::StreamExt;
@ -29,15 +33,8 @@ use tokio::{
sync::mpsc,
};
use tracing::{debug, info_span, instrument, warn};
use tvix_castore::proto as castorepb;
use tvix_castore::{
blobservice::{BlobReader, BlobService},
directoryservice::DirectoryService,
proto::{node::Node, NamedNode},
B3Digest,
};
use self::root_nodes::RootNodes;
pub use self::root_nodes::RootNodes;
use self::{
file_attr::{gen_file_attr, ROOT_FILE_ATTR},
inode_tracker::InodeTracker,
@ -75,6 +72,7 @@ use self::{
/// merkle structure is a DAG, not a tree, this also means we can't do "bucketed
/// allocation", aka reserve Directory.size inodes for each directory node we
/// explore.
/// Tests for this live in the tvix-store crate.
pub struct TvixStoreFs<BS, DS, RN> {
blob_service: BS,
directory_service: DS,

18
tvix/castore/src/fs/root_nodes.rs Normal file
View file

@ -0,0 +1,18 @@
use std::pin::Pin;
use crate::{proto::node::Node, Error};
use futures::Stream;
use tonic::async_trait;
/// Provides an interface for looking up root nodes in tvix-castore by given
/// a lookup key (usually the basename), and optionally allow a listing.
#[async_trait]
pub trait RootNodes: Send + Sync {
/// Looks up a root CA node based on the basename of the node in the root
/// directory of the filesystem.
async fn get_by_basename(&self, name: &[u8]) -> Result<Option<Node>, Error>;
/// Lists all root CA nodes in the filesystem. An error can be returned
/// in case listing is not allowed
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<Node, Error>> + Send>>;
}

0
tvix/store/src/fs/virtiofs.rs → tvix/castore/src/fs/virtiofs.rs
View file

4
tvix/castore/src/lib.rs
View file

@ -4,6 +4,10 @@ mod errors;
pub mod blobservice;
pub mod directoryservice;
pub mod fixtures;
#[cfg(feature = "fs")]
pub mod fs;
pub mod import;
pub mod proto;
pub mod tonic;

51
tvix/store/Cargo.toml
View file

@ -14,7 +14,6 @@ data-encoding = "2.3.3"
futures = "0.3.28"
lazy_static = "1.4.0"
nix-compat = { path = "../nix-compat", features = ["async"] }
parking_lot = "0.12.1"
pin-project-lite = "0.2.13"
prost = "0.12.1"
sha2 = "0.10.6"
@ -35,42 +34,10 @@ async-recursion = "1.0.5"
reqwest = { version = "0.11.22", features = ["rustls-tls", "stream"], default-features = false }
xz2 = "0.1.7"
[dependencies.fuse-backend-rs]
optional = true
version = "0.11.0"
[dependencies.vhost]
optional = true
version = "0.6"
[dependencies.vhost-user-backend]
optional = true
version = "0.8"
[dependencies.virtio-queue]
optional = true
version = "0.7"
[dependencies.vm-memory]
optional = true
version = "0.10"
[dependencies.vmm-sys-util]
optional = true
version = "0.11"
[dependencies.virtio-bindings]
optional = true
version = "0.2.1"
[dependencies.tonic-reflection]
optional = true
version = "0.10.2"
[dependencies.libc]
optional = true
version = "0.2.144"
[build-dependencies]
prost-build = "0.12.1"
tonic-build = "0.10.2"
@ -80,19 +47,11 @@ test-case = "2.2.2"
tempfile = "3.3.0"
tokio-retry = "0.3.0"
[dev-dependencies.libc]
version = "0.2.144"
[features]
default = ["fuse", "tonic-reflection"]
fs = ["dep:libc", "dep:fuse-backend-rs"]
virtiofs = [
"fs",
"dep:vhost",
"dep:vhost-user-backend",
"dep:virtio-queue",
"dep:vm-memory",
"dep:vmm-sys-util",
"dep:virtio-bindings",
"fuse-backend-rs?/vhost-user-fs", # impl FsCacheReqHandler for SlaveFsCacheReq
"fuse-backend-rs?/virtiofs",
]
fuse = ["fs"]
fuse = ["tvix-castore/fuse"]
virtiofs = ["tvix-castore/virtiofs"]
tonic-reflection = ["dep:tonic-reflection", "tvix-castore/tonic-reflection"]

14
tvix/store/src/bin/tvix-store.rs
View file

@ -29,14 +29,14 @@ use tvix_store::proto::GRPCPathInfoServiceWrapper;
use tvix_store::proto::NarInfo;
use tvix_store::proto::PathInfo;
#[cfg(feature = "fs")]
use tvix_store::fs::TvixStoreFs;
#[cfg(any(feature = "fuse", feature = "virtiofs"))]
use tvix_store::pathinfoservice::make_fs;
#[cfg(feature = "fuse")]
use tvix_store::fs::fuse::FuseDaemon;
use tvix_castore::fs::fuse::FuseDaemon;
#[cfg(feature = "virtiofs")]
use tvix_store::fs::virtiofs::start_virtiofs_daemon;
use tvix_castore::fs::virtiofs::start_virtiofs_daemon;
#[cfg(feature = "tonic-reflection")]
use tvix_castore::proto::FILE_DESCRIPTOR_SET as CASTORE_FILE_DESCRIPTOR_SET;
@ -365,7 +365,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let path_info_service: Arc<dyn PathInfoService> = path_info_service.into();
let mut fuse_daemon = tokio::task::spawn_blocking(move || {
let f = TvixStoreFs::new(
let fs = make_fs(
blob_service,
directory_service,
path_info_service,
@ -373,7 +373,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
);
info!("mounting tvix-store on {:?}", &dest);
FuseDaemon::new(f, &dest, threads)
FuseDaemon::new(fs, &dest, threads)
})
.await??;
@ -409,7 +409,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let path_info_service: Arc<dyn PathInfoService> = path_info_service.into();
tokio::task::spawn_blocking(move || {
let fs = TvixStoreFs::new(
let fs = make_fs(
blob_service,
directory_service,
path_info_service,

61
tvix/store/src/fs/root_nodes.rs
View file

@ -1,61 +0,0 @@
use std::{ops::Deref, pin::Pin};
use futures::{Stream, StreamExt};
use nix_compat::store_path::StorePath;
use tonic::async_trait;
use tvix_castore::{proto::node::Node, Error};
use crate::pathinfoservice::PathInfoService;
/// Provides an interface for looking up root nodes in tvix-castore by given
/// a lookup key (usually the basename), and optionally allow a listing.
///
#[async_trait]
pub trait RootNodes: Send + Sync {
/// Looks up a root CA node based on the basename of the node in the root
/// directory of the filesystem.
async fn get_by_basename(&self, name: &[u8]) -> Result<Option<Node>, Error>;
/// Lists all root CA nodes in the filesystem. An error can be returned
/// in case listing is not allowed
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<Node, Error>> + Send>>;
}
/// Implements root node lookup for any [PathInfoService]. This represents a flat
/// directory structure like /nix/store where each entry in the root filesystem
/// directory corresponds to a CA node.
#[async_trait]
impl<T> RootNodes for T
where
T: Deref<Target = dyn PathInfoService> + Send + Sync,
{
async fn get_by_basename(&self, name: &[u8]) -> Result<Option<Node>, Error> {
let Ok(store_path) = StorePath::from_bytes(name) else {
return Ok(None);
};
Ok(self
.deref()
.get(*store_path.digest())
.await?
.map(|path_info| {
path_info
.node
.expect("missing root node")
.node
.expect("empty node")
}))
}
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<Node, Error>> + Send>> {
Box::pin(self.deref().list().map(|result| {
result.map(|path_info| {
path_info
.node
.expect("missing root node")
.node
.expect("empty node")
})
}))
}
}

3
tvix/store/src/lib.rs
View file

@ -1,6 +1,3 @@
#[cfg(feature = "fs")]
pub mod fs;
pub mod nar;
pub mod pathinfoservice;
pub mod proto;

84
tvix/store/src/pathinfoservice/fs/mod.rs Normal file
View file

@ -0,0 +1,84 @@
use futures::Stream;
use futures::StreamExt;
use std::ops::Deref;
use std::pin::Pin;
use tonic::async_trait;
use tvix_castore::fs::{RootNodes, TvixStoreFs};
use tvix_castore::proto as castorepb;
use tvix_castore::Error;
use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService};
use super::PathInfoService;
#[cfg(test)]
mod tests;
/// Helper to construct a [TvixStoreFs] from a [BlobService], [DirectoryService]
/// and [PathInfoService].
/// This avoids users to have to interact with the wrapper struct directly, as
/// it leaks into the type signature of TvixStoreFS.
pub fn make_fs<BS, DS, PS>(
blob_service: BS,
directory_service: DS,
path_info_service: PS,
list_root: bool,
) -> TvixStoreFs<BS, DS, RootNodesWrapper<PS>>
where
BS: Deref<Target = dyn BlobService> + Send + Clone + 'static,
DS: Deref<Target = dyn DirectoryService> + Send + Clone + 'static,
PS: Deref<Target = dyn PathInfoService> + Send + Sync + Clone + 'static,
{
TvixStoreFs::new(
blob_service,
directory_service,
RootNodesWrapper(path_info_service),
list_root,
)
}
/// Wrapper to satisfy Rust's orphan rules for trait implementations, as
/// RootNodes is coming from the [tvix-castore] crate.
#[doc(hidden)]
#[derive(Clone, Debug)]
pub struct RootNodesWrapper<T>(pub(crate) T);
/// Implements root node lookup for any [PathInfoService]. This represents a flat
/// directory structure like /nix/store where each entry in the root filesystem
/// directory corresponds to a CA node.
#[cfg(any(feature = "fuse", feature = "virtiofs"))]
#[async_trait]
impl<T> RootNodes for RootNodesWrapper<T>
where
T: Deref<Target = dyn PathInfoService> + Send + Sync,
{
async fn get_by_basename(&self, name: &[u8]) -> Result<Option<castorepb::node::Node>, Error> {
let Ok(store_path) = nix_compat::store_path::StorePath::from_bytes(name) else {
return Ok(None);
};
Ok(self
.0
.deref()
.get(*store_path.digest())
.await?
.map(|path_info| {
path_info
.node
.expect("missing root node")
.node
.expect("empty node")
}))
}
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<castorepb::node::Node, Error>> + Send>> {
Box::pin(self.0.deref().list().map(|result| {
result.map(|path_info| {
path_info
.node
.expect("missing root node")
.node
.expect("empty node")
})
}))
}
}

18
tvix/store/src/fs/tests.rs → tvix/store/src/pathinfoservice/fs/tests.rs
View file

@ -1,22 +1,22 @@
use crate::pathinfoservice::PathInfoService;
use crate::proto::PathInfo;
use crate::tests::fixtures;
use crate::tests::utils::{gen_blob_service, gen_directory_service, gen_pathinfo_service};
use futures::StreamExt;
use std::io::Cursor;
use std::os::unix::prelude::MetadataExt;
use std::path::Path;
use std::sync::Arc;
use tempfile::TempDir;
use tokio::{fs, io};
use tokio_stream::wrappers::ReadDirStream;
use tvix_castore::blobservice::BlobService;
use tvix_castore::directoryservice::DirectoryService;
use tempfile::TempDir;
use crate::fs::{fuse::FuseDaemon, TvixStoreFs};
use crate::pathinfoservice::PathInfoService;
use crate::proto::PathInfo;
use crate::tests::fixtures;
use crate::tests::utils::{gen_blob_service, gen_directory_service, gen_pathinfo_service};
use tvix_castore::fs::fuse::FuseDaemon;
use tvix_castore::proto as castorepb;
use super::make_fs;
const BLOB_A_NAME: &str = "00000000000000000000000000000000-test";
const BLOB_B_NAME: &str = "55555555555555555555555555555555-test";
const HELLOWORLD_BLOB_NAME: &str = "66666666666666666666666666666666-test";
@ -44,7 +44,7 @@ fn do_mount<P: AsRef<Path>>(
mountpoint: P,
list_root: bool,
) -> io::Result<FuseDaemon> {
let fs = TvixStoreFs::new(
let fs = make_fs(
blob_service,
directory_service,
path_info_service,

6
tvix/store/src/pathinfoservice/mod.rs
View file

@ -4,6 +4,9 @@ mod memory;
mod nix_http;
mod sled;
#[cfg(any(feature = "fuse", feature = "virtiofs"))]
mod fs;
use futures::Stream;
use std::pin::Pin;
use tonic::async_trait;
@ -18,6 +21,9 @@ pub use self::memory::MemoryPathInfoService;
pub use self::nix_http::NixHTTPPathInfoService;
pub use self::sled::SledPathInfoService;
#[cfg(any(feature = "fuse", feature = "virtiofs"))]
pub use self::fs::make_fs;
/// The base trait all PathInfo services need to implement.
#[async_trait]
pub trait PathInfoService: Send + Sync {