feat(tvix/store/bin): add progress bar infrastructure
This adds the tracing-indicatif crate, and configures it as a layer in our tracing_subscriber pipeline to emit progress for every span that's configured so. It also moves from using std::io::stderr to write logs to using their writer, to avoid clobbering output. Progress bar styles are defined in a lazy_static, moving this into a general tracing is left for later. This adds some usage of this to the `imports` and `copy` commands. The output can still be improved a bit - we should probably split each task up into a smaller (instrumented) helper functions, so we can create a progress bar for each task. Change-Id: I59a1915aa4e0caa89c911632dec59c4cbeba1b89 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11747 Reviewed-by: flokli <flokli@flokli.de> Reviewed-by: Simon Hauser <simon.hauser@helsinki-systems.de> Tested-by: BuildkiteCI Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink
flokli
parent
9b77ce9f8f
commit
20513e7a52
8 changed files with 452 additions and 19 deletions
95
tvix/Cargo.lock
generated
95
tvix/Cargo.lock
generated
|
|
@ -677,6 +677,19 @@ dependencies = [
|
|||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.15.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
|
||||
dependencies = [
|
||||
"encode_unicode",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"unicode-width",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.9.6"
|
||||
|
|
@ -1007,6 +1020,12 @@ version = "1.9.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
|
||||
|
||||
[[package]]
|
||||
name = "encode_unicode"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.33"
|
||||
|
|
@ -1664,6 +1683,20 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indicatif"
|
||||
version = "0.17.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3"
|
||||
dependencies = [
|
||||
"console",
|
||||
"instant",
|
||||
"number_prefix",
|
||||
"portable-atomic",
|
||||
"unicode-width",
|
||||
"vt100",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.12"
|
||||
|
|
@ -2137,6 +2170,12 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "number_prefix"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.32.2"
|
||||
|
|
@ -2476,6 +2515,12 @@ dependencies = [
|
|||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
|
|
@ -3980,6 +4025,18 @@ dependencies = [
|
|||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-indicatif"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "069580424efe11d97c3fef4197fa98c004fa26672cc71ad8770d224e23b1951d"
|
||||
dependencies = [
|
||||
"indicatif",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-log"
|
||||
version = "0.2.0"
|
||||
|
|
@ -4073,6 +4130,7 @@ dependencies = [
|
|||
"fuse-backend-rs",
|
||||
"futures",
|
||||
"hex-literal",
|
||||
"indicatif",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"object_store",
|
||||
|
|
@ -4099,6 +4157,7 @@ dependencies = [
|
|||
"tonic-reflection",
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-indicatif",
|
||||
"url",
|
||||
"vhost",
|
||||
"vhost-user-backend",
|
||||
|
|
@ -4210,6 +4269,7 @@ dependencies = [
|
|||
"tokio-tar",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"tracing-indicatif",
|
||||
"tvix-build",
|
||||
"tvix-castore",
|
||||
"tvix-eval",
|
||||
|
|
@ -4244,6 +4304,7 @@ dependencies = [
|
|||
"count-write",
|
||||
"data-encoding",
|
||||
"futures",
|
||||
"indicatif",
|
||||
"lazy_static",
|
||||
"lru",
|
||||
"nix-compat",
|
||||
|
|
@ -4275,6 +4336,7 @@ dependencies = [
|
|||
"tonic-reflection",
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-indicatif",
|
||||
"tracing-opentelemetry",
|
||||
"tracing-subscriber",
|
||||
"tvix-castore",
|
||||
|
|
@ -4488,6 +4550,39 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "vt100"
|
||||
version = "0.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"log",
|
||||
"unicode-width",
|
||||
"vte",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "vte"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"utf8parse",
|
||||
"vte_generate_state_changes",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "vte_generate_state_changes"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wait-timeout"
|
||||
version = "0.2.0"
|
||||
|
|
|
|||
270
tvix/Cargo.nix
270
tvix/Cargo.nix
|
|
@ -2126,6 +2126,47 @@ rec {
|
|||
};
|
||||
resolvedDefaultFeatures = [ "default" "std" ];
|
||||
};
|
||||
"console" = rec {
|
||||
crateName = "console";
|
||||
version = "0.15.8";
|
||||
edition = "2018";
|
||||
sha256 = "1sz4nl9nz8pkmapqni6py7jxzi7nzqjxzb3ya4kxvmkb0zy867qf";
|
||||
authors = [
|
||||
"Armin Ronacher <armin.ronacher@active-4.com>"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "encode_unicode";
|
||||
packageId = "encode_unicode";
|
||||
target = { target, features }: (target."windows" or false);
|
||||
}
|
||||
{
|
||||
name = "lazy_static";
|
||||
packageId = "lazy_static";
|
||||
}
|
||||
{
|
||||
name = "libc";
|
||||
packageId = "libc";
|
||||
}
|
||||
{
|
||||
name = "unicode-width";
|
||||
packageId = "unicode-width";
|
||||
optional = true;
|
||||
}
|
||||
{
|
||||
name = "windows-sys";
|
||||
packageId = "windows-sys 0.52.0";
|
||||
target = { target, features }: (target."windows" or false);
|
||||
features = [ "Win32_Foundation" "Win32_System_Console" "Win32_Storage_FileSystem" "Win32_UI_Input_KeyboardAndMouse" ];
|
||||
}
|
||||
];
|
||||
features = {
|
||||
"default" = [ "unicode-width" "ansi-parsing" ];
|
||||
"unicode-width" = [ "dep:unicode-width" ];
|
||||
"windows-console-colors" = [ "ansi-parsing" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "ansi-parsing" "unicode-width" ];
|
||||
};
|
||||
"const-oid" = rec {
|
||||
crateName = "const-oid";
|
||||
version = "0.9.6";
|
||||
|
|
@ -3072,6 +3113,21 @@ rec {
|
|||
};
|
||||
resolvedDefaultFeatures = [ "default" "use_std" ];
|
||||
};
|
||||
"encode_unicode" = rec {
|
||||
crateName = "encode_unicode";
|
||||
version = "0.3.6";
|
||||
edition = "2015";
|
||||
sha256 = "07w3vzrhxh9lpjgsg2y5bwzfar2aq35mdznvcp3zjl0ssj7d4mx3";
|
||||
authors = [
|
||||
"Torbjørn Birch Moltu <t.b.moltu@lyse.net>"
|
||||
];
|
||||
features = {
|
||||
"ascii" = [ "dep:ascii" ];
|
||||
"clippy" = [ "dep:clippy" ];
|
||||
"default" = [ "std" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "default" "std" ];
|
||||
};
|
||||
"encoding_rs" = rec {
|
||||
crateName = "encoding_rs";
|
||||
version = "0.8.33";
|
||||
|
|
@ -5057,6 +5113,55 @@ rec {
|
|||
};
|
||||
resolvedDefaultFeatures = [ "default" "serde" "std" ];
|
||||
};
|
||||
"indicatif" = rec {
|
||||
crateName = "indicatif";
|
||||
version = "0.17.8";
|
||||
edition = "2021";
|
||||
sha256 = "18xyqxw9i5x4sbpzckhfz3nm984iq9r7nbi2lk76nz888n7mlfkn";
|
||||
dependencies = [
|
||||
{
|
||||
name = "console";
|
||||
packageId = "console";
|
||||
usesDefaultFeatures = false;
|
||||
features = [ "ansi-parsing" ];
|
||||
}
|
||||
{
|
||||
name = "instant";
|
||||
packageId = "instant";
|
||||
target = { target, features }: ("wasm32" == target."arch" or null);
|
||||
}
|
||||
{
|
||||
name = "number_prefix";
|
||||
packageId = "number_prefix";
|
||||
}
|
||||
{
|
||||
name = "portable-atomic";
|
||||
packageId = "portable-atomic";
|
||||
}
|
||||
{
|
||||
name = "unicode-width";
|
||||
packageId = "unicode-width";
|
||||
optional = true;
|
||||
}
|
||||
{
|
||||
name = "vt100";
|
||||
packageId = "vt100";
|
||||
optional = true;
|
||||
}
|
||||
];
|
||||
features = {
|
||||
"default" = [ "unicode-width" "console/unicode-width" ];
|
||||
"futures" = [ "dep:futures-core" ];
|
||||
"improved_unicode" = [ "unicode-segmentation" "unicode-width" "console/unicode-width" ];
|
||||
"in_memory" = [ "vt100" ];
|
||||
"rayon" = [ "dep:rayon" ];
|
||||
"tokio" = [ "dep:tokio" ];
|
||||
"unicode-segmentation" = [ "dep:unicode-segmentation" ];
|
||||
"unicode-width" = [ "dep:unicode-width" ];
|
||||
"vt100" = [ "dep:vt100" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "default" "in_memory" "unicode-width" "vt100" ];
|
||||
};
|
||||
"instant" = rec {
|
||||
crateName = "instant";
|
||||
version = "0.1.12";
|
||||
|
|
@ -6440,6 +6545,19 @@ rec {
|
|||
];
|
||||
|
||||
};
|
||||
"number_prefix" = rec {
|
||||
crateName = "number_prefix";
|
||||
version = "0.4.0";
|
||||
edition = "2015";
|
||||
sha256 = "1wvh13wvlajqxkb1filsfzbrnq0vrmrw298v2j3sy82z1rm282w3";
|
||||
authors = [
|
||||
"Benjamin Sago <ogham@bsago.me>"
|
||||
];
|
||||
features = {
|
||||
"default" = [ "std" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "default" "std" ];
|
||||
};
|
||||
"object" = rec {
|
||||
crateName = "object";
|
||||
version = "0.32.2";
|
||||
|
|
@ -7535,6 +7653,18 @@ rec {
|
|||
];
|
||||
|
||||
};
|
||||
"portable-atomic" = rec {
|
||||
crateName = "portable-atomic";
|
||||
version = "1.6.0";
|
||||
edition = "2018";
|
||||
sha256 = "1h77x9qx7pns0d66vdrmdbmwpi7586h7ysnkdnhrn5mwi2cyyw3i";
|
||||
features = {
|
||||
"critical-section" = [ "dep:critical-section" ];
|
||||
"default" = [ "fallback" ];
|
||||
"serde" = [ "dep:serde" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "default" "fallback" ];
|
||||
};
|
||||
"powerfmt" = rec {
|
||||
crateName = "powerfmt";
|
||||
version = "0.2.0";
|
||||
|
|
@ -12414,6 +12544,33 @@ rec {
|
|||
};
|
||||
resolvedDefaultFeatures = [ "default" "pin-project" "std" "std-future" ];
|
||||
};
|
||||
"tracing-indicatif" = rec {
|
||||
crateName = "tracing-indicatif";
|
||||
version = "0.3.6";
|
||||
edition = "2021";
|
||||
sha256 = "07cmn4ilw8hdfzc1mirccwkgl160k3x9fhgg7xydj4gy9r181586";
|
||||
libName = "tracing_indicatif";
|
||||
dependencies = [
|
||||
{
|
||||
name = "indicatif";
|
||||
packageId = "indicatif";
|
||||
features = [ "in_memory" ];
|
||||
}
|
||||
{
|
||||
name = "tracing";
|
||||
packageId = "tracing";
|
||||
}
|
||||
{
|
||||
name = "tracing-core";
|
||||
packageId = "tracing-core";
|
||||
}
|
||||
{
|
||||
name = "tracing-subscriber";
|
||||
packageId = "tracing-subscriber";
|
||||
}
|
||||
];
|
||||
|
||||
};
|
||||
"tracing-log" = rec {
|
||||
crateName = "tracing-log";
|
||||
version = "0.2.0";
|
||||
|
|
@ -12828,6 +12985,10 @@ rec {
|
|||
name = "futures";
|
||||
packageId = "futures";
|
||||
}
|
||||
{
|
||||
name = "indicatif";
|
||||
packageId = "indicatif";
|
||||
}
|
||||
{
|
||||
name = "lazy_static";
|
||||
packageId = "lazy_static";
|
||||
|
|
@ -12915,6 +13076,10 @@ rec {
|
|||
name = "tracing";
|
||||
packageId = "tracing";
|
||||
}
|
||||
{
|
||||
name = "tracing-indicatif";
|
||||
packageId = "tracing-indicatif";
|
||||
}
|
||||
{
|
||||
name = "url";
|
||||
packageId = "url";
|
||||
|
|
@ -13386,6 +13551,10 @@ rec {
|
|||
name = "tracing";
|
||||
packageId = "tracing";
|
||||
}
|
||||
{
|
||||
name = "tracing-indicatif";
|
||||
packageId = "tracing-indicatif";
|
||||
}
|
||||
{
|
||||
name = "tvix-build";
|
||||
packageId = "tvix-build";
|
||||
|
|
@ -13548,6 +13717,10 @@ rec {
|
|||
name = "futures";
|
||||
packageId = "futures";
|
||||
}
|
||||
{
|
||||
name = "indicatif";
|
||||
packageId = "indicatif";
|
||||
}
|
||||
{
|
||||
name = "lazy_static";
|
||||
packageId = "lazy_static";
|
||||
|
|
@ -13662,6 +13835,10 @@ rec {
|
|||
name = "tracing";
|
||||
packageId = "tracing";
|
||||
}
|
||||
{
|
||||
name = "tracing-indicatif";
|
||||
packageId = "tracing-indicatif";
|
||||
}
|
||||
{
|
||||
name = "tracing-opentelemetry";
|
||||
packageId = "tracing-opentelemetry";
|
||||
|
|
@ -14247,6 +14424,97 @@ rec {
|
|||
"with-serde" = [ "serde" "serde_derive" ];
|
||||
};
|
||||
};
|
||||
"vt100" = rec {
|
||||
crateName = "vt100";
|
||||
version = "0.15.2";
|
||||
edition = "2021";
|
||||
sha256 = "1pklc8y984axmxr0cd363srr2d27wd5rj15xlcmkjznvy0xqdkc4";
|
||||
authors = [
|
||||
"Jesse Luehrs <doy@tozt.net>"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "itoa";
|
||||
packageId = "itoa";
|
||||
}
|
||||
{
|
||||
name = "log";
|
||||
packageId = "log";
|
||||
}
|
||||
{
|
||||
name = "unicode-width";
|
||||
packageId = "unicode-width";
|
||||
}
|
||||
{
|
||||
name = "vte";
|
||||
packageId = "vte";
|
||||
}
|
||||
];
|
||||
devDependencies = [
|
||||
{
|
||||
name = "vte";
|
||||
packageId = "vte";
|
||||
}
|
||||
];
|
||||
|
||||
};
|
||||
"vte" = rec {
|
||||
crateName = "vte";
|
||||
version = "0.11.1";
|
||||
edition = "2021";
|
||||
sha256 = "15r1ff4j8ndqj9vsyil3wqwxhhl7jsz5g58f31n0h1wlpxgjn0pm";
|
||||
authors = [
|
||||
"Joe Wilm <joe@jwilm.com>"
|
||||
"Christian Duerr <contact@christianduerr.com>"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "arrayvec";
|
||||
packageId = "arrayvec";
|
||||
optional = true;
|
||||
usesDefaultFeatures = false;
|
||||
}
|
||||
{
|
||||
name = "utf8parse";
|
||||
packageId = "utf8parse";
|
||||
}
|
||||
{
|
||||
name = "vte_generate_state_changes";
|
||||
packageId = "vte_generate_state_changes";
|
||||
}
|
||||
];
|
||||
features = {
|
||||
"ansi" = [ "log" ];
|
||||
"arrayvec" = [ "dep:arrayvec" ];
|
||||
"default" = [ "no_std" ];
|
||||
"log" = [ "dep:log" ];
|
||||
"nightly" = [ "utf8parse/nightly" ];
|
||||
"no_std" = [ "arrayvec" ];
|
||||
"serde" = [ "dep:serde" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "arrayvec" "default" "no_std" ];
|
||||
};
|
||||
"vte_generate_state_changes" = rec {
|
||||
crateName = "vte_generate_state_changes";
|
||||
version = "0.1.1";
|
||||
edition = "2018";
|
||||
sha256 = "1zs5q766q7jmc80c5c80gpzy4qpg5lnydf94mgdzrpy7h5q82myj";
|
||||
procMacro = true;
|
||||
authors = [
|
||||
"Christian Duerr <contact@christianduerr.com>"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "proc-macro2";
|
||||
packageId = "proc-macro2";
|
||||
}
|
||||
{
|
||||
name = "quote";
|
||||
packageId = "quote";
|
||||
}
|
||||
];
|
||||
|
||||
};
|
||||
"wait-timeout" = rec {
|
||||
crateName = "wait-timeout";
|
||||
version = "0.2.0";
|
||||
|
|
@ -15725,7 +15993,7 @@ rec {
|
|||
"Win32_Web" = [ "Win32" ];
|
||||
"Win32_Web_InternetExplorer" = [ "Win32_Web" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Threading" "Win32_System_WindowsProgramming" "Win32_UI" "Win32_UI_Shell" "default" ];
|
||||
resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Threading" "Win32_System_WindowsProgramming" "Win32_UI" "Win32_UI_Input" "Win32_UI_Input_KeyboardAndMouse" "Win32_UI_Shell" "default" ];
|
||||
};
|
||||
"windows-targets 0.48.5" = rec {
|
||||
crateName = "windows-targets";
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ data-encoding = "2.3.3"
|
|||
digest = "0.10.7"
|
||||
fastcdc = { version = "3.1.0", features = ["tokio"] }
|
||||
futures = "0.3.30"
|
||||
indicatif = "0.17.8"
|
||||
lazy_static = "1.4.0"
|
||||
object_store = { version = "0.9.1", features = ["http"] }
|
||||
parking_lot = "0.12.1"
|
||||
|
|
@ -28,6 +29,7 @@ tokio = { version = "1.32.0", features = ["fs", "macros", "net", "rt", "rt-multi
|
|||
tonic = "0.11.0"
|
||||
tower = "0.4.13"
|
||||
tracing = "0.1.37"
|
||||
tracing-indicatif = "0.3.6"
|
||||
url = "2.4.0"
|
||||
walkdir = "2.4.0"
|
||||
zstd = "0.13.0"
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ use std::os::unix::ffi::OsStringExt;
|
|||
use std::os::unix::fs::MetadataExt;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use tracing::instrument;
|
||||
use tracing::Span;
|
||||
use tracing_indicatif::span_ext::IndicatifSpanExt;
|
||||
use walkdir::DirEntry;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
|
|
@ -37,6 +39,7 @@ where
|
|||
BS: BlobService + Clone,
|
||||
DS: DirectoryService,
|
||||
{
|
||||
Span::current().pb_start();
|
||||
let iter = WalkDir::new(path.as_ref())
|
||||
.follow_links(false)
|
||||
.follow_root_links(false)
|
||||
|
|
@ -44,7 +47,17 @@ where
|
|||
.into_iter();
|
||||
|
||||
let entries = dir_entries_to_ingestion_stream(blob_service, iter, path.as_ref());
|
||||
ingest_entries(directory_service, entries).await
|
||||
ingest_entries(
|
||||
directory_service,
|
||||
entries.inspect(|e| {
|
||||
if let Ok(e) = e {
|
||||
let s = Span::current();
|
||||
s.pb_inc(1);
|
||||
s.pb_set_message(&format!("Ingesting {}", e.path()));
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Converts an iterator of [walkdir::DirEntry]s into a stream of ingestion entries.
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ use crate::proto::FileNode;
|
|||
use crate::proto::SymlinkNode;
|
||||
use crate::B3Digest;
|
||||
use futures::{Stream, StreamExt};
|
||||
use tracing::Span;
|
||||
use tracing_indicatif::span_ext::IndicatifSpanExt;
|
||||
|
||||
use tracing::Level;
|
||||
|
||||
|
|
@ -44,7 +46,7 @@ pub mod fs;
|
|||
/// map and upload it to the [DirectoryService] through a lazily created [DirectoryPutter].
|
||||
///
|
||||
/// On success, returns the root node.
|
||||
#[instrument(skip_all, ret(level = Level::TRACE), err)]
|
||||
#[instrument(skip_all, fields(indicatif.pb_show=1), ret(level = Level::TRACE), err)]
|
||||
pub async fn ingest_entries<DS, S, E>(
|
||||
directory_service: DS,
|
||||
mut entries: S,
|
||||
|
|
@ -58,6 +60,8 @@ where
|
|||
let mut directories: HashMap<PathBuf, Directory> = HashMap::default();
|
||||
let mut maybe_directory_putter: Option<Box<dyn DirectoryPutter>> = None;
|
||||
|
||||
Span::current().pb_start();
|
||||
|
||||
let root_node = loop {
|
||||
let mut entry = entries
|
||||
.next()
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ serde = "1.0.195"
|
|||
serde_json = "1.0"
|
||||
sha2 = "0.10.8"
|
||||
sha1 = "0.10.6"
|
||||
tracing-indicatif = "0.3.6"
|
||||
md-5 = "0.10.6"
|
||||
url = "2.4.0"
|
||||
walkdir = "2.4.0"
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@ walkdir = "2.4.0"
|
|||
reqwest = { version = "0.11.22", features = ["rustls-tls-native-roots", "stream"], default-features = false }
|
||||
lru = "0.12.3"
|
||||
parking_lot = "0.12.2"
|
||||
indicatif = "0.17.8"
|
||||
tracing-indicatif = "0.3.6"
|
||||
|
||||
[dependencies.tonic-reflection]
|
||||
optional = true
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ use clap::Subcommand;
|
|||
use futures::future::try_join_all;
|
||||
use futures::StreamExt;
|
||||
use futures::TryStreamExt;
|
||||
use indicatif::ProgressStyle;
|
||||
use nix_compat::path_info::ExportedPathInfo;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
|
|
@ -14,10 +15,13 @@ use tokio_listener::SystemOptions;
|
|||
use tokio_listener::UserOptions;
|
||||
use tonic::transport::Server;
|
||||
use tracing::info;
|
||||
use tracing::info_span;
|
||||
use tracing::instrument;
|
||||
use tracing::Level;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::Layer;
|
||||
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
|
||||
use tracing::Span;
|
||||
use tracing_indicatif::filter::IndicatifFilter;
|
||||
use tracing_indicatif::{span_ext::IndicatifSpanExt, IndicatifLayer};
|
||||
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
|
||||
use tvix_castore::import::fs::ingest_path;
|
||||
use tvix_store::nar::NarCalculationService;
|
||||
use tvix_store::proto::NarInfo;
|
||||
|
|
@ -31,6 +35,20 @@ use tvix_store::pathinfoservice::PathInfoService;
|
|||
use tvix_store::proto::path_info_service_server::PathInfoServiceServer;
|
||||
use tvix_store::proto::GRPCPathInfoServiceWrapper;
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
|
||||
// FUTUREWORK: move this to tracing crate
|
||||
lazy_static! {
|
||||
pub static ref PB_PROGRESS_STYLE: ProgressStyle = ProgressStyle::with_template(
|
||||
"{span_child_prefix}{bar:30} {wide_msg} [{elapsed_precise}] {pos:>7}/{len:7}"
|
||||
)
|
||||
.expect("invalid progress template");
|
||||
pub static ref PB_SPINNER_STYLE: ProgressStyle = ProgressStyle::with_template(
|
||||
"{span_child_prefix}{spinner} {wide_msg} [{elapsed_precise}] {pos:>7}/{len:7}"
|
||||
)
|
||||
.expect("invalid progress template");
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "fuse", feature = "virtiofs"))]
|
||||
use tvix_store::pathinfoservice::make_fs;
|
||||
|
||||
|
|
@ -212,24 +230,32 @@ fn default_threads() -> usize {
|
|||
}
|
||||
|
||||
#[tokio::main]
|
||||
#[instrument(fields(indicatif.pb_show=1))]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
// configure log settings
|
||||
let level = cli.log_level.unwrap_or(Level::INFO);
|
||||
|
||||
let indicatif_layer = IndicatifLayer::new().with_progress_style(PB_SPINNER_STYLE.clone());
|
||||
|
||||
// Set up the tracing subscriber.
|
||||
let subscriber = tracing_subscriber::registry().with(
|
||||
tracing_subscriber::fmt::Layer::new()
|
||||
.with_writer(std::io::stderr)
|
||||
.compact()
|
||||
.with_filter(
|
||||
EnvFilter::builder()
|
||||
.with_default_directive(level.into())
|
||||
.from_env()
|
||||
.expect("invalid RUST_LOG"),
|
||||
),
|
||||
);
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(
|
||||
tracing_subscriber::fmt::Layer::new()
|
||||
.with_writer(indicatif_layer.get_stderr_writer())
|
||||
.compact()
|
||||
.with_filter(
|
||||
EnvFilter::builder()
|
||||
.with_default_directive(level.into())
|
||||
.from_env()
|
||||
.expect("invalid RUST_LOG"),
|
||||
),
|
||||
)
|
||||
.with(indicatif_layer.with_filter(
|
||||
// only show progress for spans with indicatif.pb_show field being set
|
||||
IndicatifFilter::new(false),
|
||||
));
|
||||
|
||||
// Add the otlp layer (when otlp is enabled, and it's not disabled in the CLI)
|
||||
// then init the registry.
|
||||
|
|
@ -355,9 +381,19 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
let nar_calculation_service: Arc<dyn NarCalculationService> =
|
||||
nar_calculation_service.into();
|
||||
|
||||
let root_span = {
|
||||
let s = Span::current();
|
||||
s.pb_set_style(&PB_PROGRESS_STYLE);
|
||||
s.pb_set_message("Importing paths");
|
||||
s.pb_set_length(paths.len() as u64);
|
||||
s.pb_start();
|
||||
s
|
||||
};
|
||||
|
||||
let tasks = paths
|
||||
.into_iter()
|
||||
.map(|path| {
|
||||
let paths_span = root_span.clone();
|
||||
tokio::task::spawn({
|
||||
let blob_service = blob_service.clone();
|
||||
let directory_service = directory_service.clone();
|
||||
|
|
@ -380,6 +416,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
println!("{}", output_path.to_absolute_path());
|
||||
}
|
||||
}
|
||||
paths_span.pb_inc(1);
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
@ -416,15 +453,26 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
// Arc the PathInfoService, as we clone it .
|
||||
let path_info_service: Arc<dyn PathInfoService> = path_info_service.into();
|
||||
|
||||
let lookups_span = info_span!(
|
||||
"lookup pathinfos",
|
||||
"indicatif.pb_show" = tracing::field::Empty
|
||||
);
|
||||
lookups_span.pb_set_length(reference_graph.closure.len() as u64);
|
||||
lookups_span.pb_set_style(&PB_PROGRESS_STYLE);
|
||||
lookups_span.pb_start();
|
||||
|
||||
// From our reference graph, lookup all pathinfos that might exist.
|
||||
let elems: Vec<_> = futures::stream::iter(reference_graph.closure)
|
||||
.map(|elem| {
|
||||
let path_info_service = path_info_service.clone();
|
||||
async move {
|
||||
path_info_service
|
||||
let resp = path_info_service
|
||||
.get(*elem.path.digest())
|
||||
.await
|
||||
.map(|resp| (elem, resp))
|
||||
.map(|resp| (elem, resp));
|
||||
|
||||
Span::current().pb_inc(1);
|
||||
resp
|
||||
}
|
||||
})
|
||||
.buffer_unordered(50)
|
||||
|
|
|
|||
Loading…
Reference in a new issue