refactor(tvix/store): use bytes for node names and symlink targets

Some paths might use names that are not valid UTF-8. We should be able
to represent them.

We don't actually need to touch the PathInfo structures, as they need to
represent StorePaths, which come with their own harder restrictions,
which can't encode non-UTF8 data.

While this doesn't change any of the wire format of the gRPC messages,
it does however change the interface of tvix_eval::EvalIO - its
read_dir() method does now return a list of Vec<u8>, rather than
SmolStr. Maybe this should be OsString instead?

Change-Id: I821016d9a58ec441ee081b0b9f01c9240723af0b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8974
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Tested-by: BuildkiteCI

tvix/eval/src/builtins/impure.rs

@@ -38,7 +38,8 @@ mod impure_builtins {
         let dir = generators::request_read_dir(&co, path).await;
         let res = dir.into_iter().map(|(name, ftype)| {
             (
-                NixString::from(name.as_str()),
+                // TODO: propagate Vec<u8> into NixString.
+                NixString::from(String::from_utf8(name).expect("parsing file name as string")),
                 Value::String(
                     match ftype {
                         FileType::Directory => "directory",

tvix/eval/src/io.rs

@@ -15,12 +15,14 @@
 //! In the context of Nix builds, callers also use this interface to determine
 //! how store paths are opened and so on.
 
-use smol_str::SmolStr;
 use std::{
     io,
     path::{Path, PathBuf},
 };
 
+#[cfg(target_family = "unix")]
+use std::os::unix::ffi::OsStringExt;
+
 /// Types of files as represented by `builtins.readDir` in Nix.
 #[derive(Debug)]
 pub enum FileType {
@@ -40,7 +42,7 @@ pub trait EvalIO {
 
     /// Read the directory at the specified path and return the names
     /// of its entries associated with their [`FileType`].
-    fn read_dir(&self, path: &Path) -> Result<Vec<(SmolStr, FileType)>, io::Error>;
+    fn read_dir(&self, path: &Path) -> Result<Vec<(Vec<u8>, FileType)>, io::Error>;
 
     /// Import the given path. What this means depends on the
     /// implementation, for example for a `std::io`-based
@@ -63,6 +65,7 @@ pub trait EvalIO {
 #[cfg(feature = "impure")]
 pub struct StdIO;
 
+// TODO: we might want to make this whole impl to be target_family = "unix".
 #[cfg(feature = "impure")]
 impl EvalIO for StdIO {
     fn path_exists(&self, path: &Path) -> Result<bool, io::Error> {
@@ -73,7 +76,7 @@ impl EvalIO for StdIO {
         std::fs::read_to_string(&path)
     }
 
-    fn read_dir(&self, path: &Path) -> Result<Vec<(SmolStr, FileType)>, io::Error> {
+    fn read_dir(&self, path: &Path) -> Result<Vec<(Vec<u8>, FileType)>, io::Error> {
         let mut result = vec![];
 
         for entry in path.read_dir()? {
@@ -90,7 +93,7 @@ impl EvalIO for StdIO {
                 FileType::Unknown
             };
 
-            result.push((SmolStr::new(entry.file_name().to_string_lossy()), val));
+            result.push((entry.file_name().into_vec(), val))
         }
 
         Ok(result)
@@ -122,7 +125,7 @@ impl EvalIO for DummyIO {
         ))
     }
 
-    fn read_dir(&self, _: &Path) -> Result<Vec<(SmolStr, FileType)>, io::Error> {
+    fn read_dir(&self, _: &Path) -> Result<Vec<(Vec<u8>, FileType)>, io::Error> {
         Err(io::Error::new(
             io::ErrorKind::Unsupported,
             "I/O methods are not implemented in DummyIO",

tvix/eval/src/vm/generators.rs

@@ -10,7 +10,6 @@
 use core::pin::Pin;
 use genawaiter::rc::Co;
 pub use genawaiter::rc::Gen;
-use smol_str::SmolStr;
 use std::fmt::Display;
 use std::future::Future;
 
@@ -187,7 +186,7 @@ pub enum VMResponse {
     Path(PathBuf),
 
     /// VM response with the contents of a directory.
-    Directory(Vec<(SmolStr, FileType)>),
+    Directory(Vec<(Vec<u8>, FileType)>),
 
     /// VM response with a span to use at the current point.
     Span(LightSpan),
@@ -736,7 +735,7 @@ pub(crate) async fn request_path_exists(co: &GenCo, path: PathBuf) -> Value {
     }
 }
 
-pub(crate) async fn request_read_dir(co: &GenCo, path: PathBuf) -> Vec<(SmolStr, FileType)> {
+pub(crate) async fn request_read_dir(co: &GenCo, path: PathBuf) -> Vec<(Vec<u8>, FileType)> {
     match co.yield_(VMRequest::ReadDir(path)).await {
         VMResponse::Directory(dir) => dir,
         msg => panic!(