9b77ce9f8f
The derivation name for the check will now be calculated from the hash of all Cargo related files (including in subdirs), this makes it less likely for the drv to be cached and for CI to miss an outdated Cargo.lock. Change-Id: I900e9355be3f8a9d6f01162e8ef0da4d8901af30 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11753 Reviewed-by: flokli <flokli@flokli.de> Tested-by: BuildkiteCI
298 lines
9.3 KiB
Nix
298 lines
9.3 KiB
Nix
# Nix helpers for projects under //tvix
|
|
{ pkgs, lib, depot, ... }:
|
|
|
|
let
|
|
# crate override for crates that need protobuf
|
|
protobufDep = prev: (prev.nativeBuildInputs or [ ]) ++ [ pkgs.buildPackages.protobuf ];
|
|
iconvDarwinDep = lib.optional pkgs.stdenv.isDarwin pkgs.libiconv;
|
|
|
|
# On Darwin, some crates producing binaries need to be able to link against security.
|
|
darwinDeps = lib.optionals pkgs.stdenv.isDarwin (with pkgs.buildPackages.darwin.apple_sdk.frameworks; [
|
|
Security
|
|
SystemConfiguration
|
|
]);
|
|
|
|
# Filters the given source, only keeping files related to the build, preventing unnecessary rebuilds.
|
|
# Includes src in the root, all other .rs files, as well as Cargo.toml.
|
|
# Additional files to be included can be specified in extraFileset.
|
|
filterRustCrateSrc =
|
|
{ root # The original src
|
|
, extraFileset ? null # Additional filesets to include (e.g. fileFilter for proto files)
|
|
}:
|
|
lib.fileset.toSource {
|
|
inherit root;
|
|
fileset = (lib.fileset.intersection
|
|
(lib.fileset.fromSource root) # We build our final fileset from the original src
|
|
(lib.fileset.unions ([
|
|
(root + "/src")
|
|
(lib.fileset.fileFilter (f: f.hasExt "rs") root)
|
|
# We assume that every Rust crate will at a minimum have .rs files and a Cargo.toml
|
|
(lib.fileset.fileFilter (f: f.name == "Cargo.toml") root)
|
|
] ++ lib.optional (extraFileset != null) extraFileset)));
|
|
};
|
|
|
|
# Load the crate2nix crate tree.
|
|
crates = import ./Cargo.nix {
|
|
inherit pkgs;
|
|
nixpkgs = pkgs.path;
|
|
|
|
# Hack to fix Darwin build
|
|
# See https://github.com/NixOS/nixpkgs/issues/218712
|
|
buildRustCrateForPkgs = pkgs:
|
|
if pkgs.stdenv.isDarwin then
|
|
let
|
|
buildRustCrate = pkgs.buildRustCrate;
|
|
buildRustCrate_ = args: buildRustCrate args // { dontStrip = true; };
|
|
override = o: args: buildRustCrate.override o (args // { dontStrip = true; });
|
|
in
|
|
pkgs.makeOverridable override { }
|
|
else pkgs.buildRustCrate;
|
|
|
|
defaultCrateOverrides = pkgs.defaultCrateOverrides // {
|
|
zstd-sys = prev: {
|
|
nativeBuildInputs = prev.nativeBuildInputs or [ ];
|
|
buildInputs = prev.buildInputs or [ ] ++ iconvDarwinDep;
|
|
};
|
|
|
|
opentelemetry-proto = prev: {
|
|
nativeBuildInputs = protobufDep prev;
|
|
};
|
|
|
|
prost-build = prev: {
|
|
nativeBuildInputs = protobufDep prev;
|
|
};
|
|
|
|
prost-wkt-types = prev: {
|
|
nativeBuildInputs = protobufDep prev;
|
|
};
|
|
|
|
tonic-reflection = prev: {
|
|
nativeBuildInputs = protobufDep prev;
|
|
};
|
|
|
|
tvix-build = prev: {
|
|
src = filterRustCrateSrc rec {
|
|
root = prev.src.origSrc;
|
|
extraFileset = (lib.fileset.fileFilter (f: f.hasExt "proto") root);
|
|
};
|
|
PROTO_ROOT = depot.tvix.build.protos.protos;
|
|
nativeBuildInputs = protobufDep prev;
|
|
buildInputs = darwinDeps;
|
|
};
|
|
|
|
tvix-castore = prev: {
|
|
src = filterRustCrateSrc rec {
|
|
root = prev.src.origSrc;
|
|
extraFileset = (lib.fileset.fileFilter (f: f.hasExt "proto") root);
|
|
};
|
|
PROTO_ROOT = depot.tvix.castore.protos.protos;
|
|
nativeBuildInputs = protobufDep prev;
|
|
};
|
|
|
|
tvix-cli = prev: {
|
|
src = filterRustCrateSrc { root = prev.src.origSrc; };
|
|
buildInputs = prev.buildInputs or [ ] ++ darwinDeps;
|
|
};
|
|
|
|
tvix-store = prev: {
|
|
src = filterRustCrateSrc rec {
|
|
root = prev.src.origSrc;
|
|
extraFileset = (lib.fileset.fileFilter (f: f.hasExt "proto") root);
|
|
};
|
|
PROTO_ROOT = depot.tvix.store.protos.protos;
|
|
nativeBuildInputs = protobufDep prev;
|
|
# fuse-backend-rs uses DiskArbitration framework to handle mount/unmount on Darwin
|
|
buildInputs = prev.buildInputs or [ ]
|
|
++ darwinDeps
|
|
++ lib.optional pkgs.stdenv.isDarwin pkgs.buildPackages.darwin.apple_sdk.frameworks.DiskArbitration;
|
|
};
|
|
|
|
tvix-eval-builtin-macros = prev: {
|
|
src = filterRustCrateSrc { root = prev.src.origSrc; };
|
|
};
|
|
|
|
tvix-eval = prev: {
|
|
src = filterRustCrateSrc rec {
|
|
root = prev.src.origSrc;
|
|
extraFileset = (root + "/proptest-regressions");
|
|
};
|
|
};
|
|
|
|
tvix-glue = prev: {
|
|
src = filterRustCrateSrc {
|
|
root = prev.src.origSrc;
|
|
};
|
|
};
|
|
|
|
tvix-serde = prev: {
|
|
src = filterRustCrateSrc { root = prev.src.origSrc; };
|
|
};
|
|
|
|
nix-compat = prev: {
|
|
src = filterRustCrateSrc rec {
|
|
root = prev.src.origSrc;
|
|
extraFileset = (root + "/testdata");
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
# Cargo dependencies to be used with nixpkgs rustPlatform functions.
|
|
cargoDeps = pkgs.rustPlatform.importCargoLock {
|
|
lockFile = ./Cargo.lock;
|
|
# Extract the hashes from `crates` / Cargo.nix, we already get them from cargo2nix.
|
|
# This returns an attribute set containing "${crateName}-${version}" as key,
|
|
# and the outputHash as value.
|
|
outputHashes = builtins.listToAttrs
|
|
(map
|
|
(crateName:
|
|
(lib.nameValuePair "${crateName}-${crates.internal.crates.${crateName}.version}" crates.internal.crates.${crateName}.src.outputHash)
|
|
) [
|
|
"bigtable_rs"
|
|
"wu-manber"
|
|
]);
|
|
};
|
|
|
|
# The cleaned sources.
|
|
src = depot.third_party.gitignoreSource ./.;
|
|
|
|
# Target containing *all* tvix proto files.
|
|
# Useful for workspace-wide cargo invocations (doc, clippy)
|
|
protos = pkgs.symlinkJoin {
|
|
name = "tvix-all-protos";
|
|
paths = [
|
|
depot.tvix.build.protos.protos
|
|
depot.tvix.castore.protos.protos
|
|
depot.tvix.store.protos.protos
|
|
];
|
|
};
|
|
|
|
in
|
|
{
|
|
inherit crates protos;
|
|
|
|
# Run crate2nix generate, ensure the output doesn't differ afterwards
|
|
# (and doesn't fail).
|
|
#
|
|
# Currently this re-downloads every crate every time
|
|
# crate2nix-check (but not crate2nix) is built.
|
|
# TODO(amjoseph): be less wasteful with bandwidth.
|
|
#
|
|
crate2nix-check =
|
|
let
|
|
outputHashAlgo = "sha256";
|
|
in
|
|
pkgs.stdenv.mkDerivation {
|
|
inherit src;
|
|
|
|
# Important: we include the hash of all Cargo related files in the derivation name.
|
|
# This forces the FOD to be rebuilt/re-verified whenever one of them changes.
|
|
name = "tvix-crate2nix-check-" + builtins.substring 0 8 (builtins.hashString "sha256"
|
|
(lib.concatMapStrings (f: builtins.hashFile "sha256" f)
|
|
([ ./Cargo.toml ./Cargo.lock ] ++ (map (m: ./. + "/${m}/Cargo.toml") (lib.importTOML ./Cargo.toml).workspace.members))
|
|
)
|
|
);
|
|
|
|
nativeBuildInputs = with pkgs; [ git cacert cargo ];
|
|
buildPhase = ''
|
|
export CARGO_HOME=$(mktemp -d)
|
|
|
|
# The following command can be omitted, in which case
|
|
# crate2nix-generate will run it automatically, but won't show the
|
|
# output, which makes it look like the build is somehow "stuck" for a
|
|
# minute or two.
|
|
cargo metadata > /dev/null
|
|
|
|
# running this command counteracts depotfmt brokenness
|
|
git init
|
|
|
|
${depot.tools.crate2nix-generate}/bin/crate2nix-generate
|
|
|
|
# technically unnecessary, but provides more-helpful output in case of error
|
|
diff -ur Cargo.nix ${src}/Cargo.nix
|
|
|
|
# the FOD hash will check that the (re-)generated Cargo.nix matches the committed Cargo.nix
|
|
cp Cargo.nix $out
|
|
'';
|
|
|
|
# This is an FOD in order to allow `cargo` to perform network access.
|
|
outputHashMode = "flat";
|
|
inherit outputHashAlgo;
|
|
outputHash = builtins.hashFile outputHashAlgo ./Cargo.nix;
|
|
env.SSL_CERT_FILE = "${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt";
|
|
};
|
|
|
|
# Provide the Tvix logo in both .webp and .png format.
|
|
logo = pkgs.runCommand "logo"
|
|
{
|
|
nativeBuildInputs = [ pkgs.imagemagick ];
|
|
} ''
|
|
mkdir -p $out
|
|
cp ${./logo.webp} $out/logo.webp
|
|
convert $out/logo.webp $out/logo.png
|
|
'';
|
|
|
|
# Provide a shell for the combined dependencies of all Tvix Rust
|
|
# projects. Note that as this is manually maintained it may be
|
|
# lacking something, but it is required for some people's workflows.
|
|
#
|
|
# This shell can be entered with e.g. `mg shell //tvix:shell`.
|
|
# This is a separate file, so it can be used individually in the tvix josh
|
|
# workspace too.
|
|
shell = (import ./shell.nix { inherit pkgs; });
|
|
|
|
# Build the Rust documentation for publishing on docs.tvix.dev.
|
|
rust-docs = pkgs.stdenv.mkDerivation {
|
|
inherit cargoDeps src;
|
|
name = "tvix-rust-docs";
|
|
PROTO_ROOT = protos;
|
|
|
|
nativeBuildInputs = with pkgs; [
|
|
cargo
|
|
pkg-config
|
|
protobuf
|
|
rustc
|
|
rustPlatform.cargoSetupHook
|
|
];
|
|
|
|
buildInputs = [
|
|
pkgs.fuse
|
|
] ++ iconvDarwinDep;
|
|
|
|
buildPhase = ''
|
|
cargo doc --document-private-items
|
|
mv target/doc $out
|
|
'';
|
|
};
|
|
|
|
# Run cargo clippy. We run it with -Dwarnings, so warnings cause a nonzero
|
|
# exit code.
|
|
clippy = pkgs.stdenv.mkDerivation {
|
|
inherit cargoDeps src;
|
|
name = "tvix-clippy";
|
|
PROTO_ROOT = protos;
|
|
|
|
buildInputs = [
|
|
pkgs.fuse
|
|
];
|
|
nativeBuildInputs = with pkgs; [
|
|
cargo
|
|
clippy
|
|
pkg-config
|
|
protobuf
|
|
rustc
|
|
rustPlatform.cargoSetupHook
|
|
];
|
|
|
|
buildPhase = "cargo clippy --tests --all-features --benches --examples -- -Dwarnings | tee $out";
|
|
};
|
|
|
|
meta.ci.targets = [
|
|
"clippy"
|
|
"crate2nix-check"
|
|
"shell"
|
|
"rust-docs"
|
|
];
|
|
|
|
utils = import ./utils.nix { inherit lib depot; };
|
|
}
|