From 25393d80806f7c7caed188130d3f7c9574e61ece Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Wed, 27 May 2020 22:03:36 +0100 Subject: [PATCH] refactor(3p/nix): Introduce CMake as the build system for Nix Completes the switch from Meson to CMake for the core build system in Nix. Meson was added originally because someone else had already done the work for integrating it in Nix and it was an upgrade from the previous setup. However over time it became clear that Meson is not quite mature enough for projects like Nix that have occasionally peculiar configuration constraints. Some issues encountered with Meson (some of these are due to the Meson setup in Nix): * Difficulty with generating correct compile_commands.json for external tools like clangd * Difficulty linking to libc++ when using clang * Ugly shell invocations for certain parts of the build system (I want these to be gone!!!) This CMake setup mimics the Meson configuration, but there are some differences (some temporary): * headers are now included separately for each library (see a previous commit that changes includes appropriately) * autoheaders-style configuration is currently hardcoded. Before blindly copying this I want to evaluate how much of it actually exists for portability concerns that I don't have (such as support for OS X). * Nix is built with libc++ by default. * [libstore] SQL schema is now inlined via a generated header, not an included string literal Abseil is still built as part of this build, rather than an external dependency, because it chokes on differently configured compiler invocations. Note that because of the move to libc++ an unwanted behaviour is introduced: glog log messages no longer have a body. I have yet to debug what is going on there. --- third_party/nix/.gitignore | 1 - third_party/nix/CMakeLists.txt | 31 +++++ third_party/nix/config.h.in | 128 ++++++++++++++++++++ third_party/nix/default.nix | 18 +-- third_party/nix/src/CMakeLists.txt | 80 ++++++++++++ third_party/nix/src/libexpr/CMakeLists.txt | 75 ++++++++++++ third_party/nix/src/libmain/CMakeLists.txt | 25 ++++ third_party/nix/src/libstore/CMakeLists.txt | 102 ++++++++++++++++ third_party/nix/src/libstore/local-store.cc | 9 +- third_party/nix/src/libutil/CMakeLists.txt | 53 ++++++++ 10 files changed, 500 insertions(+), 22 deletions(-) create mode 100644 third_party/nix/CMakeLists.txt create mode 100644 third_party/nix/config.h.in create mode 100644 third_party/nix/src/CMakeLists.txt create mode 100644 third_party/nix/src/libexpr/CMakeLists.txt create mode 100644 third_party/nix/src/libmain/CMakeLists.txt create mode 100644 third_party/nix/src/libstore/CMakeLists.txt create mode 100644 third_party/nix/src/libutil/CMakeLists.txt diff --git a/third_party/nix/.gitignore b/third_party/nix/.gitignore index 6009737ee..7f2d47713 100644 --- a/third_party/nix/.gitignore +++ b/third_party/nix/.gitignore @@ -4,7 +4,6 @@ perl/Makefile.config # / /aclocal.m4 /autom4te.cache -/config.* /configure /nix.spec /stamp-h1 diff --git a/third_party/nix/CMakeLists.txt b/third_party/nix/CMakeLists.txt new file mode 100644 index 000000000..fb7076aa5 --- /dev/null +++ b/third_party/nix/CMakeLists.txt @@ -0,0 +1,31 @@ +# -*- mode: cmake; -*- +cmake_minimum_required(VERSION 3.16) +project(nix CXX) +set(CMAKE_CXX_STANDARD 17) + +# The following lines import CMake-native dependencies which may +# contain useful definitions. Other dependencies are not treated +# specially by CMake and are only linked into the resulting binary. +find_package(BZip2) +find_package(Boost COMPONENTS context) # probably coroutine + ::headers, lets find out +find_package(CURL) +find_package(SQLite3) +find_package(Threads) +find_package(glog) +find_package(LibLZMA) + +# Abseil really doesn't like being precompiled. It is included here as +# an imported CMake project (i.e. it will be built /with/ this +# project). +# +# In development mode, the 'abseil_cpp' folder is symlinked to +# ../abseil_cpp (the relative location in the depot). In derivation +# builds this symlink is automatically replaced with an appropriate +# link to the Abseil sources in the Nix store. +add_subdirectory(abseil_cpp) + +# generate a configuration file (autoheader-style) to configure +# certain symbols that Nix depends on. +configure_file(config.h.in nix_config.h @ONLY) + +add_subdirectory(src) diff --git a/third_party/nix/config.h.in b/third_party/nix/config.h.in new file mode 100644 index 000000000..54855763d --- /dev/null +++ b/third_party/nix/config.h.in @@ -0,0 +1,128 @@ +// This file configures various build-time settings in Nix. In +// previous iterations it was mostly responsible for configuring +// OS-dependent settings, which are still preserved below but should +// be removed. + +#ifndef NIX_CONFIG_H +#define NIX_CONFIG_H + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "2.3.4" + +/* Platform identifier (`cpu-os`) */ +// TODO(tazjin): generate +#define SYSTEM "x86_64-linux" + + // TODO(tazjin): some of these values are nonsensical for Nix +#define NIX_PREFIX "/usr/local" +#define NIX_STORE_DIR "/nix/store" +#define NIX_DATA_DIR "/usr/local/share" +#define NIX_LOG_DIR "/nix/var/log/nix" +#define NIX_STATE_DIR "/nix/var/nix" +#define NIX_CONF_DIR "/usr/local/etc/nix" +#define NIX_LIBEXEC_DIR "/usr/local/libexec" +#define NIX_BIN_DIR "/usr/local/bin" +#define NIX_MAN_DIR "/usr/local/share/man" +#define SANDBOX_SHELL "/nix/store/zq8biwi5mj2lrn68kx0lk0fkpbqypyxd-busybox-1.31.1-x86_64-unknown-linux-musl/bin/busybox" + + +// These are hardcoded either because support for non-Linux is being +// dropped, or because a decision was made to make inclusion of these +// libraries mandatory. + +#define HAVE_STRUCT_DIRENT_D_TYPE 1 + +#define HAVE_LUTIMES 1 + +// Whether link() works on symlinks +#define CAN_LINK_SYMLINK 1 + +/* Whether to use the Boehm garbage collector. */ +#define HAVE_BOEHMGC 1 + +/* Define if the Boost library is available. */ +#define HAVE_BOOST 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_BZLIB_H 1 + +/* Define if the compiler supports basic C++17 syntax */ +#define HAVE_CXX17 1 + +/* Define to 1 if you have the header file, and it defines `DIR` */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR` */ +#define HAVE_DIR_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_EDITLINE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `lchown` function. */ +#define HAVE_LCHOWN 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE 1 + +/* Define to 1 if you have the `lutimes` function. */ +#define HAVE_LUTIMES 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `pipe2` function. */ +#define HAVE_PIPE2 1 + +/* Define to 1 if you have the `posix_fallocate` function. */ +#define HAVE_POSIX_FALLOCATE 1 + +/* Define to 1 if you have the `pubsetbuf` function. */ +#define HAVE_PUBSETBUF 1 + +/* Whether seccomp is available and should be used for sandboxing. */ +#define HAVE_SECCOMP 1 + +/* Define to 1 if you have the `setresuid` function. */ +#define HAVE_SETRESUID 1 + +/* Define to 1 if you have the `setreuid` function. */ +#define HAVE_SETREUID 1 + +/* Whether to use libsodium for cryptography. */ +#define HAVE_SODIUM 1 + +/* Define to 1 if you have the `statvfs` function. */ +#define HAVE_STATVFS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strsignal` function. */ +#define HAVE_STRSIGNAL 1 + +/* Define to 1 if you have the `sysconf` function. */ +#define HAVE_SYSCONF 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + + +#endif diff --git a/third_party/nix/default.nix b/third_party/nix/default.nix index 3c7fa4cdb..aa2fd3b7a 100644 --- a/third_party/nix/default.nix +++ b/third_party/nix/default.nix @@ -2,8 +2,6 @@ , buildType ? "release", ... }: let - stdenv = with pkgs; overrideCC clangStdenv clang_10; - aws-s3-cpp = pkgs.aws-sdk-cpp.override { apis = ["s3" "transfer"]; customMemoryManagement = false; @@ -14,22 +12,20 @@ let largeBoehm = pkgs.boehmgc.override { enableLargeConfig = true; }; -in stdenv.mkDerivation { +in pkgs.llvmPackages.libcxxStdenv.mkDerivation { pname = "tazjix"; version = "2.3.4"; src = ./.; - # Abseil's sources need to be linked into a subproject. + # Abseil's sources need to be symlinked into Nix' sources. postUnpack = '' - ln -fs ${pkgs.abseil_cpp.src} nix/subprojects/abseil_cpp + ln -fs ${pkgs.abseil_cpp.src} nix/abseil_cpp ''; nativeBuildInputs = with pkgs; [ bison clang-tools cmake - meson - ninja pkgconfig libxml2 libxslt @@ -56,14 +52,6 @@ in stdenv.mkDerivation { xz ]; - mesonBuildType = buildType; - mesonFlags = [ - "-Dsandbox_shell=${pkgs.busybox-sandbox-shell}/bin/busybox" - ]; - - # cmake is only included to build Abseil and its hook should not run - dontUseCmakeConfigure = true; - # Install the various symlinks to the Nix binary which users expect # to exist. postInstall = '' diff --git a/third_party/nix/src/CMakeLists.txt b/third_party/nix/src/CMakeLists.txt new file mode 100644 index 000000000..37ff3b54a --- /dev/null +++ b/third_party/nix/src/CMakeLists.txt @@ -0,0 +1,80 @@ +# -*- mode: cmake; -*- + +# The 'nix' binary is composed of various sources below this +# directory. In the previous build system, they were all built from +# this location and this setup mimics that (with the exception of the +# various Nix libraries). + +add_subdirectory(libutil) +add_subdirectory(libstore) +add_subdirectory(libmain) +add_subdirectory(libexpr) + +add_executable(nix) +set_property(TARGET nix PROPERTY CXX_STANDARD 17) +include_directories(${PROJECT_BINARY_DIR}) +target_include_directories(nix PUBLIC "${nix_SOURCE_DIR}/src") + +target_sources(nix + PRIVATE + nix/command.hh + nix/legacy.hh + nix-env/user-env.hh + nix-store/dotgraph.hh + nix-store/graphml.hh + + nix/add-to-store.cc + nix/build.cc + nix/cat.cc + nix/command.cc + nix/copy.cc + nix/doctor.cc + nix/dump-path.cc + nix/edit.cc + nix/eval.cc + nix/hash.cc + nix/installables.cc + nix/legacy.cc + nix/log.cc + nix/ls.cc + nix/main.cc + nix/optimise-store.cc + nix/path-info.cc + nix/ping-store.cc + nix/repl.cc + nix/run.cc + nix/search.cc + nix/show-config.cc + nix/show-derivation.cc + nix/sigs.cc + nix/upgrade-nix.cc + nix/verify.cc + nix/why-depends.cc + + build-remote/build-remote.cc + nix-build/nix-build.cc + nix-channel/nix-channel.cc + nix-collect-garbage/nix-collect-garbage.cc + nix-copy-closure/nix-copy-closure.cc + nix-daemon/nix-daemon.cc + nix-env/nix-env.cc + nix-env/user-env.cc + nix-instantiate/nix-instantiate.cc + nix-prefetch-url/nix-prefetch-url.cc + nix-store/dotgraph.cc + nix-store/graphml.cc + nix-store/nix-store.cc +) + +target_link_libraries(nix + nixexpr + nixmain + nixstore + nixutil + + absl::strings + editline + glog +) + +INSTALL(TARGETS nix DESTINATION bin) diff --git a/third_party/nix/src/libexpr/CMakeLists.txt b/third_party/nix/src/libexpr/CMakeLists.txt new file mode 100644 index 000000000..3456052ea --- /dev/null +++ b/third_party/nix/src/libexpr/CMakeLists.txt @@ -0,0 +1,75 @@ +# -*- mode: cmake; -*- +add_library(nixexpr SHARED) +set_property(TARGET nixexpr PROPERTY CXX_STANDARD 17) +include_directories(${PROJECT_BINARY_DIR}) # for 'generated/' +target_include_directories(nixexpr PUBLIC "${nix_SOURCE_DIR}/src") + +# Generate lexer & parser for inclusion: +find_package(BISON) +find_package(FLEX) + +BISON_TARGET(NixParser parser.y + ${PROJECT_BINARY_DIR}/generated/parser-tab.cc + DEFINES_FILE ${PROJECT_BINARY_DIR}/generated/parser-tab.hh) + +FLEX_TARGET(NixLexer lexer.l + ${PROJECT_BINARY_DIR}/generated/lexer-tab.cc + DEFINES_FILE ${PROJECT_BINARY_DIR}/generated/lexer-tab.hh) + +ADD_FLEX_BISON_DEPENDENCY(NixLexer NixParser) + +target_sources(nixexpr + PUBLIC + attr-path.hh + attr-set.hh + common-eval-args.hh + eval.hh + eval-inline.hh + function-trace.hh + get-drvs.hh + json-to-value.hh + names.hh + nixexpr.hh + primops.hh + symbol-table.hh + value.hh + value-to-json.hh + value-to-xml.hh + + PRIVATE + ${PROJECT_BINARY_DIR}/generated/parser-tab.hh + ${PROJECT_BINARY_DIR}/generated/parser-tab.cc + ${PROJECT_BINARY_DIR}/generated/lexer-tab.hh + ${PROJECT_BINARY_DIR}/generated/lexer-tab.cc + primops/context.cc + primops/fetchGit.cc + primops/fetchMercurial.cc + primops/fromTOML.cc + attr-path.cc + attr-set.cc + common-eval-args.cc + eval.cc + function-trace.cc + get-drvs.cc + json-to-value.cc + names.cc + nixexpr.cc + primops.cc + symbol-table.cc + value-to-json.cc + value-to-xml.cc +) + +target_link_libraries(nixexpr + nixmain + nixstore + nixutil + + absl::btree + absl::node_hash_set + absl::strings + gc + gccpp +) + +INSTALL(TARGETS nixexpr DESTINATION lib) diff --git a/third_party/nix/src/libmain/CMakeLists.txt b/third_party/nix/src/libmain/CMakeLists.txt new file mode 100644 index 000000000..825e8ee5d --- /dev/null +++ b/third_party/nix/src/libmain/CMakeLists.txt @@ -0,0 +1,25 @@ +# -*- mode: cmake; -*- +add_library(nixmain SHARED) +set_property(TARGET nixmain PROPERTY CXX_STANDARD 17) +include_directories(${PROJECT_BINARY_DIR}) # for config.h +target_include_directories(nixmain PUBLIC "${nix_SOURCE_DIR}/src") + +target_sources(nixmain + PUBLIC + common-args.hh + shared.hh + PRIVATE + common-args.cc + shared.cc + stack.cc +) + +target_link_libraries(nixmain + nixstore + nixutil + + absl::strings + glog +) + +INSTALL(TARGETS nixmain DESTINATION lib) diff --git a/third_party/nix/src/libstore/CMakeLists.txt b/third_party/nix/src/libstore/CMakeLists.txt new file mode 100644 index 000000000..21adb0fb0 --- /dev/null +++ b/third_party/nix/src/libstore/CMakeLists.txt @@ -0,0 +1,102 @@ +# -*- mode: cmake; -*- +add_library(nixstore SHARED) +set_property(TARGET nixstore PROPERTY CXX_STANDARD 17) +include_directories(${PROJECT_BINARY_DIR}) # for config.h +target_include_directories(nixstore PUBLIC "${nix_SOURCE_DIR}/src") + +# The database schema is stored in schema.sql, but needs to be +# available during the build as static data. +# +# These commands create an includeable source-file out of it. +file(READ "schema.sql" NIX_SCHEMA) + +string(CONFIGURE + "#pragma once + namespace nix { + constexpr char kNixSqlSchema[] = R\"(${NIX_SCHEMA})\"\; + }" + NIX_SCHEMA_GEN) + +file(WRITE ${PROJECT_BINARY_DIR}/generated/schema.sql.hh ${NIX_SCHEMA_GEN}) + +target_sources(nixstore + PUBLIC + binary-cache-store.hh + builtins.hh + crypto.hh + derivations.hh + download.hh + fs-accessor.hh + globals.hh + local-store.hh + machines.hh + nar-accessor.hh + nar-info-disk-cache.hh + nar-info.hh + parsed-derivations.hh + pathlocks.hh + profiles.hh + references.hh + remote-fs-accessor.hh + remote-store.hh + s3-binary-cache-store.hh + s3.hh + serve-protocol.hh + sqlite.hh + ssh.hh + store-api.hh + worker-protocol.hh + + PRIVATE + ${PROJECT_BINARY_DIR}/generated/schema.sql.hh + binary-cache-store.cc + build.cc + crypto.cc + derivations.cc + download.cc + export-import.cc + gc.cc + globals.cc + http-binary-cache-store.cc + legacy-ssh-store.cc + local-binary-cache-store.cc + local-fs-store.cc + local-store.cc + machines.cc + misc.cc + nar-accessor.cc + nar-info.cc + nar-info-disk-cache.cc + optimise-store.cc + parsed-derivations.cc + pathlocks.cc + profiles.cc + references.cc + remote-fs-accessor.cc + remote-store.cc + s3-binary-cache-store.cc + sqlite.cc + ssh.cc + ssh-store.cc + store-api.cc + builtins/buildenv.cc + builtins/fetchurl.cc +) + +target_link_libraries(nixstore + nixutil + + BZip2::BZip2 + Boost::context + CURL::libcurl + LibLZMA::LibLZMA + SQLite::SQLite3 + absl::strings + brotlidec + brotlienc + glog + seccomp + sodium +) + +INSTALL(TARGETS nixstore DESTINATION lib) diff --git a/third_party/nix/src/libstore/local-store.cc b/third_party/nix/src/libstore/local-store.cc index f9d03c520..2f35dd69c 100644 --- a/third_party/nix/src/libstore/local-store.cc +++ b/third_party/nix/src/libstore/local-store.cc @@ -374,10 +374,7 @@ void LocalStore::openDB(State& state, bool create) { /* Initialise the database schema, if necessary. */ if (create) { - const char* schema = -#include "schema.sql.gen.hh" - ; - db.exec(schema); + db.exec(kNixSqlSchema); } } @@ -435,13 +432,13 @@ static void canonicaliseTimestampAndPermissions(const Path& path, if (errno != ENOSYS || (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) { #else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) { #endif throw SysError(format("changing modification time of '%1%'") % path); } } } // namespace nix -} +} // namespace nix void canonicaliseTimestampAndPermissions(const Path& path) { struct stat st; diff --git a/third_party/nix/src/libutil/CMakeLists.txt b/third_party/nix/src/libutil/CMakeLists.txt new file mode 100644 index 000000000..2aa513a39 --- /dev/null +++ b/third_party/nix/src/libutil/CMakeLists.txt @@ -0,0 +1,53 @@ +# -*- mode: cmake; -*- +add_library(nixutil SHARED) +set_property(TARGET nixutil PROPERTY CXX_STANDARD 17) +include_directories(${PROJECT_BINARY_DIR}) # for config.h +target_compile_features(nixutil PUBLIC cxx_std_17) + +target_sources(nixutil + PUBLIC + affinity.hh + archive.hh + args.hh + compression.hh + config.hh + finally.hh + hash.hh + istringstream_nocopy.hh + json.hh + lazy.hh + lru-cache.hh + monitor-fd.hh + pool.hh + prefork-compat.hh + ref.hh + serialise.hh + sync.hh + thread-pool.hh + types.hh + util.hh + xml-writer.hh + + PRIVATE + affinity.cc + archive.cc + args.cc + compression.cc + config.cc + hash.cc + json.cc + serialise.cc + thread-pool.cc + util.cc + xml-writer.cc +) + +target_link_libraries(nixutil + absl::strings + glog +) + +# Install header files to include/libutil and mark them for automatic +# inclusion in targets that link to this one. +target_include_directories(nixutil PUBLIC "${nix_SOURCE_DIR}/src") +INSTALL(TARGETS nixutil DESTINATION lib)