refactor(nix/buildLisp): prepare multi implementation support
Concept is roughly: * receive extra argument `implementation` that refers to the name of an implementation or rather an attribute in an internal attribute set telling buildLisp how to do certain build steps. * We assume an implementation can execute lisp files as scripts and that we can implement the following main tasks in lisp: - Building a library (`genCompileLisp`) - Building an executable (`genDumpLisp`) - Loading a library dynamically (`genLoadLisp`) Based on that we can implement: - Running a test suite (`genTestLisp`) - A REPL preloaded with a libraries and their dependencies (`lispWith`) Additional attributes for implementing these parts genericly are added as needed (`faslExt` and `runScript`). * `genCompileLisp` no longer prints a shell script which concatenates the individual FASLs. Instead it does the step previously done by the shell script itself. In essence `genCompileLisp` now writes a lisp script which compiles and installs the library to build. This will allow us extra freedom for different implementations, e. g. for ECL we'll want to build a object file archive additionally to fasl files in order to be able to link proper executables. * `genLoadLisp` and `genTestLisp` are almost generic (the former just sometimes would need to use different file extensions), but we integrate them into the implementation “API” to facilitate minor tweaks we need to do like the `fasc` extension for ECL's native FASL files. Change-Id: I1b8ccc0063159638ec7af534e9a6b5384e750193 Reviewed-on: https://cl.tvl.fyi/c/depot/+/3292 Tested-by: BuildkiteCI Reviewed-by: tazjin <mail@tazj.in>
This commit is contained in:
sterni
parent
708fba53c3
commit
d344637fe2
1 changed files with 186 additions and 103 deletions
|
|
@ -14,57 +14,20 @@ let
|
|||
# Internal helper definitions
|
||||
#
|
||||
|
||||
# 'genLoadLisp' generates Lisp code that instructs SBCL to load all
|
||||
# the provided Lisp libraries.
|
||||
genLoadLisp = deps: lib.concatStringsSep "\n"
|
||||
(map (lib: "(load \"${lib}/${lib.lispName}.fasl\")") (allDeps deps));
|
||||
defaultImplementation = "sbcl";
|
||||
|
||||
# 'genCompileLisp' generates a Lisp file that instructs SBCL to
|
||||
# compile the provided list of Lisp source files to $out.
|
||||
genCompileLisp = srcs: deps: writeText "compile.lisp" ''
|
||||
;; This file compiles the specified sources into the Nix build
|
||||
;; directory, creating one FASL file for each source.
|
||||
(require 'sb-posix)
|
||||
# Generates lisp code which instructs the given lisp implementation to load
|
||||
# all the given dependencies.
|
||||
genLoadLispGeneric = impl: deps:
|
||||
lib.concatStringsSep "\n"
|
||||
(map (lib: "(load \"${lib}/${lib.lispName}.${impl.faslExt}\")")
|
||||
(allDeps impl deps));
|
||||
|
||||
${genLoadLisp deps}
|
||||
|
||||
(defun nix-compile-lisp (file srcfile)
|
||||
(let ((outfile (make-pathname :type "fasl"
|
||||
:directory (or (sb-posix:getenv "NIX_BUILD_TOP")
|
||||
(error "not running in a Nix build"))
|
||||
:name (substitute #\- #\/ srcfile))))
|
||||
(multiple-value-bind (_outfile _warnings-p failure-p)
|
||||
(compile-file srcfile :output-file outfile)
|
||||
(if failure-p (sb-posix:exit 1)
|
||||
(progn
|
||||
;; For the case of multiple files belonging to the same
|
||||
;; library being compiled, load them in order:
|
||||
(load outfile)
|
||||
|
||||
;; Write them to the FASL list in the same order:
|
||||
(format file "cat ~a~%" (namestring outfile)))))))
|
||||
|
||||
(let ((*compile-verbose* t)
|
||||
;; FASL files are compiled into the working directory of the
|
||||
;; build and *then* moved to the correct out location.
|
||||
(pwd (sb-posix:getcwd)))
|
||||
|
||||
(with-open-file (file "cat_fasls"
|
||||
:direction :output
|
||||
:if-does-not-exist :create)
|
||||
|
||||
;; These forms were inserted by the Nix build:
|
||||
${
|
||||
lib.concatStringsSep "\n" (map (src: "(nix-compile-lisp file \"${src}\")") srcs)
|
||||
}
|
||||
))
|
||||
'';
|
||||
|
||||
# 'genTestLisp' generates a Lisp file that loads all sources and deps and
|
||||
# executes expression
|
||||
genTestLisp = name: srcs: deps: expression: writeText "${name}.lisp" ''
|
||||
# 'genTestLispGeneric' generates a Lisp file that loads all sources and deps
|
||||
# and executes expression for a given implementation description.
|
||||
genTestLispGeneric = impl: { name, srcs, deps, expression }: writeText "${name}.lisp" ''
|
||||
;; Dependencies
|
||||
${genLoadLisp deps}
|
||||
${impl.genLoadLisp deps}
|
||||
|
||||
;; Sources
|
||||
${lib.concatStringsSep "\n" (map (src: "(load \"${src}\")") srcs)}
|
||||
|
|
@ -78,9 +41,16 @@ let
|
|||
dependsOn = a: b: builtins.elem a b.lispDeps;
|
||||
|
||||
# 'allDeps' flattens the list of dependencies (and their
|
||||
# dependencies) into one ordered list of unique deps.
|
||||
allDeps = deps: (lib.toposort dependsOn (lib.unique (
|
||||
lib.flatten (deps ++ (map (d: d.lispDeps) deps))
|
||||
# dependencies) into one ordered list of unique deps which
|
||||
# all use the given implementation.
|
||||
allDeps = impl: deps: let
|
||||
# The override _should_ propagate itself recursively, as every derivation
|
||||
# would only expose its actually used dependencies
|
||||
deps' = builtins.map (dep: dep.overrideLisp or (lib.const dep) (_: {
|
||||
implementation = impl.name;
|
||||
})) deps;
|
||||
in (lib.toposort dependsOn (lib.unique (
|
||||
lib.flatten (deps' ++ (map (d: d.lispDeps) deps'))
|
||||
))).result;
|
||||
|
||||
# 'allNative' extracts all native dependencies of a dependency list
|
||||
|
|
@ -90,26 +60,6 @@ let
|
|||
lib.flatten (native ++ (map (d: d.lispNativeDeps) deps))
|
||||
);
|
||||
|
||||
# 'genDumpLisp' generates a Lisp file that instructs SBCL to dump
|
||||
# the currently loaded image as an executable to $out/bin/$name.
|
||||
#
|
||||
# TODO(tazjin): Compression is currently unsupported because the
|
||||
# SBCL in nixpkgs is, by default, not compiled with zlib support.
|
||||
genDumpLisp = name: main: deps: writeText "dump.lisp" ''
|
||||
(require 'sb-posix)
|
||||
|
||||
${genLoadLisp deps}
|
||||
|
||||
(let* ((bindir (concatenate 'string (sb-posix:getenv "out") "/bin"))
|
||||
(outpath (make-pathname :name "${name}"
|
||||
:directory bindir)))
|
||||
(save-lisp-and-die outpath
|
||||
:executable t
|
||||
:toplevel (function ${main})
|
||||
:purify t))
|
||||
;;
|
||||
'';
|
||||
|
||||
# Add an `overrideLisp` attribute to a function result that works
|
||||
# similar to `overrideAttrs`, but is used specifically for the
|
||||
# arguments passed to Lisp builders.
|
||||
|
|
@ -119,44 +69,176 @@ let
|
|||
|
||||
# 'testSuite' builds a Common Lisp test suite that loads all of srcs and deps,
|
||||
# and then executes expression to check its result
|
||||
testSuite = { name, expression, srcs, deps ? [], native ? [] }:
|
||||
testSuite = { name, expression, srcs, deps ? [], native ? [], impl }:
|
||||
let
|
||||
lispNativeDeps = allNative native deps;
|
||||
lispDeps = allDeps deps;
|
||||
lispDeps = allDeps impl deps;
|
||||
in runCommandNoCC name {
|
||||
LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps;
|
||||
LANG = "C.UTF-8";
|
||||
} ''
|
||||
echo "Running test suite ${name}"
|
||||
|
||||
${sbcl}/bin/sbcl --script ${genTestLisp name srcs deps expression} \
|
||||
| tee $out
|
||||
${impl.runScript} ${
|
||||
impl.genTestLisp {
|
||||
inherit name srcs deps expression;
|
||||
}
|
||||
} | tee $out
|
||||
|
||||
echo "Test suite ${name} succeeded"
|
||||
'';
|
||||
|
||||
# 'impls' is an attribute set of attribute sets which describe how to do common
|
||||
# tasks when building for different Common Lisp implementations. Each
|
||||
# implementation set has the following members:
|
||||
#
|
||||
# Required members:
|
||||
#
|
||||
# - runScript :: string
|
||||
# Describes how to invoke the implementation from the shell, so it runs a
|
||||
# lisp file as a script and exits.
|
||||
# - faslExt :: string
|
||||
# File extension of the implementations loadable (FASL) files.
|
||||
# Implementations are free to generate native object files, but with the way
|
||||
# buildLisp works it is required that we can also 'load' libraries, so
|
||||
# (additionally) building a FASL or equivalent is required.
|
||||
# - genLoadLisp :: [ dependency ] -> string
|
||||
# Returns lisp code to 'load' the given dependencies. 'genLoadLispGeneric'
|
||||
# should work for most dependencies.
|
||||
# - genCompileLisp :: { name, srcs, deps } -> file
|
||||
# Builds a lisp file which instructs the implementation to build a library
|
||||
# from the given source files when executed. After running at least
|
||||
# the file "$out/${name}.${impls.${implementation}.faslExt}" should have
|
||||
# been created.
|
||||
# - genDumpLisp :: { name, main, deps } -> file
|
||||
# Builds a lisp file which instructs the implementation to build an
|
||||
# executable which runs 'main' (and exits) where 'main' is available from
|
||||
# 'deps'. The executable should be created as "$out/bin/${name}", usually
|
||||
# by dumping the lisp image with the replaced toplevel function replaced.
|
||||
# - genTestLisp :: { name, srcs, deps, expression } -> file
|
||||
# Builds a lisp file which loads the given 'deps' and 'srcs' files and
|
||||
# then evaluates 'expression'. Depending on whether 'expression' returns
|
||||
# true or false, the script must exit with a zero or non-zero exit code.
|
||||
# 'genTestLispGeneric' will work for most implementations.
|
||||
# - lispWith :: [ dependency ] -> drv
|
||||
# Builds a script (or dumped image) which when executed loads (or has
|
||||
# loaded) all given dependencies. When built this should create an executable
|
||||
# at "$out/bin/${implementation}".
|
||||
impls = lib.mapAttrs (name: v: { inherit name; } // v) {
|
||||
sbcl = {
|
||||
runScript = "${sbcl}/bin/sbcl --script";
|
||||
faslExt = "fasl";
|
||||
|
||||
# 'genLoadLisp' generates Lisp code that instructs SBCL to load all
|
||||
# the provided Lisp libraries.
|
||||
genLoadLisp = genLoadLispGeneric impls.sbcl;
|
||||
|
||||
# 'genCompileLisp' generates a Lisp file that instructs SBCL to
|
||||
# compile the provided list of Lisp source files to "$out/${name}.fasl".
|
||||
genCompileLisp = { name, srcs, deps }: writeText "sbcl-compile.lisp" ''
|
||||
;; This file compiles the specified sources into the Nix build
|
||||
;; directory, creating one FASL file for each source.
|
||||
(require 'sb-posix)
|
||||
|
||||
${impls.sbcl.genLoadLisp deps}
|
||||
|
||||
(defun nix-compile-lisp (srcfile)
|
||||
(let ((outfile (make-pathname :type "fasl"
|
||||
:directory (or (sb-posix:getenv "NIX_BUILD_TOP")
|
||||
(error "not running in a Nix build"))
|
||||
:name (substitute #\- #\/ srcfile))))
|
||||
(multiple-value-bind (out-truename _warnings-p failure-p)
|
||||
(compile-file srcfile :output-file outfile)
|
||||
(if failure-p (sb-posix:exit 1)
|
||||
(progn
|
||||
;; For the case of multiple files belonging to the same
|
||||
;; library being compiled, load them in order:
|
||||
(load out-truename)
|
||||
|
||||
;; Return pathname as a string for cat-ting it later
|
||||
(namestring out-truename))))))
|
||||
|
||||
(let ((*compile-verbose* t)
|
||||
(catted-fasl (make-pathname :type "fasl"
|
||||
:directory (or (sb-posix:getenv "out")
|
||||
(error "not running in a Nix build"))
|
||||
:name "${name}")))
|
||||
|
||||
(with-open-file (file catted-fasl
|
||||
:direction :output
|
||||
:if-does-not-exist :create)
|
||||
|
||||
;; SBCL's FASL files can just be bundled together using cat
|
||||
(sb-ext:run-program "cat"
|
||||
(mapcar #'nix-compile-lisp
|
||||
;; These forms were inserted by the Nix build:
|
||||
'(${
|
||||
lib.concatMapStringsSep "\n" (src: "\"${src}\"") srcs
|
||||
}))
|
||||
:output file :search t)))
|
||||
'';
|
||||
|
||||
# 'genDumpLisp' generates a Lisp file that instructs SBCL to dump
|
||||
# the currently loaded image as an executable to $out/bin/$name.
|
||||
#
|
||||
# TODO(tazjin): Compression is currently unsupported because the
|
||||
# SBCL in nixpkgs is, by default, not compiled with zlib support.
|
||||
genDumpLisp = { name, main, deps }: writeText "sbcl-dump.lisp" ''
|
||||
(require 'sb-posix)
|
||||
|
||||
${impls.sbcl.genLoadLisp deps}
|
||||
|
||||
(let* ((bindir (concatenate 'string (sb-posix:getenv "out") "/bin"))
|
||||
(outpath (make-pathname :name "${name}"
|
||||
:directory bindir)))
|
||||
(save-lisp-and-die outpath
|
||||
:executable t
|
||||
:toplevel (function ${main})
|
||||
:purify t))
|
||||
'';
|
||||
|
||||
genTestLisp = genTestLispGeneric impls.sbcl;
|
||||
|
||||
lispWith = deps:
|
||||
let lispDeps = filter (d: !d.lispBinary) (allDeps impls.sbcl deps);
|
||||
in writeShellScriptBin "sbcl" ''
|
||||
export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}"
|
||||
export LANG="C.UTF-8"
|
||||
exec ${sbcl}/bin/sbcl ${
|
||||
lib.optionalString (deps != [])
|
||||
"--load ${writeText "load.lisp" (impls.sbcl.genLoadLisp lispDeps)}"
|
||||
} $@
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
#
|
||||
# Public API functions
|
||||
#
|
||||
|
||||
# 'library' builds a list of Common Lisp files into a single FASL
|
||||
# which can then be loaded into SBCL.
|
||||
# 'library' builds a list of Common Lisp files into an implementation
|
||||
# specific library format, usually a single FASL file, which can then be
|
||||
# loaded and built into an executable via 'program'.
|
||||
library =
|
||||
{ name
|
||||
, implementation ? defaultImplementation
|
||||
, srcs
|
||||
, deps ? []
|
||||
, native ? []
|
||||
, tests ? null
|
||||
}:
|
||||
let
|
||||
impl = impls."${implementation}" or
|
||||
(builtins.throw "Unkown Common Lisp Implementation ${implementation}");
|
||||
lispNativeDeps = (allNative native deps);
|
||||
lispDeps = allDeps deps;
|
||||
lispDeps = allDeps impl deps;
|
||||
testDrv = if ! isNull tests
|
||||
then testSuite {
|
||||
name = tests.name or "${name}-test";
|
||||
srcs = srcs ++ (tests.srcs or []);
|
||||
deps = deps ++ (tests.deps or []);
|
||||
expression = tests.expression;
|
||||
inherit impl;
|
||||
}
|
||||
else null;
|
||||
in lib.fix (self: runCommandNoCC "${name}-cllib" {
|
||||
|
|
@ -167,28 +249,28 @@ let
|
|||
lispName = name;
|
||||
lispBinary = false;
|
||||
tests = testDrv;
|
||||
sbcl = sbclWith [ self ];
|
||||
${implementation} = impl.lispWith [ self ];
|
||||
};
|
||||
} ''
|
||||
${if ! isNull testDrv
|
||||
then "echo 'Test ${testDrv} succeeded'"
|
||||
else "echo 'No tests run'"}
|
||||
${sbcl}/bin/sbcl --script ${genCompileLisp srcs lispDeps}
|
||||
|
||||
echo "Compilation finished, assembling FASL files"
|
||||
|
||||
# FASL files can be combined by simply concatenating them
|
||||
# together, but it needs to be in the compilation order.
|
||||
mkdir $out
|
||||
|
||||
chmod +x cat_fasls
|
||||
./cat_fasls > $out/${name}.fasl
|
||||
${impl.runScript} ${
|
||||
impl.genCompileLisp {
|
||||
inherit name srcs;
|
||||
deps = lispDeps;
|
||||
}
|
||||
}
|
||||
'');
|
||||
|
||||
# 'program' creates an executable containing a dumped image of the
|
||||
# 'program' creates an executable, usually containing a dumped image of the
|
||||
# specified sources and dependencies.
|
||||
program =
|
||||
{ name
|
||||
, implementation ? defaultImplementation
|
||||
, main ? "${name}:main"
|
||||
, srcs
|
||||
, deps ? []
|
||||
|
|
@ -196,9 +278,12 @@ let
|
|||
, tests ? null
|
||||
}:
|
||||
let
|
||||
lispDeps = allDeps deps;
|
||||
impl = impls."${implementation}" or
|
||||
(builtins.throw "Unkown Common Lisp Implementation ${implementation}");
|
||||
lispDeps = allDeps impl deps;
|
||||
libPath = lib.makeLibraryPath (allNative native lispDeps);
|
||||
selfLib = library {
|
||||
# overriding is used internally to propagate the implementation to use
|
||||
selfLib = (makeOverridable library) {
|
||||
inherit name srcs native;
|
||||
deps = lispDeps;
|
||||
};
|
||||
|
|
@ -210,6 +295,7 @@ let
|
|||
srcs ++ (tests.srcs or []));
|
||||
deps = deps ++ (tests.deps or []);
|
||||
expression = tests.expression;
|
||||
inherit impl;
|
||||
}
|
||||
else null;
|
||||
in lib.fix (self: runCommandNoCC "${name}" {
|
||||
|
|
@ -222,7 +308,7 @@ let
|
|||
lispNativeDeps = native;
|
||||
lispBinary = true;
|
||||
tests = testDrv;
|
||||
sbcl = sbclWith [ self ];
|
||||
${implementation} = impl.lispWith [ self ];
|
||||
};
|
||||
} ''
|
||||
${if ! isNull testDrv
|
||||
|
|
@ -230,8 +316,11 @@ let
|
|||
else ""}
|
||||
mkdir -p $out/bin
|
||||
|
||||
${sbcl}/bin/sbcl --script ${
|
||||
genDumpLisp name main ([ selfLib ] ++ lispDeps)
|
||||
${impl.runScript} ${
|
||||
impl.genDumpLisp {
|
||||
inherit name main;
|
||||
deps = ([ selfLib ] ++ lispDeps);
|
||||
}
|
||||
}
|
||||
|
||||
wrapProgram $out/bin/${name} --prefix LD_LIBRARY_PATH : "${libPath}"
|
||||
|
|
@ -243,18 +332,12 @@ let
|
|||
inherit name;
|
||||
srcs = lib.singleton (builtins.toFile "${name}.lisp" "(require '${name})");
|
||||
};
|
||||
|
||||
# 'sbclWith' creates an image with the specified libraries /
|
||||
# programs loaded.
|
||||
sbclWith = deps:
|
||||
let lispDeps = filter (d: !d.lispBinary) (allDeps deps);
|
||||
in writeShellScriptBin "sbcl" ''
|
||||
export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}"
|
||||
export LANG="C.UTF-8"
|
||||
exec ${sbcl}/bin/sbcl ${lib.optionalString (deps != []) "--load ${writeText "load.lisp" (genLoadLisp lispDeps)}"} $@
|
||||
'';
|
||||
in {
|
||||
library = makeOverridable library;
|
||||
program = makeOverridable program;
|
||||
inherit sbclWith bundled;
|
||||
inherit bundled;
|
||||
|
||||
# 'sbclWith' creates an image with the specified libraries /
|
||||
# programs loaded in SBCL.
|
||||
sbclWith = impls.sbcl.lispWith;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue