feat(nix/buildLisp): add ccl

This adds support for Clozure's CL implementation to buildLisp. This is
quite trivial in comparison to ECL since SBCL and CCL have very similar
in how they work (so much so that CCL also suffers from b/136).

Also the similarities in the code actually added here are striking, so
I'll try to make an effort to reduce the code duplication in the
future.

To fix builds with CCL the following changes were made:

* //3p/lisp/nibbles: The double inclusion of the types.lisp file was
  fixed. CCL doesn't like double definitions and refuses to compile
  otherwise.

* //3p/lisp/physical-quantities: Update to a new bug fix release which
  contains a compilation fix for CCL.

* //3p/lisp/routes: apply a patch fixing the build which was previously
  failing due to a double definition.

* //3p/lisp/usocket: only depend on sb-bsd-sockets for SBCL and ECL, the
  latter of which seems to have a SBCL compatible implementation of the
  package.

* Conditionally include a few CCL-specific source files and add
  `badImplementation` entries for the remaining failures which are
  //fun/gemma (to be expected) and //web/panettone which fails with an
  incredibly vague message.

Change-Id: I666efdc39a0f16ee1bb6e23225784c709b04e740
Reviewed-on: https://cl.tvl.fyi/c/depot/+/3350
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>

fun/gemma/default.nix

@@ -50,6 +50,7 @@ in depot.nix.buildLisp.program {
 
   # depends on SBCL
   brokenOn = [
+    "ccl"
     "ecl"
   ];
 }

nix/buildLisp/default.nix

@@ -8,7 +8,8 @@
 
 let
   inherit (builtins) map elemAt match filter;
-  inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl ecl-static;
+  inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl ecl-static ccl;
+  inherit (pkgs.stdenv) targetPlatform;
 
   #
   # Internal helper definitions
@@ -26,6 +27,7 @@ let
           (lambda (error hook)
             (declare (ignore hook))
             (format *error-output* "~%Unhandled error: ~a~%" error)
+            #+ccl (quit 1)
             #+ecl (ext:quit 1)))
   '';
 
@@ -457,6 +459,106 @@ let
         ln -s "${ecl-static}/lib/ecl-${ecl-static.version}/lib${name}.a" "$out/${name}.a"
       '';
     };
+    ccl = {
+      # Relatively bespoke wrapper script necessary to make CCL just™ execute
+      # a lisp file as a script.
+      runScript = pkgs.writers.writeBash "ccl" ''
+        # don't print intro message etc.
+        args=("--quiet")
+
+        # makes CCL crash on error instead of entering the debugger
+        args+=("--load" "${disableDebugger}")
+
+        # load files from command line in order
+        for f in "$@"; do
+          args+=("--load" "$f")
+        done
+
+        # Exit if everything was processed successfully
+        args+=("--eval" "(quit)")
+
+        exec ${ccl}/bin/ccl ''${args[@]}
+      '';
+
+      # See https://ccl.clozure.com/docs/ccl.html#building-definitions
+      faslExt =
+        /**/ if targetPlatform.isPowerPC && targetPlatform.is32bit then "pfsl"
+        else if targetPlatform.isPowerPC && targetPlatform.is64bit then "p64fsl"
+        else if targetPlatform.isx86_64 && targetPlatform.isLinux then "lx64fsl"
+        else if targetPlatform.isx86_32 && targetPlatform.isLinux then "lx32fsl"
+        else if targetPlatform.isAarch32 && targetPlatform.isLinux then "lafsl"
+        else if targetPlatform.isx86_32 && targetPlatform.isDarwin then "dx32fsl"
+        else if targetPlatform.isx86_64 && targetPlatform.isDarwin then "dx64fsl"
+        else if targetPlatform.isx86_64 && targetPlatform.isDarwin then "dx64fsl"
+        else if targetPlatform.isx86_32 && targetPlatform.isFreeBSD then "fx32fsl"
+        else if targetPlatform.isx86_64 && targetPlatform.isFreeBSD then "fx64fsl"
+        else if targetPlatform.isx86_32 && targetPlatform.isWindows then "wx32fsl"
+        else if targetPlatform.isx86_64 && targetPlatform.isWindows then "wx64fsl"
+        else builtins.throw "Don't know what FASLs are called for this platform: "
+          + pkgs.stdenv.targetPlatform.system;
+
+      genLoadLisp = genLoadLispGeneric impls.ccl;
+
+      genCompileLisp = { name, srcs, deps }: writeText "ccl-compile.lisp" ''
+        ${impls.ccl.genLoadLisp deps}
+
+        (defun getenv-or-fail (var)
+          (or (getenv var)
+              (error (format nil "Missing expected environment variable ~A" var))))
+
+        (defun nix-compile-file (srcfile)
+          "Trivial wrapper around COMPILE-FILE which causes CCL to exit if
+          compilation fails and LOADs the compiled file on success."
+          (let ((output (make-pathname :name (substitute #\_ #\/ srcfile)
+                                       :type "${impls.ccl.faslExt}"
+                                       :directory (getenv-or-fail "NIX_BUILD_TOP"))))
+            (multiple-value-bind (out-truename _warnings-p failure-p)
+                (compile-file srcfile :output-file output :print t :verbose t)
+                (declare (ignore _warnings-p))
+              (if failure-p (quit 1)
+                  (progn (load out-truename) out-truename)))))
+
+        (fasl-concatenate (make-pathname :name "${name}" :type "${impls.ccl.faslExt}"
+                                         :directory (getenv-or-fail "out"))
+                          (mapcar #'nix-compile-file
+                                  ;; These forms where inserted by the Nix build
+                                  '(${
+                                      lib.concatMapStrings (src: ''
+                                        "${src}"
+                                      '') srcs
+                                   })))
+      '';
+
+      genDumpLisp = { name, main, deps }: writeText "ccl-dump.lisp" ''
+        ${impls.ccl.genLoadLisp deps}
+
+        (let* ((out (or (getenv "out") (error "Not running in a Nix build")))
+               (bindir (concatenate 'string out "/bin/"))
+               (executable (make-pathname :directory bindir :name "${name}")))
+
+          (save-application executable
+                            :purify t
+                            :error-handler :quit
+                            :toplevel-function (function ${main})
+                            :mode #o755
+                            ;; TODO(sterni): use :native t on macOS
+                            :prepend-kernel t))
+      '';
+
+      wrapProgram = true;
+
+      genTestLisp = genTestLispGeneric impls.ccl;
+
+      lispWith = deps:
+        let lispDeps = filter (d: !d.lispBinary) (allDeps impls.ccl deps);
+        in writeShellScriptBin "ccl" ''
+          export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}"
+          exec ${ccl}/bin/ccl ${
+            lib.optionalString (deps != [])
+              "--load ${writeText "load.lisp" (impls.ccl.genLoadLisp lispDeps)}"
+          } "$@"
+      '';
+    };
   };
 
   #

third_party/lisp/cffi.nix

@@ -19,6 +19,7 @@ in buildLisp.library {
     {
       ecl = src + "/src/cffi-ecl.lisp";
       sbcl = src + "/src/cffi-sbcl.lisp";
+      ccl = src + "/src/cffi-openmcl.lisp";
     }
   ] ++ map (f: src + ("/src/" + f)) [
     "package.lisp"

third_party/lisp/cl-fad.nix

@@ -22,6 +22,9 @@ in buildLisp.library {
 
   srcs = map (f: src + ("/" + f)) [
     "packages.lisp"
+  ] ++ [
+    { ccl =  "${src}/openmcl.lisp"; }
+  ] ++ map (f: src + ("/" + f)) [
     "fad.lisp"
     "path.lisp"
     "temporary-files.lisp"

third_party/lisp/cl-plus-ssl.nix

@@ -17,8 +17,11 @@ in buildLisp.library {
     trivial-features
     trivial-garbage
     trivial-gray-streams
-    (buildLisp.bundled "uiop")
-    (buildLisp.bundled "sb-posix")
+    {
+      scbl = buildLisp.bundled "uiop";
+      default = buildLisp.bundled "asdf";
+    }
+    { sbcl = buildLisp.bundled "sb-posix"; }
   ];
 
   native = [ pkgs.openssl ];

third_party/lisp/closer-mop.nix

@@ -18,6 +18,7 @@ in depot.nix.buildLisp.library {
     {
       sbcl = "${src}/closer-sbcl.lisp";
       ecl = "${src}/closer-ecl.lisp";
+      ccl = "${src}/closer-clozure.lisp";
     }
   ];
 }

third_party/lisp/ironclad.nix

@@ -43,6 +43,8 @@ in depot.nix.buildLisp.library {
     { sbcl = getSrc "opt/sbcl/x86oid-vm.lisp"; }
 
     { ecl = getSrc "opt/ecl/c-functions.lisp"; }
+
+    { ccl = getSrc "opt/ccl/x86oid-vm.lisp"; }
   ] ++ map getSrc [
 
     "common.lisp"

third_party/lisp/nibbles.nix

@@ -21,7 +21,6 @@ in depot.nix.buildLisp.library {
     "package.lisp"
     "types.lisp"
     "macro-utils.lisp"
-    "types.lisp"
     "vectors.lisp"
     "streams.lisp"
   ] ++ [

third_party/lisp/physical-quantities.nix

@@ -4,9 +4,10 @@ let
   src = pkgs.fetchFromGitHub {
     owner = "mrossini-ethz";
     repo = "physical-quantities";
-    rev = "8feb66ef3293fcb9ff4c4bd3ee872bfc385a590e";
-    sha256 = "1qznv0hmn2n7g9dxx1iw0qpr0pf2lnbahn0x0b3v50xzcb65kgig";
+    rev = "v0.2.1";
+    sha256 = "0mb2s94s6fhw5vfa89naalw7ld11sdsszlqpz0c65dvpfyfmmdmh";
   };
+
 in depot.nix.buildLisp.library {
   name = "physical-quantities";
 

third_party/lisp/routes.nix

@@ -2,11 +2,22 @@
 
 let
 
-  src = pkgs.fetchFromGitHub {
-    owner = "archimag";
-    repo = "cl-routes";
-    rev = "1b79e85aa653e1ec87e21ca745abe51547866fa9";
-    sha256 = "1zpk3cp2v8hm50ppjl10yxr437vv4552r8hylvizglzrq2ibsbr1";
+  src = pkgs.applyPatches {
+    name = "routes-source";
+    src = pkgs.fetchFromGitHub {
+      owner = "archimag";
+      repo = "cl-routes";
+      rev = "1b79e85aa653e1ec87e21ca745abe51547866fa9";
+      sha256 = "1zpk3cp2v8hm50ppjl10yxr437vv4552r8hylvizglzrq2ibsbr1";
+    };
+
+    patches = [
+      (pkgs.fetchpatch {
+        name = "fix-build-with-ccl.patch";
+        url = "https://github.com/archimag/cl-routes/commit/2296cdc316ef8e34310f2718b5d35a30040deee0.patch";
+        sha256 = "007c19kmymalam3v6l6y2qzch8xs3xnphrcclk1jrpggvigcmhax";
+      })
+    ];
   };
 
 in depot.nix.buildLisp.library {
@@ -24,5 +35,4 @@ in depot.nix.buildLisp.library {
     "route.lisp"
     "mapper.lisp"
   ];
-
 }

third_party/lisp/trivial-features.nix

@@ -10,6 +10,7 @@ in depot.nix.buildLisp.library {
     {
       sbcl = src + "/src/tf-sbcl.lisp";
       ecl = src + "/src/tf-ecl.lisp";
+      ccl = src + "/src/tf-openmcl.lisp";
     }
   ];
 }

third_party/lisp/usocket.nix

@@ -14,7 +14,10 @@ in buildLisp.library {
   name = "usocket";
   deps = with depot.third_party.lisp; [
     (buildLisp.bundled "asdf")
-    (buildLisp.bundled "sb-bsd-sockets")
+    {
+      ecl = buildLisp.bundled "sb-bsd-sockets";
+      sbcl = buildLisp.bundled "sb-bsd-sockets";
+    }
     split-sequence
   ];
 
@@ -40,5 +43,9 @@ in buildLisp.library {
     # but usocket also has some ECL specific code
     { ecl = "${src}/backend/sbcl.lisp"; }
     { ecl = "${src}/backend/ecl.lisp"; }
+
+    # Same for CCL
+    { ccl = "${src}/backend/openmcl.lisp"; }
+    { ccl = "${src}/backend/clozure.lisp"; }
   ]);
 }

web/panettone/default.nix

@@ -50,5 +50,6 @@ depot.nix.buildLisp.program {
 
   brokenOn = [
     "ecl" # dependencies use dynamic cffi
+    "ccl" # The value NIL is not of the expected type STRING. when loading model.lisp
   ];
 }