build modules at same time as main kernel vmlinux

This changes the practice for building kernel modules: now we expect
that the appropriate Kconfig symbols are set to =m in
config.kernel.config, and then use pkgs.kmodloader to create
a service that loads and unloads all the modules depended on by
a particular requirement.

Note that modules won't be installed on the target device just by
virue of having been built: only the modules that are referenced by a
kmodloader package will be in the closure.

An example may make this clearer: see modules/firewall/default.nix
in this commit.

Why?

If you have a compiled Linux kernel source tree and you change some
symbol from "is not set" to m and then run make modules, you cannot in
general expect that newly compiled module to work. This is because
there are places in the build of the main kernel where it looks to see
which modules _may_ be defined and uses that information to
accommodate them.

For example in an in-kernel build of

  https://github.com/torvalds/linux/blob/master/net/netfilter/core.c#L689

some symbols are defined only if CONFIG_NF_CONNTRACK is set, meaning
this code won't work if we have it unset initially then try later to
enable it and build modules only. Or see

  https://github.com/torvalds/linux/blob/master/include/linux/netdevice.h#L160

modules/firewall/default.nix

@@ -10,45 +10,8 @@ let
   inherit (pkgs) liminix;
   inherit (pkgs.liminix.services) oneshot;
 
-  kconf = isModule :
-    # setting isModule false is utterly untested and mostly
-    # unimplemented: I say this to preempt any "how on earth is this
-    # even supposed to work?" questions
-    let yes = if isModule then "m" else "y";
-    in {
-      NETFILTER = "y";
-      NETFILTER_ADVANCED = "y";
-      NETFILTER_NETLINK = yes;
-      NF_CONNTRACK = yes;
-
-      IP6_NF_IPTABLES=  yes;
-      IP_NF_IPTABLES = yes;
-      IP_NF_NAT = yes;
-      IP_NF_TARGET_MASQUERADE = yes;
-
-      NFT_CT = yes;
-      NFT_FIB_IPV4 = yes;
-      NFT_FIB_IPV6 = yes;
-      NFT_LOG = yes;
-      NFT_MASQ = yes;
-      NFT_NAT = yes;
-      NFT_REJECT = yes;
-      NFT_REJECT_INET = yes;
-
-      NF_CT_PROTO_DCCP = "y";
-      NF_CT_PROTO_SCTP = "y";
-      NF_CT_PROTO_UDPLITE = "y";
-      NF_LOG_SYSLOG = yes;
-      NF_NAT = yes;
-      NF_NAT_MASQUERADE = "y";
-      NF_TABLES = yes;
-      NF_TABLES_INET = "y";
-      NF_TABLES_IPV4 = "y";
-      NF_TABLES_IPV6 = "y";
-    };
-  kmodules = pkgs.kernel-modules.override {
-    kernelSrc = config.system.outputs.kernel.src;
-    modulesupport = config.system.outputs.kernel.modulesupport;
+  kmodules = pkgs.kmodloader.override {
+    inherit (config.system.outputs) kernel;
     targets = [
       "nft_fib_ipv4"
       "nft_fib_ipv6"
@@ -82,12 +45,6 @@ let
       "xt_nat"
       "xt_tcpudp"
     ];
-    kconfig = kconf true;
-  };
-  loadModules = oneshot {
-    name = "firewall-modules";
-    up = "sh ${kmodules}/load.sh";
-    down = "sh ${kmodules}/unload.sh";
   };
 in
 {
@@ -107,11 +64,41 @@ in
       in svc // {
         build = args :
           let args' = args // {
-                dependencies = (args.dependencies or []) ++ [loadModules];
+                dependencies = (args.dependencies or []) ++ [kmodules];
               };
           in svc.build args' ;
       };
 
-    kernel.config = kconf true;
+    kernel.config = {
+      NETFILTER = "y";
+      NETFILTER_ADVANCED = "y";
+      NETFILTER_NETLINK = "m";
+      NF_CONNTRACK = "m";
+
+      IP6_NF_IPTABLES=  "m";
+      IP_NF_IPTABLES = "m";
+      IP_NF_NAT = "m";
+      IP_NF_TARGET_MASQUERADE = "m";
+
+      NFT_CT = "m";
+      NFT_FIB_IPV4 = "m";
+      NFT_FIB_IPV6 = "m";
+      NFT_LOG = "m";
+      NFT_MASQ = "m";
+      NFT_NAT = "m";
+      NFT_REJECT = "m";
+      NFT_REJECT_INET = "m";
+
+      NF_CT_PROTO_DCCP = "y";
+      NF_CT_PROTO_SCTP = "y";
+      NF_CT_PROTO_UDPLITE = "y";
+      NF_LOG_SYSLOG = "m";
+      NF_NAT = "m";
+      NF_NAT_MASQUERADE = "y";
+      NF_TABLES = "m";
+      NF_TABLES_INET = "y";
+      NF_TABLES_IPV4 = "y";
+      NF_TABLES_IPV6 = "y";
+    };
   };
 }

modules/kernel/default.nix

@@ -80,8 +80,7 @@ in {
           config.kernel.conditionalConfig;
         k = liminix.builders.kernel.override {
           config = mergedConfig;
-          version = builtins.trace config.kernel.version config.kernel.version;
-          inherit (config.kernel)  src extraPatchPhase;
+          inherit (config.kernel) version src extraPatchPhase;
           targets = config.kernel.makeTargets;
         };
       in {

pkgs/default.nix

@@ -67,6 +67,7 @@ in {
   initramfs-peek = callPackage ./initramfs-peek {};
   kernel-backport = callPackage ./kernel-backport {};
   kernel-modules  = callPackage ./kernel-modules {};
+  kmodloader = callPackage ./kmodloader {};
   levitate = callPackage ./levitate {};
   libubootenv = callPackage ./libubootenv {};
   linotify = callPackage ./linotify {};

pkgs/kernel-modules/Makefile

@@ -1,3 +0,0 @@
-
-
-# obj-m += net/ipv4/netfilter/nft_fib_ipv4.o

pkgs/kernel-modules/default.nix

@@ -1,50 +0,0 @@
-{
-  stdenv
-, buildPackages
-, kernelSrc  ? null
-, modulesupport ? null
-, targets ? []
-, kconfig ? {}
-, openssl
-, writeText
-, lib
-}:
-let
-  writeConfig = import ../kernel/write-kconfig.nix { inherit lib writeText; };
-  arch = stdenv.hostPlatform.linuxArch;
-in stdenv.mkDerivation {
-  name = "kernel-modules";
-
-  nativeBuildInputs = [buildPackages.stdenv.cc] ++
-                      (with buildPackages.pkgs; [
-                        bc bison flex
-                        openssl
-                        cpio
-                        kmod
-                      ]);
-  CC = "${stdenv.cc.bintools.targetPrefix}gcc";
-  HOST_EXTRACFLAGS = with buildPackages.pkgs;
-    "-I${buildPackages.openssl.dev}/include -L${buildPackages.openssl.out}/lib";
-  CROSS_COMPILE = stdenv.cc.bintools.targetPrefix;
-  ARCH = arch;
-  KBUILD_BUILD_HOST = "liminix.builder";
-
-  buildPhase = ''
-    cat ${writeConfig "kconfig" kconfig}  > .more-config
-    cat .more-config >> .config
-    make olddefconfig
-    for v in $(cat .more-config) ; do grep $v .config || (echo Missing $v && exit 1);done
-    make modules
-  '';
-  src =  modulesupport;
-  installPhase = ''
-    mkdir -p $out/lib/modules/0.0
-    find . -name \*.ko | cpio --verbose --make-directories -p $out/lib/modules/0.0
-    depmod -b $out -v 0.0
-    touch $out/load.sh
-    for i in ${lib.concatStringsSep " " targets}; do
-      modprobe -S 0.0 -d $out --show-depends $i >> $out/load.sh
-    done
-    tac < $out/load.sh | sed 's/^insmod/rmmod/g' > $out/unload.sh
-  '';
-}

pkgs/kernel/default.nix

@@ -104,8 +104,7 @@ stdenv.mkDerivation rec {
     mkdir -p $headers
     cp -a include .config $headers/
     mkdir -p $modulesupport
-    cp modules.* $modulesupport
-    make clean modules_prepare
+    make modules
     cp -a . $modulesupport
   '';
 }

pkgs/kmodloader/default.nix

@@ -0,0 +1,41 @@
+{
+  liminix
+, lib
+, targets ? []
+, kernel ? null
+, runCommand
+, pkgsBuildBuild
+} :
+let
+  inherit (liminix.services) oneshot;
+  inherit (lib) concatStringsSep;
+  loader =  runCommand "modules" {
+    nativeBuildInputs = with pkgsBuildBuild ;[
+      kmod cpio gawk
+    ];
+  } ''
+    kernel=${kernel.modulesupport}
+
+    mkdir -p lib/modules/0.0
+    (cd $kernel && find . -name \*.ko | cpio --verbose --make-directories -p $NIX_BUILD_TOP/lib/modules/0.0)
+    cp $kernel/modules.* lib/modules/0.0
+    depmod -b . 0.0
+
+    (for i in ${lib.concatStringsSep " " targets}; do
+      modprobe -S 0.0 -d  $NIX_BUILD_TOP --show-depends $i | sed "s,^insmod $NIX_BUILD_TOP/lib/modules/0.0/,,g"
+    done) | awk '!a[$0]++' > load-order
+
+    mkdir $out
+    for i in $(cat load-order); do
+      install -v $NIX_BUILD_TOP/lib/modules/0.0/$i -D $out/$i
+    done
+    echo "O=$out" > $out/load.sh
+    sed "s,^,insmod \$O/,g" < load-order >> $out/load.sh
+    echo "O=$out" > $out/unload.sh
+    tac load-order | sed "s,^,rmmod \$O/,g" > $out/unload.sh
+  '';
+in oneshot {
+  name = "kmodloader-"  + (concatStringsSep "-" targets);
+  up = "sh ${loader}/load.sh";
+  down = "sh ${loader}/unload.sh";
+}