feat(modules): Disable modules with secrets

hive.nix

@@ -75,6 +75,7 @@ in
     {
       # Import the default modules
       imports = [
+        ./modules
         "${sources.infrastructure}/modules"
         (import "${sources.lix-module}/module.nix" {
           lix = pkgs.applyPatches {

keys/default.nix

@@ -13,6 +13,7 @@ rec {
   #          If not, you will face an angry maintainer
   _keys = (import "${_sources.infrastructure}/keys")._keys // {
     krz01 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP4o65gWOgNrxbSd3kiQIGZUM+YD6kuZOQtblvzUGsfB" ];
+    router02 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE5t0InDV9nTLEqXrenqMJZAjkCAmfzHk6LLLHme3k3j" ];
   };
 
   getKeys = ls: builtins.concatLists (builtins.map (getAttr _keys) ls);

machines/krz01/_configuration.nix

@@ -1,11 +1,4 @@
-{
+{ config, lib, ... }:
-  config,
-  lib,
-  pkgs,
-  meta,
-  name,
-  ...
-}:
 
 lib.extra.mkConfig {
   enabledModules = [
@@ -15,10 +8,13 @@ lib.extra.mkConfig {
   enabledServices = [
     # INFO: This list needs to stay sorted alphabetically
     # Machine learning API machine
-    "microvm-ml01"
+    # "microvm-ml01"
-    "microvm-router01"
+    # "microvm-router01"
     "nvidia-tesla-k80"
+    "ollama"
+    "whisper"
     "proxmox"
+    "networking"
   ];
 
   extraConfig = {
@@ -33,46 +29,12 @@ lib.extra.mkConfig {
       ];
     };
 
-    services.netbird.enable = true;
-
     # We are going to use CUDA here.
     nixpkgs.config.cudaSupport = true;
     hardware.graphics.enable = true;
-    environment.systemPackages = [
-      ((pkgs.openai-whisper-cpp.override { cudaPackages = pkgs.cudaPackages_11; }).overrideAttrs (old: {
-        src = pkgs.fetchFromGitHub {
-          owner = "ggerganov";
-          repo = "whisper.cpp";
-          rev = "v1.7.1";
-          hash = "sha256-EDFUVjud79ZRCzGbOh9L9NcXfN3ikvsqkVSOME9F9oo=";
-        };
-        env = {
-          WHISPER_CUBLAS = "";
-          GGML_CUDA = "1";
-        };
-        # We only need Compute Capability 3.7.
-        CUDA_ARCH_FLAGS = [ "sm_37" ];
-        # We are GPU-only anyway.
-        patches = (old.patches or [ ]) ++ [
-          ./no-weird-microarch.patch
-          ./all-nvcc-arch.patch
-        ];
-      }))
-    ];
 
-    services = {
+    services.netbird.enable = true;
-      ollama = {
+    networking.firewall.trustedInterfaces = [ "wt0" ];
-        enable = true;
-        host = meta.network.${name}.netbirdIp;
-        package = pkgs.callPackage ./ollama.nix {
-          cudaPackages = pkgs.cudaPackages_11;
-          # We need to thread our nvidia x11 driver for CUDA.
-          extraLibraries = [ config.hardware.nvidia.package ];
-        };
-      };
-    };
-
-    networking.firewall.interfaces.wt0.allowedTCPPorts = [ config.services.ollama.port ];
   };
 
   root = ./.;

machines/krz01/networking.nix

@@ -0,0 +1,24 @@
+{
+  systemd.network = {
+    networks = {
+      "10-eno1" = {
+        matchConfig.Name = [ "eno1" ];
+        networkConfig = {
+          Bridge = "vmbr0";
+        };
+      };
+
+      "10-vmbr0" = {
+        matchConfig.Name = "vmbr0";
+        linkConfig.RequiredForOnline = "routable";
+      };
+    };
+
+    netdevs."vmbr0" = {
+      netdevConfig = {
+        Name = "vmbr0";
+        Kind = "bridge";
+      };
+    };
+  };
+}

machines/krz01/ollama/K80-support.patch

@@ -0,0 +1,179 @@
+From 2abd226ff3093c5a9e18a618fba466853e7ebaf7 Mon Sep 17 00:00:00 2001
+From: Raito Bezarius <masterancpp@gmail.com>
+Date: Tue, 8 Oct 2024 18:27:41 +0200
+Subject: [PATCH] K80 support
+
+Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
+---
+ docs/development.md     |  6 +++-
+ docs/gpu.md             |  1 +
+ gpu/amd_linux.go        |  6 +++-
+ gpu/gpu.go              | 63 ++++++++++++++++++++++++++++++++++++-----
+ scripts/build_docker.sh |  2 +-
+ scripts/build_linux.sh  |  2 +-
+ 6 files changed, 69 insertions(+), 11 deletions(-)
+
+diff --git a/docs/development.md b/docs/development.md
+index 2f7b9ecf..9da35931 100644
+--- a/docs/development.md
++++ b/docs/development.md
+@@ -51,7 +51,11 @@ Typically the build scripts will auto-detect CUDA, however, if your Linux distro
+ or installation approach uses unusual paths, you can specify the location by
+ specifying an environment variable `CUDA_LIB_DIR` to the location of the shared
+ libraries, and `CUDACXX` to the location of the nvcc compiler. You can customize
+-a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "50;60;70")
++a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "35;37;50;60;70")
++
++To support GPUs older than Compute Capability 5.0, you will need to use an older version of
++the Driver from [Unix Driver Archive](https://www.nvidia.com/en-us/drivers/unix/) (tested with 470) and [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) (tested with cuda V11).  When you build Ollama, you will need to set two environment variable to adjust the minimum compute capability Ollama supports via `export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3\" \"-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5\"'"` and the `CMAKE_CUDA_ARCHITECTURES`.  To find the Compute Capability of your older GPU, refer to [GPU Compute Capability](https://developer.nvidia.com/cuda-gpus).
++
+ 
+ Then generate dependencies:
+ 
+diff --git a/docs/gpu.md b/docs/gpu.md
+index a6b559f0..66627611 100644
+--- a/docs/gpu.md
++++ b/docs/gpu.md
+@@ -28,6 +28,7 @@ Check your compute compatibility to see if your card is supported:
+ | 5.0                | GeForce GTX         | `GTX 750 Ti` `GTX 750` `NVS 810`                                                                            |
+ |                    | Quadro              | `K2200` `K1200` `K620` `M1200` `M520` `M5000M` `M4000M` `M3000M` `M2000M` `M1000M` `K620M` `M600M` `M500M`  |
+ 
++For building locally to support older GPUs, see [developer.md](./development.md#linux-cuda-nvidia)
+ 
+ ### GPU Selection
+ 
+diff --git a/gpu/amd_linux.go b/gpu/amd_linux.go
+index 6b08ac2e..768fb97a 100644
+--- a/gpu/amd_linux.go
++++ b/gpu/amd_linux.go
+@@ -159,7 +159,11 @@ func AMDGetGPUInfo() []GpuInfo {
+ 			return []GpuInfo{}
+ 		}
+ 
+-		if int(major) < RocmComputeMin {
++		minVer, err := strconv.Atoi(RocmComputeMajorMin)
++		if err != nil {
++			slog.Error("invalid RocmComputeMajorMin setting", "value", RocmComputeMajorMin, "error", err)
++		}
++		if int(major) < minVer {
+ 			slog.Warn(fmt.Sprintf("amdgpu too old gfx%d%x%x", major, minor, patch), "gpu", gpuID)
+ 			continue
+ 		}
+diff --git a/gpu/gpu.go b/gpu/gpu.go
+index 781e23df..60d68c33 100644
+--- a/gpu/gpu.go
++++ b/gpu/gpu.go
+@@ -16,6 +16,7 @@ import (
+ 	"os"
+ 	"path/filepath"
+ 	"runtime"
++	"strconv"
+ 	"strings"
+ 	"sync"
+ 	"unsafe"
+@@ -38,9 +39,11 @@ const (
+ var gpuMutex sync.Mutex
+ 
+ // With our current CUDA compile flags, older than 5.0 will not work properly
+-var CudaComputeMin = [2]C.int{5, 0}
++// (string values used to allow ldflags overrides at build time)
++var CudaComputeMajorMin = "5"
++var CudaComputeMinorMin = "0"
+ 
+-var RocmComputeMin = 9
++var RocmComputeMajorMin = "9"
+ 
+ // TODO find a better way to detect iGPU instead of minimum memory
+ const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU
+@@ -175,11 +178,57 @@ func GetGPUInfo() GpuInfoList {
+ 	var memInfo C.mem_info_t
+ 	resp := []GpuInfo{}
+ 
+-	// NVIDIA first
+-	for i := 0; i < gpuHandles.deviceCount; i++ {
+-		// TODO once we support CPU compilation variants of GPU libraries refine this...
+-		if cpuVariant == "" && runtime.GOARCH == "amd64" {
+-			continue
++	// Load ALL libraries
++	cHandles = initCudaHandles()
++	minMajorVer, err := strconv.Atoi(CudaComputeMajorMin)
++	if err != nil {
++		slog.Error("invalid CudaComputeMajorMin setting", "value", CudaComputeMajorMin, "error", err)
++	}
++	minMinorVer, err := strconv.Atoi(CudaComputeMinorMin)
++	if err != nil {
++		slog.Error("invalid CudaComputeMinorMin setting", "value", CudaComputeMinorMin, "error", err)
++	}
++
++	// NVIDIA
++	for i := range cHandles.deviceCount {
++		if cHandles.cudart != nil || cHandles.nvcuda != nil {
++			gpuInfo := CudaGPUInfo{
++				GpuInfo: GpuInfo{
++					Library: "cuda",
++				},
++				index: i,
++			}
++			var driverMajor int
++			var driverMinor int
++			if cHandles.cudart != nil {
++				C.cudart_bootstrap(*cHandles.cudart, C.int(i), &memInfo)
++			} else {
++				C.nvcuda_bootstrap(*cHandles.nvcuda, C.int(i), &memInfo)
++				driverMajor = int(cHandles.nvcuda.driver_major)
++				driverMinor = int(cHandles.nvcuda.driver_minor)
++			}
++			if memInfo.err != nil {
++				slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
++				C.free(unsafe.Pointer(memInfo.err))
++				continue
++			}
++
++			if int(memInfo.major) < minMajorVer || (int(memInfo.major) == minMajorVer && int(memInfo.minor) < minMinorVer) {
++				slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor))
++				continue
++			}
++			gpuInfo.TotalMemory = uint64(memInfo.total)
++			gpuInfo.FreeMemory = uint64(memInfo.free)
++			gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
++			gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor)
++			gpuInfo.MinimumMemory = cudaMinimumMemory
++			gpuInfo.DependencyPath = depPath
++			gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
++			gpuInfo.DriverMajor = driverMajor
++			gpuInfo.DriverMinor = driverMinor
++
++			// TODO potentially sort on our own algorithm instead of what the underlying GPU library does...
++			cudaGPUs = append(cudaGPUs, gpuInfo)
+ 		}
+ 		gpuInfo := GpuInfo{
+ 			Library: "cuda",
+diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh
+index e91c56ed..c03bc25f 100755
+--- a/scripts/build_docker.sh
++++ b/scripts/build_docker.sh
+@@ -3,7 +3,7 @@
+ set -eu
+ 
+ export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")}
+-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
++export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"}
+ 
+ # We use 2 different image repositories to handle combining architecture images into multiarch manifest
+ # (The ROCm image is x86 only and is not a multiarch manifest)
+diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh
+index 27c4ff1f..e7e6d0dd 100755
+--- a/scripts/build_linux.sh
++++ b/scripts/build_linux.sh
+@@ -3,7 +3,7 @@
+ set -eu
+ 
+ export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")}
+-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
++export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"}
+ 
+ BUILD_ARCH=${BUILD_ARCH:-"amd64 arm64"}
+ export AMDGPU_TARGETS=${AMDGPU_TARGETS:=""}
+-- 
+2.46.0
+

machines/krz01/ollama/all-nvcc-arch.patch

@@ -0,0 +1,26 @@
+From 2278389ef9ac9231349440aa68f9544ddc69cdc7 Mon Sep 17 00:00:00 2001
+From: Raito Bezarius <masterancpp@gmail.com>
+Date: Wed, 9 Oct 2024 13:37:08 +0200
+Subject: [PATCH] fix: sm_37 for nvcc
+
+Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
+---
+ Makefile | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Makefile b/Makefile
+index 2ccb750..70dfd9b 100644
+--- a/Makefile
++++ b/Makefile
+@@ -537,7 +537,7 @@ endif #GGML_CUDA_NVCC
+ ifdef CUDA_DOCKER_ARCH
+ 	MK_NVCCFLAGS += -Wno-deprecated-gpu-targets -arch=$(CUDA_DOCKER_ARCH)
+ else ifndef CUDA_POWER_ARCH
+-	MK_NVCCFLAGS += -arch=native
++	MK_NVCCFLAGS += -arch=sm_37
+ endif # CUDA_DOCKER_ARCH
+ 
+ ifdef GGML_CUDA_FORCE_DMMV
+-- 
+2.46.0
+

machines/krz01/ollama/default.nix

@@ -0,0 +1,20 @@
+{
+  config,
+  pkgs,
+  meta,
+  name,
+  ...
+}:
+{
+  services = {
+    ollama = {
+      enable = true;
+      host = meta.network.${name}.netbirdIp;
+      package = pkgs.callPackage ./package.nix {
+        cudaPackages = pkgs.cudaPackages_11;
+        # We need to thread our nvidia x11 driver for CUDA.
+        extraLibraries = [ config.hardware.nvidia.package ];
+      };
+    };
+  };
+}

machines/krz01/ollama/disable-git.patch

@@ -0,0 +1,20 @@
+diff --git c/llm/generate/gen_common.sh i/llm/generate/gen_common.sh
+index 3825c155..238a74a7 100644
+--- c/llm/generate/gen_common.sh
++++ i/llm/generate/gen_common.sh
+@@ -69,6 +69,7 @@ git_module_setup() {
+ }
+ 
+ apply_patches() {
++    return
+     # apply temporary patches until fix is upstream
+     for patch in ../patches/*.patch; do
+         git -c 'user.name=nobody' -c 'user.email=<>' -C ${LLAMACPP_DIR} am ${patch}
+@@ -133,6 +134,7 @@ install() {
+ 
+ # Keep the local tree clean after we're done with the build
+ cleanup() {
++    return
+     (cd ${LLAMACPP_DIR}/ && git checkout CMakeLists.txt)
+ 
+     if [ -n "$(ls -A ../patches/*.diff)" ]; then

machines/krz01/ollama/no-weird-microarch.patch

@@ -0,0 +1,34 @@
+From 51568b61ef63ecd97867562571411082c32751d3 Mon Sep 17 00:00:00 2001
+From: Raito Bezarius <masterancpp@gmail.com>
+Date: Wed, 9 Oct 2024 13:36:51 +0200
+Subject: [PATCH] fix: avx & f16c in Makefile
+
+Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
+---
+ Makefile | 8 ++++----
+ 1 file changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index 32b7cbb..2ccb750 100644
+--- a/Makefile
++++ b/Makefile
+@@ -361,12 +361,12 @@ ifndef RISCV
+ 
+ ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64))
+ 	# Use all CPU extensions that are available:
+-	MK_CFLAGS     += -march=native -mtune=native
+-	HOST_CXXFLAGS += -march=native -mtune=native
++	# MK_CFLAGS     += -march=native -mtune=native
++	# HOST_CXXFLAGS += -march=native -mtune=native
+ 
+ 	# Usage AVX-only
+-	#MK_CFLAGS   += -mfma -mf16c -mavx
+-	#MK_CXXFLAGS += -mfma -mf16c -mavx
++	MK_CFLAGS   += -mf16c -mavx
++	MK_CXXFLAGS += -mf16c -mavx
+ 
+ 	# Usage SSSE3-only (Not is SSE3!)
+ 	#MK_CFLAGS   += -mssse3
+-- 
+2.46.0
+

machines/krz01/ollama/package.nix

@@ -0,0 +1,243 @@
+{
+  lib,
+  buildGoModule,
+  fetchFromGitHub,
+  buildEnv,
+  linkFarm,
+  overrideCC,
+  makeWrapper,
+  stdenv,
+  addDriverRunpath,
+  nix-update-script,
+
+  cmake,
+  gcc11,
+  clblast,
+  libdrm,
+  rocmPackages,
+  cudaPackages,
+  darwin,
+  autoAddDriverRunpath,
+  extraLibraries ? [ ],
+
+  nixosTests,
+  testers,
+  ollama,
+  ollama-rocm,
+  ollama-cuda,
+
+  config,
+  # one of `[ null false "rocm" "cuda" ]`
+  acceleration ? null,
+}:
+
+assert builtins.elem acceleration [
+  null
+  false
+  "rocm"
+  "cuda"
+];
+
+let
+  pname = "ollama";
+  version = "2024-09-10-cc35";
+
+  src = fetchFromGitHub {
+    owner = "aliotard";
+    repo = "ollama";
+    rev = "34827c01f7723c7f5f9f5e392fe85f5a4a5d5fc0";
+    hash = "sha256-xFNuqcW7YWeyCyw5QLBnCHHTSMITR6LJkJT0CXZC+Y8=";
+    fetchSubmodules = true;
+  };
+
+  vendorHash = "sha256-hSxcREAujhvzHVNwnRTfhi0MKI3s8HNavER2VLz6SYk=";
+
+  validateFallback = lib.warnIf (config.rocmSupport && config.cudaSupport) (lib.concatStrings [
+    "both `nixpkgs.config.rocmSupport` and `nixpkgs.config.cudaSupport` are enabled, "
+    "but they are mutually exclusive; falling back to cpu"
+  ]) (!(config.rocmSupport && config.cudaSupport));
+  shouldEnable =
+    mode: fallback: (acceleration == mode) || (fallback && acceleration == null && validateFallback);
+
+  rocmRequested = shouldEnable "rocm" config.rocmSupport;
+  cudaRequested = shouldEnable "cuda" config.cudaSupport;
+
+  enableRocm = rocmRequested && stdenv.isLinux;
+  enableCuda = cudaRequested && stdenv.isLinux;
+
+  rocmLibs = [
+    rocmPackages.clr
+    rocmPackages.hipblas
+    rocmPackages.rocblas
+    rocmPackages.rocsolver
+    rocmPackages.rocsparse
+    rocmPackages.rocm-device-libs
+    rocmPackages.rocm-smi
+  ];
+  rocmClang = linkFarm "rocm-clang" { llvm = rocmPackages.llvm.clang; };
+  rocmPath = buildEnv {
+    name = "rocm-path";
+    paths = rocmLibs ++ [ rocmClang ];
+  };
+
+  cudaLibs = [
+    cudaPackages.cuda_cudart
+    cudaPackages.libcublas
+    cudaPackages.cuda_cccl
+  ];
+  cudaToolkit = buildEnv {
+    name = "cuda-merged";
+    paths = map lib.getLib cudaLibs ++ [
+      (lib.getOutput "static" cudaPackages.cuda_cudart)
+      (lib.getBin (cudaPackages.cuda_nvcc.__spliced.buildHost or cudaPackages.cuda_nvcc))
+    ];
+  };
+
+  metalFrameworks = with darwin.apple_sdk_11_0.frameworks; [
+    Accelerate
+    Metal
+    MetalKit
+    MetalPerformanceShaders
+  ];
+
+  wrapperOptions =
+    [
+      # ollama embeds llama-cpp binaries which actually run the ai models
+      # these llama-cpp binaries are unaffected by the ollama binary's DT_RUNPATH
+      # LD_LIBRARY_PATH is temporarily required to use the gpu
+      # until these llama-cpp binaries can have their runpath patched
+      "--suffix LD_LIBRARY_PATH : '${addDriverRunpath.driverLink}/lib'"
+      "--suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath (map lib.getLib extraLibraries)}'"
+    ]
+    ++ lib.optionals enableRocm [
+      "--suffix LD_LIBRARY_PATH : '${rocmPath}/lib'"
+      "--set-default HIP_PATH '${rocmPath}'"
+    ]
+    ++ lib.optionals enableCuda [
+      "--suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath (map lib.getLib cudaLibs)}'"
+    ];
+  wrapperArgs = builtins.concatStringsSep " " wrapperOptions;
+
+  goBuild =
+    if enableCuda then buildGoModule.override { stdenv = overrideCC stdenv gcc11; } else buildGoModule;
+  inherit (lib) licenses platforms maintainers;
+in
+goBuild {
+  inherit
+    pname
+    version
+    src
+    vendorHash
+    ;
+
+  env =
+    lib.optionalAttrs enableRocm {
+      ROCM_PATH = rocmPath;
+      CLBlast_DIR = "${clblast}/lib/cmake/CLBlast";
+    }
+    // lib.optionalAttrs enableCuda { CUDA_LIB_DIR = "${cudaToolkit}/lib"; }
+    // {
+      CMAKE_CUDA_ARCHITECTURES = "35;37";
+    };
+
+  nativeBuildInputs =
+    [ cmake ]
+    ++ lib.optionals enableRocm [ rocmPackages.llvm.bintools ]
+    ++ lib.optionals enableCuda [ cudaPackages.cuda_nvcc ]
+    ++ lib.optionals (enableRocm || enableCuda) [
+      makeWrapper
+      autoAddDriverRunpath
+    ]
+    ++ lib.optionals stdenv.isDarwin metalFrameworks;
+
+  buildInputs =
+    lib.optionals enableRocm (rocmLibs ++ [ libdrm ])
+    ++ lib.optionals enableCuda cudaLibs
+    ++ lib.optionals stdenv.isDarwin metalFrameworks;
+
+  patches = [
+    # disable uses of `git` in the `go generate` script
+    # ollama's build script assumes the source is a git repo, but nix removes the git directory
+    # this also disables necessary patches contained in `ollama/llm/patches/`
+    # those patches are applied in `postPatch`
+    ./disable-git.patch
+  ];
+
+  postPatch = ''
+    # replace inaccurate version number with actual release version
+    substituteInPlace version/version.go --replace-fail 0.0.0 '${version}'
+
+    # apply ollama's patches to `llama.cpp` submodule
+    for diff in llm/patches/*; do
+      patch -p1 -d llm/llama.cpp < $diff
+    done
+  '';
+
+  overrideModAttrs = _: _: {
+    # don't run llama.cpp build in the module fetch phase
+    preBuild = "";
+  };
+
+  preBuild = ''
+    # disable uses of `git`, since nix removes the git directory
+    export OLLAMA_SKIP_PATCHING=true
+    # build llama.cpp libraries for ollama
+    go generate ./...
+  '';
+
+  postFixup =
+    ''
+      # the app doesn't appear functional at the moment, so hide it
+      mv "$out/bin/app" "$out/bin/.ollama-app"
+    ''
+    + lib.optionalString (enableRocm || enableCuda) ''
+      # expose runtime libraries necessary to use the gpu
+      wrapProgram "$out/bin/ollama" ${wrapperArgs}
+    '';
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X=github.com/ollama/ollama/version.Version=${version}"
+    "-X=github.com/ollama/ollama/server.mode=release"
+    "-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3"
+    "-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5"
+  ];
+
+  passthru = {
+    tests =
+      {
+        inherit ollama;
+        version = testers.testVersion {
+          inherit version;
+          package = ollama;
+        };
+      }
+      // lib.optionalAttrs stdenv.isLinux {
+        inherit ollama-rocm ollama-cuda;
+        service = nixosTests.ollama;
+        service-cuda = nixosTests.ollama-cuda;
+        service-rocm = nixosTests.ollama-rocm;
+      };
+
+    updateScript = nix-update-script { };
+  };
+
+  meta = {
+    description =
+      "Get up and running with large language models locally"
+      + lib.optionalString rocmRequested ", using ROCm for AMD GPU acceleration"
+      + lib.optionalString cudaRequested ", using CUDA for NVIDIA GPU acceleration";
+    homepage = "https://github.com/ollama/ollama";
+    changelog = "https://github.com/ollama/ollama/releases/tag/v${version}";
+    license = licenses.mit;
+    platforms = if (rocmRequested || cudaRequested) then platforms.linux else platforms.unix;
+    mainProgram = "ollama";
+    maintainers = with maintainers; [
+      abysssol
+      dit7ya
+      elohmeier
+      roydubnium
+    ];
+  };
+}

machines/krz01/whisper/all-nvcc-arch.patch

@@ -0,0 +1,26 @@
+From 2278389ef9ac9231349440aa68f9544ddc69cdc7 Mon Sep 17 00:00:00 2001
+From: Raito Bezarius <masterancpp@gmail.com>
+Date: Wed, 9 Oct 2024 13:37:08 +0200
+Subject: [PATCH] fix: sm_37 for nvcc
+
+Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
+---
+ Makefile | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Makefile b/Makefile
+index 2ccb750..70dfd9b 100644
+--- a/Makefile
++++ b/Makefile
+@@ -537,7 +537,7 @@ endif #GGML_CUDA_NVCC
+ ifdef CUDA_DOCKER_ARCH
+ 	MK_NVCCFLAGS += -Wno-deprecated-gpu-targets -arch=$(CUDA_DOCKER_ARCH)
+ else ifndef CUDA_POWER_ARCH
+-	MK_NVCCFLAGS += -arch=native
++	MK_NVCCFLAGS += -arch=sm_37
+ endif # CUDA_DOCKER_ARCH
+ 
+ ifdef GGML_CUDA_FORCE_DMMV
+-- 
+2.46.0
+

machines/krz01/whisper/default.nix

@@ -0,0 +1,25 @@
+{ pkgs, ... }:
+{
+  environment.systemPackages = [
+    ((pkgs.openai-whisper-cpp.override { cudaPackages = pkgs.cudaPackages_11; }).overrideAttrs (old: {
+      src = pkgs.fetchFromGitHub {
+        owner = "ggerganov";
+        repo = "whisper.cpp";
+        rev = "v1.7.1";
+        hash = "sha256-EDFUVjud79ZRCzGbOh9L9NcXfN3ikvsqkVSOME9F9oo=";
+      };
+      env = {
+        WHISPER_CUBLAS = "";
+        GGML_CUDA = "1";
+      };
+      # We only need Compute Capability 3.7.
+      CUDA_ARCH_FLAGS = [ "sm_37" ];
+      # We are GPU-only anyway.
+      patches = (old.patches or [ ]) ++ [
+        ./no-weird-microarch.patch
+        ./all-nvcc-arch.patch
+      ];
+    }))
+  ];
+
+}

machines/krz01/whisper/no-weird-microarch.patch

@@ -0,0 +1,34 @@
+From 51568b61ef63ecd97867562571411082c32751d3 Mon Sep 17 00:00:00 2001
+From: Raito Bezarius <masterancpp@gmail.com>
+Date: Wed, 9 Oct 2024 13:36:51 +0200
+Subject: [PATCH] fix: avx & f16c in Makefile
+
+Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
+---
+ Makefile | 8 ++++----
+ 1 file changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index 32b7cbb..2ccb750 100644
+--- a/Makefile
++++ b/Makefile
+@@ -361,12 +361,12 @@ ifndef RISCV
+ 
+ ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64))
+ 	# Use all CPU extensions that are available:
+-	MK_CFLAGS     += -march=native -mtune=native
+-	HOST_CXXFLAGS += -march=native -mtune=native
++	# MK_CFLAGS     += -march=native -mtune=native
++	# HOST_CXXFLAGS += -march=native -mtune=native
+ 
+ 	# Usage AVX-only
+-	#MK_CFLAGS   += -mfma -mf16c -mavx
+-	#MK_CXXFLAGS += -mfma -mf16c -mavx
++	MK_CFLAGS   += -mf16c -mavx
++	MK_CXXFLAGS += -mf16c -mavx
+ 
+ 	# Usage SSSE3-only (Not is SSE3!)
+ 	#MK_CFLAGS   += -mssse3
+-- 
+2.46.0
+

machines/router02/_configuration.nix

@@ -0,0 +1,16 @@
+{ lib, ... }:
+
+lib.extra.mkConfig {
+  enabledModules = [
+    # List of modules to enable
+  ];
+
+  enabledServices = [
+    # List of services to enable
+    "networking"
+  ];
+
+  extraConfig = { };
+
+  root = ./.;
+}

machines/router02/_hardware-configuration.nix

@@ -0,0 +1,49 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ lib, modulesPath, ... }:
+
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  boot = {
+    initrd = {
+      availableKernelModules = [
+        "ata_piix"
+        "uhci_hcd"
+        "virtio_pci"
+        "virtio_scsi"
+        "sd_mod"
+        "sr_mod"
+      ];
+      kernelModules = [ ];
+    };
+    kernelModules = [ ];
+    extraModulePackages = [ ];
+  };
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-uuid/77ecdf8d-ecda-42ed-aa4f-caa78d5d5e06";
+    fsType = "ext4";
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-uuid/A044-5792";
+    fsType = "vfat";
+    options = [
+      "fmask=0022"
+      "dmask=0022"
+    ];
+  };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

machines/router02/networking.nix

@@ -0,0 +1,18 @@
+{
+  systemd.network = {
+    networks = {
+      "10-ens18" = {
+        name = "ens18";
+
+        networkConfig = {
+          Description = "ENS uplink";
+          Address = [ "129.199.146.34/24" ];
+          Gateway = "129.199.146.254";
+          LLDP = true;
+          # Only to the switch we are connected to directly, e.g. the hypervisor or the switch.
+          EmitLLDP = "nearest-bridge";
+        };
+      };
+    };
+  };
+}

machines/router02/secrets/secrets.nix

@@ -0,0 +1,3 @@
+(import ../../../keys).mkSecrets [ "router02" ] [
+  # List of secrets for router02
+]

meta/network.nix

@@ -1,7 +1,7 @@
 {
   krz01 = {
     interfaces = {
-      eno1 = {
+      vmbr0 = {
         ipv4 = [
           {
             address = "129.199.146.21";
@@ -21,4 +21,11 @@
     hostId = "bd11e8fc";
     netbirdIp = "100.80.103.206";
   };
+  router02 = {
+    interfaces = { };
+
+    addresses.ipv4 = [ "129.199.46.34" ];
+
+    hostId = "144d0f7a";
+  };
 }

meta/nodes.nix

@@ -28,4 +28,12 @@
     stateVersion = "24.05";
     nixpkgs = "unstable";
   };
+  router02 = {
+    site = "pav01";
+
+    hashedPassword = "$y$j9T$aFhOWa05W7VKeKt3Nc.nA1$uBOvG4wf7/yWjwOxO8NLf9ipCsAkS1.5cD2EJpLx57A";
+
+    stateVersion = "24.05";
+    nixpkgs = "24.05";
+  };
 }

modules/default.nix

@@ -0,0 +1,14 @@
+{ lib, sources, ... }:
+{
+  imports = (lib.extra.mkImports ./. [ "lab-acme" ]) ++ [
+    "${sources."microvm.nix"}/nixos-modules/host"
+  ];
+
+  dgn-notify.enable = false;
+
+  dgn-records.enable = false;
+
+  # TODO think about how to use netbox with lab
+  dgn-netbox-agent.enable = false;
+  dgn-acme.enable = false;
+}

modules/lab-acme.nix

@@ -0,0 +1,53 @@
+#  Copyright :
+#  - Tom Hubrecht <tom.hubrecht@dgnum.eu>      2023
+#
+#  Ce logiciel est un programme informatique servant à déployer des
+#  configurations de serveurs via NixOS.
+#
+#  Ce logiciel est régi par la licence CeCILL soumise au droit français et
+#  respectant les principes de diffusion des logiciels libres. Vous pouvez
+#  utiliser, modifier et/ou redistribuer ce programme sous les conditions
+#  de la licence CeCILL telle que diffusée par le CEA, le CNRS et l'INRIA
+#  sur le site "http://www.cecill.info".
+#
+#  En contrepartie de l'accessibilité au code source et des droits de copie,
+#  de modification et de redistribution accordés par cette licence, il n'est
+#  offert aux utilisateurs qu'une garantie limitée. Pour les mêmes raisons,
+#  seule une responsabilité restreinte pèse sur l'auteur du programme, le
+#  titulaire des droits patrimoniaux et les concédants successifs.
+#
+#  A cet égard l'attention de l'utilisateur est attirée sur les risques
+#  associés au chargement, à l'utilisation, à la modification et/ou au
+#  développement et à la reproduction du logiciel par l'utilisateur étant
+#  donné sa spécificité de logiciel libre, qui peut le rendre complexe à
+#  manipuler et qui le réserve donc à des développeurs et des professionnels
+#  avertis possédant des connaissances informatiques approfondies. Les
+#  utilisateurs sont donc invités à charger et tester l'adéquation du
+#  logiciel à leurs besoins dans des conditions permettant d'assurer la
+#  sécurité de leurs systèmes et ou de leurs données et, plus généralement,
+#  à l'utiliser et l'exploiter dans les mêmes conditions de sécurité.
+#
+#  Le fait que vous puissiez accéder à cet en-tête signifie que vous avez
+#  pris connaissance de la licence CeCILL, et que vous en avez accepté les
+#  termes.
+
+{ config, lib, ... }:
+
+let
+  inherit (lib) mkEnableOption mkIf;
+
+  cfg = config.lab-acme;
+in
+
+{
+  options.lab-acme.enable = mkEnableOption "ACME settings." // {
+    default = true;
+  };
+
+  config = mkIf cfg.enable {
+    security.acme = {
+      acceptTerms = true;
+      defaults.email = "acme+lab@dgnum.eu";
+    };
+  };
+}

npins/sources.json

@@ -49,9 +49,9 @@
         "url": "https://git.dgnum.eu/DGNum/infrastructure"
       },
       "branch": "main",
-      "revision": "bb4a24f9e997445383ffb979e47b6d4ea2553119",
+      "revision": "c0435e694de19f8adad46fc5d0790d73406da3c2",
       "url": null,
-      "hash": "1c0b6b5k3qb0hy42nic9nwbz8dz0n9nc8245yacq37x5x46jpbsw"
+      "hash": "1f92fbcmi96sdgdwmi5y7wynl09fkcyijbc4bkvd7bjj88z4kgwn"
     },
     "lix": {
       "type": "Git",