machines/krz01/_configuration.nix

@@ -1,11 +1,4 @@
-{
-  config,
-  lib,
-  pkgs,
-  meta,
-  name,
-  ...
-}:
+{ config, lib, ... }:
 
 lib.extra.mkConfig {
   enabledModules = [
@@ -18,6 +11,8 @@ lib.extra.mkConfig {
     "microvm-ml01"
     "microvm-router01"
     "nvidia-tesla-k80"
+    "ollama"
+    "whisper"
     "proxmox"
   ];
 
@@ -33,45 +28,11 @@ lib.extra.mkConfig {
       ];
     };
 
-    services.netbird.enable = true;
-
     # We are going to use CUDA here.
     nixpkgs.config.cudaSupport = true;
     hardware.graphics.enable = true;
-    environment.systemPackages = [
-      ((pkgs.openai-whisper-cpp.override { cudaPackages = pkgs.cudaPackages_11; }).overrideAttrs (old: {
-        src = pkgs.fetchFromGitHub {
-          owner = "ggerganov";
-          repo = "whisper.cpp";
-          rev = "v1.7.1";
-          hash = "sha256-EDFUVjud79ZRCzGbOh9L9NcXfN3ikvsqkVSOME9F9oo=";
-        };
-        env = {
-          WHISPER_CUBLAS = "";
-          GGML_CUDA = "1";
-        };
-        # We only need Compute Capability 3.7.
-        CUDA_ARCH_FLAGS = [ "sm_37" ];
-        # We are GPU-only anyway.
-        patches = (old.patches or [ ]) ++ [
-          ./no-weird-microarch.patch
-          ./all-nvcc-arch.patch
-        ];
-      }))
-    ];
-
-    services = {
-      ollama = {
-        enable = true;
-        host = meta.network.${name}.netbirdIp;
-        package = pkgs.callPackage ./ollama.nix {
-          cudaPackages = pkgs.cudaPackages_11;
-          # We need to thread our nvidia x11 driver for CUDA.
-          extraLibraries = [ config.hardware.nvidia.package ];
-        };
-      };
-    };
 
+    services.netbird.enable = true;
     networking.firewall.trustedInterfaces = [ "wt0" ];
   };
 

machines/krz01/ollama/default.nix

@@ -0,0 +1,20 @@
+{
+  config,
+  pkgs,
+  meta,
+  name,
+  ...
+}:
+{
+  services = {
+    ollama = {
+      enable = true;
+      host = meta.network.${name}.netbirdIp;
+      package = pkgs.callPackage ./package.nix {
+        cudaPackages = pkgs.cudaPackages_11;
+        # We need to thread our nvidia x11 driver for CUDA.
+        extraLibraries = [ config.hardware.nvidia.package ];
+      };
+    };
+  };
+}

machines/krz01/whisper/all-nvcc-arch.patch

@@ -0,0 +1,26 @@
+From 2278389ef9ac9231349440aa68f9544ddc69cdc7 Mon Sep 17 00:00:00 2001
+From: Raito Bezarius <masterancpp@gmail.com>
+Date: Wed, 9 Oct 2024 13:37:08 +0200
+Subject: [PATCH] fix: sm_37 for nvcc
+
+Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
+---
+ Makefile | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/Makefile b/Makefile
+index 2ccb750..70dfd9b 100644
+--- a/Makefile
++++ b/Makefile
+@@ -537,7 +537,7 @@ endif #GGML_CUDA_NVCC
+ ifdef CUDA_DOCKER_ARCH
+ 	MK_NVCCFLAGS += -Wno-deprecated-gpu-targets -arch=$(CUDA_DOCKER_ARCH)
+ else ifndef CUDA_POWER_ARCH
+-	MK_NVCCFLAGS += -arch=native
++	MK_NVCCFLAGS += -arch=sm_37
+ endif # CUDA_DOCKER_ARCH
+ 
+ ifdef GGML_CUDA_FORCE_DMMV
+-- 
+2.46.0
+

machines/krz01/whisper/default.nix

@@ -0,0 +1,25 @@
+{ pkgs, ... }:
+{
+  environment.systemPackages = [
+    ((pkgs.openai-whisper-cpp.override { cudaPackages = pkgs.cudaPackages_11; }).overrideAttrs (old: {
+      src = pkgs.fetchFromGitHub {
+        owner = "ggerganov";
+        repo = "whisper.cpp";
+        rev = "v1.7.1";
+        hash = "sha256-EDFUVjud79ZRCzGbOh9L9NcXfN3ikvsqkVSOME9F9oo=";
+      };
+      env = {
+        WHISPER_CUBLAS = "";
+        GGML_CUDA = "1";
+      };
+      # We only need Compute Capability 3.7.
+      CUDA_ARCH_FLAGS = [ "sm_37" ];
+      # We are GPU-only anyway.
+      patches = (old.patches or [ ]) ++ [
+        ./no-weird-microarch.patch
+        ./all-nvcc-arch.patch
+      ];
+    }))
+  ];
+
+}

machines/krz01/whisper/no-weird-microarch.patch

@@ -0,0 +1,34 @@
+From 51568b61ef63ecd97867562571411082c32751d3 Mon Sep 17 00:00:00 2001
+From: Raito Bezarius <masterancpp@gmail.com>
+Date: Wed, 9 Oct 2024 13:36:51 +0200
+Subject: [PATCH] fix: avx & f16c in Makefile
+
+Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
+---
+ Makefile | 8 ++++----
+ 1 file changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index 32b7cbb..2ccb750 100644
+--- a/Makefile
++++ b/Makefile
+@@ -361,12 +361,12 @@ ifndef RISCV
+ 
+ ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64))
+ 	# Use all CPU extensions that are available:
+-	MK_CFLAGS     += -march=native -mtune=native
+-	HOST_CXXFLAGS += -march=native -mtune=native
++	# MK_CFLAGS     += -march=native -mtune=native
++	# HOST_CXXFLAGS += -march=native -mtune=native
+ 
+ 	# Usage AVX-only
+-	#MK_CFLAGS   += -mfma -mf16c -mavx
+-	#MK_CXXFLAGS += -mfma -mf16c -mavx
++	MK_CFLAGS   += -mf16c -mavx
++	MK_CXXFLAGS += -mf16c -mavx
+ 
+ 	# Usage SSSE3-only (Not is SSE3!)
+ 	#MK_CFLAGS   += -mssse3
+-- 
+2.46.0
+