machines/krz01/K80-support.patch

@@ -0,0 +1,179 @@
+From 2abd226ff3093c5a9e18a618fba466853e7ebaf7 Mon Sep 17 00:00:00 2001
+From: Raito Bezarius <masterancpp@gmail.com>
+Date: Tue, 8 Oct 2024 18:27:41 +0200
+Subject: [PATCH] K80 support
+
+Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
+---
+ docs/development.md     |  6 +++-
+ docs/gpu.md             |  1 +
+ gpu/amd_linux.go        |  6 +++-
+ gpu/gpu.go              | 63 ++++++++++++++++++++++++++++++++++++-----
+ scripts/build_docker.sh |  2 +-
+ scripts/build_linux.sh  |  2 +-
+ 6 files changed, 69 insertions(+), 11 deletions(-)
+
+diff --git a/docs/development.md b/docs/development.md
+index 2f7b9ecf..9da35931 100644
+--- a/docs/development.md
++++ b/docs/development.md
+@@ -51,7 +51,11 @@ Typically the build scripts will auto-detect CUDA, however, if your Linux distro
+ or installation approach uses unusual paths, you can specify the location by
+ specifying an environment variable `CUDA_LIB_DIR` to the location of the shared
+ libraries, and `CUDACXX` to the location of the nvcc compiler. You can customize
+-a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "50;60;70")
++a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "35;37;50;60;70")
++
++To support GPUs older than Compute Capability 5.0, you will need to use an older version of
++the Driver from [Unix Driver Archive](https://www.nvidia.com/en-us/drivers/unix/) (tested with 470) and [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) (tested with cuda V11).  When you build Ollama, you will need to set two environment variable to adjust the minimum compute capability Ollama supports via `export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3\" \"-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5\"'"` and the `CMAKE_CUDA_ARCHITECTURES`.  To find the Compute Capability of your older GPU, refer to [GPU Compute Capability](https://developer.nvidia.com/cuda-gpus).
++
+ 
+ Then generate dependencies:
+ 
+diff --git a/docs/gpu.md b/docs/gpu.md
+index a6b559f0..66627611 100644
+--- a/docs/gpu.md
++++ b/docs/gpu.md
+@@ -28,6 +28,7 @@ Check your compute compatibility to see if your card is supported:
+ | 5.0                | GeForce GTX         | `GTX 750 Ti` `GTX 750` `NVS 810`                                                                            |
+ |                    | Quadro              | `K2200` `K1200` `K620` `M1200` `M520` `M5000M` `M4000M` `M3000M` `M2000M` `M1000M` `K620M` `M600M` `M500M`  |
+ 
++For building locally to support older GPUs, see [developer.md](./development.md#linux-cuda-nvidia)
+ 
+ ### GPU Selection
+ 
+diff --git a/gpu/amd_linux.go b/gpu/amd_linux.go
+index 6b08ac2e..768fb97a 100644
+--- a/gpu/amd_linux.go
++++ b/gpu/amd_linux.go
+@@ -159,7 +159,11 @@ func AMDGetGPUInfo() []GpuInfo {
+ 			return []GpuInfo{}
+ 		}
+ 
+-		if int(major) < RocmComputeMin {
++		minVer, err := strconv.Atoi(RocmComputeMajorMin)
++		if err != nil {
++			slog.Error("invalid RocmComputeMajorMin setting", "value", RocmComputeMajorMin, "error", err)
++		}
++		if int(major) < minVer {
+ 			slog.Warn(fmt.Sprintf("amdgpu too old gfx%d%x%x", major, minor, patch), "gpu", gpuID)
+ 			continue
+ 		}
+diff --git a/gpu/gpu.go b/gpu/gpu.go
+index 781e23df..60d68c33 100644
+--- a/gpu/gpu.go
++++ b/gpu/gpu.go
+@@ -16,6 +16,7 @@ import (
+ 	"os"
+ 	"path/filepath"
+ 	"runtime"
++	"strconv"
+ 	"strings"
+ 	"sync"
+ 	"unsafe"
+@@ -38,9 +39,11 @@ const (
+ var gpuMutex sync.Mutex
+ 
+ // With our current CUDA compile flags, older than 5.0 will not work properly
+-var CudaComputeMin = [2]C.int{5, 0}
++// (string values used to allow ldflags overrides at build time)
++var CudaComputeMajorMin = "5"
++var CudaComputeMinorMin = "0"
+ 
+-var RocmComputeMin = 9
++var RocmComputeMajorMin = "9"
+ 
+ // TODO find a better way to detect iGPU instead of minimum memory
+ const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU
+@@ -175,11 +178,57 @@ func GetGPUInfo() GpuInfoList {
+ 	var memInfo C.mem_info_t
+ 	resp := []GpuInfo{}
+ 
+-	// NVIDIA first
+-	for i := 0; i < gpuHandles.deviceCount; i++ {
+-		// TODO once we support CPU compilation variants of GPU libraries refine this...
+-		if cpuVariant == "" && runtime.GOARCH == "amd64" {
+-			continue
++	// Load ALL libraries
++	cHandles = initCudaHandles()
++	minMajorVer, err := strconv.Atoi(CudaComputeMajorMin)
++	if err != nil {
++		slog.Error("invalid CudaComputeMajorMin setting", "value", CudaComputeMajorMin, "error", err)
++	}
++	minMinorVer, err := strconv.Atoi(CudaComputeMinorMin)
++	if err != nil {
++		slog.Error("invalid CudaComputeMinorMin setting", "value", CudaComputeMinorMin, "error", err)
++	}
++
++	// NVIDIA
++	for i := range cHandles.deviceCount {
++		if cHandles.cudart != nil || cHandles.nvcuda != nil {
++			gpuInfo := CudaGPUInfo{
++				GpuInfo: GpuInfo{
++					Library: "cuda",
++				},
++				index: i,
++			}
++			var driverMajor int
++			var driverMinor int
++			if cHandles.cudart != nil {
++				C.cudart_bootstrap(*cHandles.cudart, C.int(i), &memInfo)
++			} else {
++				C.nvcuda_bootstrap(*cHandles.nvcuda, C.int(i), &memInfo)
++				driverMajor = int(cHandles.nvcuda.driver_major)
++				driverMinor = int(cHandles.nvcuda.driver_minor)
++			}
++			if memInfo.err != nil {
++				slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
++				C.free(unsafe.Pointer(memInfo.err))
++				continue
++			}
++
++			if int(memInfo.major) < minMajorVer || (int(memInfo.major) == minMajorVer && int(memInfo.minor) < minMinorVer) {
++				slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor))
++				continue
++			}
++			gpuInfo.TotalMemory = uint64(memInfo.total)
++			gpuInfo.FreeMemory = uint64(memInfo.free)
++			gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
++			gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor)
++			gpuInfo.MinimumMemory = cudaMinimumMemory
++			gpuInfo.DependencyPath = depPath
++			gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
++			gpuInfo.DriverMajor = driverMajor
++			gpuInfo.DriverMinor = driverMinor
++
++			// TODO potentially sort on our own algorithm instead of what the underlying GPU library does...
++			cudaGPUs = append(cudaGPUs, gpuInfo)
+ 		}
+ 		gpuInfo := GpuInfo{
+ 			Library: "cuda",
+diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh
+index e91c56ed..c03bc25f 100755
+--- a/scripts/build_docker.sh
++++ b/scripts/build_docker.sh
+@@ -3,7 +3,7 @@
+ set -eu
+ 
+ export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")}
+-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
++export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"}
+ 
+ # We use 2 different image repositories to handle combining architecture images into multiarch manifest
+ # (The ROCm image is x86 only and is not a multiarch manifest)
+diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh
+index 27c4ff1f..e7e6d0dd 100755
+--- a/scripts/build_linux.sh
++++ b/scripts/build_linux.sh
+@@ -3,7 +3,7 @@
+ set -eu
+ 
+ export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")}
+-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
++export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"}
+ 
+ BUILD_ARCH=${BUILD_ARCH:-"amd64 arm64"}
+ export AMDGPU_TARGETS=${AMDGPU_TARGETS:=""}
+-- 
+2.46.0
+

machines/krz01/_configuration.nix

@@ -1,4 +1,4 @@
-{ lib, ... }:
+{ lib, pkgs, ... }:
 
 lib.extra.mkConfig {
   enabledModules = [
@@ -30,7 +30,23 @@ lib.extra.mkConfig {
 
     # We are going to use CUDA here.
     nixpkgs.config.cudaSupport = true;
-
+    services.ollama = {
+      enable = true;
+      package =
+        (pkgs.ollama.override {
+          cudaPackages = pkgs.cudaPackages_11;
+          gcc12 = pkgs.gcc11;
+        }).overrideAttrs
+          (old: {
+            CMAKE_CUDA_ARCHITECTURES = "35";
+            ldflags = old.ldflags ++ [
+              # K80 is 3.5
+              "-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3"
+              "-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5"
+            ];
+            patches = (old.patches or [ ]) ++ [ ./K80-support.patch ];
+          });
+    };
     users.users.root.hashedPassword = "$y$j9T$eNZQgDN.J5y7KTG2hXgat1$J1i5tjx5dnSZu.C9B7swXi5zMFIkUnmRrnmyLHFAt8/";
   };
 

machines/krz01/microvm-ml01.nix

@@ -1,39 +1,13 @@
 _: {
   microvm.autostart = [ "ml01" ];
   microvm.vms.ml01 = {
-    config =
-      { config, ... }:
-      {
-        nixpkgs.config.cudaSupport = true;
-        nixpkgs.config.nvidia.acceptLicense = true;
-        # Tesla K80 is not supported by the latest driver.
-        hardware.nvidia.package = config.boot.kernelPackages.nvidia_x11_legacy470;
-        # Don't ask.
-        services.xserver.videoDrivers = [ "nvidia" ];
+    config = {
       networking.hostName = "ml01";
-        services.ollama = {
-          enable = true;
-          listenAddress = "0.0.0.0:11434";
-          sandbox = true;
-          acceleration = "cuda";
-        };
-
       microvm = {
         hypervisor = "cloud-hypervisor";
         vcpu = 4;
         mem = 4096;
         balloonMem = 2048;
-          devices = [
-            # The nVidia Tesla K80
-            {
-              bus = "pci";
-              path = "0000:44:00.0";
-            }
-            {
-              bus = "pci";
-              path = "0000:45:00.0";
-            }
-          ];
         shares = [
           {
             source = "/nix/store";