infrastructure/machines/krz01/ollama/K80-support.patch

From 2abd226ff3093c5a9e18a618fba466853e7ebaf7 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Tue, 8 Oct 2024 18:27:41 +0200
Subject: [PATCH] K80 support

Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
 docs/development.md     |  6 +++-
 docs/gpu.md             |  1 +
 gpu/amd_linux.go        |  6 +++-
 gpu/gpu.go              | 63 ++++++++++++++++++++++++++++++++++++-----
 scripts/build_docker.sh |  2 +-
 scripts/build_linux.sh  |  2 +-
 6 files changed, 69 insertions(+), 11 deletions(-)

diff --git a/docs/development.md b/docs/development.md
index 2f7b9ecf..9da35931 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -51,7 +51,11 @@ Typically the build scripts will auto-detect CUDA, however, if your Linux distro
 or installation approach uses unusual paths, you can specify the location by
 specifying an environment variable `CUDA_LIB_DIR` to the location of the shared
 libraries, and `CUDACXX` to the location of the nvcc compiler. You can customize
-a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "50;60;70")
+a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "35;37;50;60;70")
+
+To support GPUs older than Compute Capability 5.0, you will need to use an older version of
+the Driver from [Unix Driver Archive](https://www.nvidia.com/en-us/drivers/unix/) (tested with 470) and [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) (tested with cuda V11).  When you build Ollama, you will need to set two environment variable to adjust the minimum compute capability Ollama supports via `export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3\" \"-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5\"'"` and the `CMAKE_CUDA_ARCHITECTURES`.  To find the Compute Capability of your older GPU, refer to [GPU Compute Capability](https://developer.nvidia.com/cuda-gpus).
+
 
 Then generate dependencies:
 
diff --git a/docs/gpu.md b/docs/gpu.md
index a6b559f0..66627611 100644
--- a/docs/gpu.md
+++ b/docs/gpu.md
@@ -28,6 +28,7 @@ Check your compute compatibility to see if your card is supported:
 | 5.0                | GeForce GTX         | `GTX 750 Ti` `GTX 750` `NVS 810`                                                                            |
 |                    | Quadro              | `K2200` `K1200` `K620` `M1200` `M520` `M5000M` `M4000M` `M3000M` `M2000M` `M1000M` `K620M` `M600M` `M500M`  |
 
+For building locally to support older GPUs, see [developer.md](./development.md#linux-cuda-nvidia)
 
 ### GPU Selection
 
diff --git a/gpu/amd_linux.go b/gpu/amd_linux.go
index 6b08ac2e..768fb97a 100644
--- a/gpu/amd_linux.go
+++ b/gpu/amd_linux.go
@@ -159,7 +159,11 @@ func AMDGetGPUInfo() []GpuInfo {
 			return []GpuInfo{}
 		}
 
-		if int(major) < RocmComputeMin {
+		minVer, err := strconv.Atoi(RocmComputeMajorMin)
+		if err != nil {
+			slog.Error("invalid RocmComputeMajorMin setting", "value", RocmComputeMajorMin, "error", err)
+		}
+		if int(major) < minVer {
 			slog.Warn(fmt.Sprintf("amdgpu too old gfx%d%x%x", major, minor, patch), "gpu", gpuID)
 			continue
 		}
diff --git a/gpu/gpu.go b/gpu/gpu.go
index 781e23df..60d68c33 100644
--- a/gpu/gpu.go
+++ b/gpu/gpu.go
@@ -16,6 +16,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"unsafe"
@@ -38,9 +39,11 @@ const (
 var gpuMutex sync.Mutex
 
 // With our current CUDA compile flags, older than 5.0 will not work properly
-var CudaComputeMin = [2]C.int{5, 0}
+// (string values used to allow ldflags overrides at build time)
+var CudaComputeMajorMin = "5"
+var CudaComputeMinorMin = "0"
 
-var RocmComputeMin = 9
+var RocmComputeMajorMin = "9"
 
 // TODO find a better way to detect iGPU instead of minimum memory
 const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU
@@ -175,11 +178,57 @@ func GetGPUInfo() GpuInfoList {
 	var memInfo C.mem_info_t
 	resp := []GpuInfo{}
 
-	// NVIDIA first
-	for i := 0; i < gpuHandles.deviceCount; i++ {
-		// TODO once we support CPU compilation variants of GPU libraries refine this...
-		if cpuVariant == "" && runtime.GOARCH == "amd64" {
-			continue
+	// Load ALL libraries
+	cHandles = initCudaHandles()
+	minMajorVer, err := strconv.Atoi(CudaComputeMajorMin)
+	if err != nil {
+		slog.Error("invalid CudaComputeMajorMin setting", "value", CudaComputeMajorMin, "error", err)
+	}
+	minMinorVer, err := strconv.Atoi(CudaComputeMinorMin)
+	if err != nil {
+		slog.Error("invalid CudaComputeMinorMin setting", "value", CudaComputeMinorMin, "error", err)
+	}
+
+	// NVIDIA
+	for i := range cHandles.deviceCount {
+		if cHandles.cudart != nil || cHandles.nvcuda != nil {
+			gpuInfo := CudaGPUInfo{
+				GpuInfo: GpuInfo{
+					Library: "cuda",
+				},
+				index: i,
+			}
+			var driverMajor int
+			var driverMinor int
+			if cHandles.cudart != nil {
+				C.cudart_bootstrap(*cHandles.cudart, C.int(i), &memInfo)
+			} else {
+				C.nvcuda_bootstrap(*cHandles.nvcuda, C.int(i), &memInfo)
+				driverMajor = int(cHandles.nvcuda.driver_major)
+				driverMinor = int(cHandles.nvcuda.driver_minor)
+			}
+			if memInfo.err != nil {
+				slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
+				C.free(unsafe.Pointer(memInfo.err))
+				continue
+			}
+
+			if int(memInfo.major) < minMajorVer || (int(memInfo.major) == minMajorVer && int(memInfo.minor) < minMinorVer) {
+				slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor))
+				continue
+			}
+			gpuInfo.TotalMemory = uint64(memInfo.total)
+			gpuInfo.FreeMemory = uint64(memInfo.free)
+			gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
+			gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor)
+			gpuInfo.MinimumMemory = cudaMinimumMemory
+			gpuInfo.DependencyPath = depPath
+			gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
+			gpuInfo.DriverMajor = driverMajor
+			gpuInfo.DriverMinor = driverMinor
+
+			// TODO potentially sort on our own algorithm instead of what the underlying GPU library does...
+			cudaGPUs = append(cudaGPUs, gpuInfo)
 		}
 		gpuInfo := GpuInfo{
 			Library: "cuda",
diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh
index e91c56ed..c03bc25f 100755
--- a/scripts/build_docker.sh
+++ b/scripts/build_docker.sh
@@ -3,7 +3,7 @@
 set -eu
 
 export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")}
-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
+export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"}
 
 # We use 2 different image repositories to handle combining architecture images into multiarch manifest
 # (The ROCm image is x86 only and is not a multiarch manifest)
diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh
index 27c4ff1f..e7e6d0dd 100755
--- a/scripts/build_linux.sh
+++ b/scripts/build_linux.sh
@@ -3,7 +3,7 @@
 set -eu
 
 export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")}
-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
+export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"}
 
 BUILD_ARCH=${BUILD_ARCH:-"amd64 arm64"}
 export AMDGPU_TARGETS=${AMDGPU_TARGETS:=""}
-- 
2.46.0