From 2abd226ff3093c5a9e18a618fba466853e7ebaf7 Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Tue, 8 Oct 2024 18:27:41 +0200 Subject: [PATCH] K80 support Signed-off-by: Raito Bezarius --- docs/development.md | 6 +++- docs/gpu.md | 1 + gpu/amd_linux.go | 6 +++- gpu/gpu.go | 63 ++++++++++++++++++++++++++++++++++++----- scripts/build_docker.sh | 2 +- scripts/build_linux.sh | 2 +- 6 files changed, 69 insertions(+), 11 deletions(-) diff --git a/docs/development.md b/docs/development.md index 2f7b9ecf..9da35931 100644 --- a/docs/development.md +++ b/docs/development.md @@ -51,7 +51,11 @@ Typically the build scripts will auto-detect CUDA, however, if your Linux distro or installation approach uses unusual paths, you can specify the location by specifying an environment variable `CUDA_LIB_DIR` to the location of the shared libraries, and `CUDACXX` to the location of the nvcc compiler. You can customize -a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "50;60;70") +a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "35;37;50;60;70") + +To support GPUs older than Compute Capability 5.0, you will need to use an older version of +the Driver from [Unix Driver Archive](https://www.nvidia.com/en-us/drivers/unix/) (tested with 470) and [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) (tested with cuda V11). When you build Ollama, you will need to set two environment variable to adjust the minimum compute capability Ollama supports via `export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3\" \"-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5\"'"` and the `CMAKE_CUDA_ARCHITECTURES`. To find the Compute Capability of your older GPU, refer to [GPU Compute Capability](https://developer.nvidia.com/cuda-gpus). + Then generate dependencies: diff --git a/docs/gpu.md b/docs/gpu.md index a6b559f0..66627611 100644 --- a/docs/gpu.md +++ b/docs/gpu.md @@ -28,6 +28,7 @@ Check your compute compatibility to see if your card is supported: | 5.0 | GeForce GTX | `GTX 750 Ti` `GTX 750` `NVS 810` | | | Quadro | `K2200` `K1200` `K620` `M1200` `M520` `M5000M` `M4000M` `M3000M` `M2000M` `M1000M` `K620M` `M600M` `M500M` | +For building locally to support older GPUs, see [developer.md](./development.md#linux-cuda-nvidia) ### GPU Selection diff --git a/gpu/amd_linux.go b/gpu/amd_linux.go index 6b08ac2e..768fb97a 100644 --- a/gpu/amd_linux.go +++ b/gpu/amd_linux.go @@ -159,7 +159,11 @@ func AMDGetGPUInfo() []GpuInfo { return []GpuInfo{} } - if int(major) < RocmComputeMin { + minVer, err := strconv.Atoi(RocmComputeMajorMin) + if err != nil { + slog.Error("invalid RocmComputeMajorMin setting", "value", RocmComputeMajorMin, "error", err) + } + if int(major) < minVer { slog.Warn(fmt.Sprintf("amdgpu too old gfx%d%x%x", major, minor, patch), "gpu", gpuID) continue } diff --git a/gpu/gpu.go b/gpu/gpu.go index 781e23df..60d68c33 100644 --- a/gpu/gpu.go +++ b/gpu/gpu.go @@ -16,6 +16,7 @@ import ( "os" "path/filepath" "runtime" + "strconv" "strings" "sync" "unsafe" @@ -38,9 +39,11 @@ const ( var gpuMutex sync.Mutex // With our current CUDA compile flags, older than 5.0 will not work properly -var CudaComputeMin = [2]C.int{5, 0} +// (string values used to allow ldflags overrides at build time) +var CudaComputeMajorMin = "5" +var CudaComputeMinorMin = "0" -var RocmComputeMin = 9 +var RocmComputeMajorMin = "9" // TODO find a better way to detect iGPU instead of minimum memory const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU @@ -175,11 +178,57 @@ func GetGPUInfo() GpuInfoList { var memInfo C.mem_info_t resp := []GpuInfo{} - // NVIDIA first - for i := 0; i < gpuHandles.deviceCount; i++ { - // TODO once we support CPU compilation variants of GPU libraries refine this... - if cpuVariant == "" && runtime.GOARCH == "amd64" { - continue + // Load ALL libraries + cHandles = initCudaHandles() + minMajorVer, err := strconv.Atoi(CudaComputeMajorMin) + if err != nil { + slog.Error("invalid CudaComputeMajorMin setting", "value", CudaComputeMajorMin, "error", err) + } + minMinorVer, err := strconv.Atoi(CudaComputeMinorMin) + if err != nil { + slog.Error("invalid CudaComputeMinorMin setting", "value", CudaComputeMinorMin, "error", err) + } + + // NVIDIA + for i := range cHandles.deviceCount { + if cHandles.cudart != nil || cHandles.nvcuda != nil { + gpuInfo := CudaGPUInfo{ + GpuInfo: GpuInfo{ + Library: "cuda", + }, + index: i, + } + var driverMajor int + var driverMinor int + if cHandles.cudart != nil { + C.cudart_bootstrap(*cHandles.cudart, C.int(i), &memInfo) + } else { + C.nvcuda_bootstrap(*cHandles.nvcuda, C.int(i), &memInfo) + driverMajor = int(cHandles.nvcuda.driver_major) + driverMinor = int(cHandles.nvcuda.driver_minor) + } + if memInfo.err != nil { + slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err)) + C.free(unsafe.Pointer(memInfo.err)) + continue + } + + if int(memInfo.major) < minMajorVer || (int(memInfo.major) == minMajorVer && int(memInfo.minor) < minMinorVer) { + slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor)) + continue + } + gpuInfo.TotalMemory = uint64(memInfo.total) + gpuInfo.FreeMemory = uint64(memInfo.free) + gpuInfo.ID = C.GoString(&memInfo.gpu_id[0]) + gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor) + gpuInfo.MinimumMemory = cudaMinimumMemory + gpuInfo.DependencyPath = depPath + gpuInfo.Name = C.GoString(&memInfo.gpu_name[0]) + gpuInfo.DriverMajor = driverMajor + gpuInfo.DriverMinor = driverMinor + + // TODO potentially sort on our own algorithm instead of what the underlying GPU library does... + cudaGPUs = append(cudaGPUs, gpuInfo) } gpuInfo := GpuInfo{ Library: "cuda", diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index e91c56ed..c03bc25f 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -3,7 +3,7 @@ set -eu export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")} -export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'" +export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"} # We use 2 different image repositories to handle combining architecture images into multiarch manifest # (The ROCm image is x86 only and is not a multiarch manifest) diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh index 27c4ff1f..e7e6d0dd 100755 --- a/scripts/build_linux.sh +++ b/scripts/build_linux.sh @@ -3,7 +3,7 @@ set -eu export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")} -export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'" +export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"} BUILD_ARCH=${BUILD_ARCH:-"amd64 arm64"} export AMDGPU_TARGETS=${AMDGPU_TARGETS:=""} -- 2.46.0