diff --git a/machines/krz01/K80-support.patch b/machines/krz01/K80-support.patch new file mode 100644 index 0000000..d8f0c3a --- /dev/null +++ b/machines/krz01/K80-support.patch @@ -0,0 +1,179 @@ +From 2abd226ff3093c5a9e18a618fba466853e7ebaf7 Mon Sep 17 00:00:00 2001 +From: Raito Bezarius +Date: Tue, 8 Oct 2024 18:27:41 +0200 +Subject: [PATCH] K80 support + +Signed-off-by: Raito Bezarius +--- + docs/development.md | 6 +++- + docs/gpu.md | 1 + + gpu/amd_linux.go | 6 +++- + gpu/gpu.go | 63 ++++++++++++++++++++++++++++++++++++----- + scripts/build_docker.sh | 2 +- + scripts/build_linux.sh | 2 +- + 6 files changed, 69 insertions(+), 11 deletions(-) + +diff --git a/docs/development.md b/docs/development.md +index 2f7b9ecf..9da35931 100644 +--- a/docs/development.md ++++ b/docs/development.md +@@ -51,7 +51,11 @@ Typically the build scripts will auto-detect CUDA, however, if your Linux distro + or installation approach uses unusual paths, you can specify the location by + specifying an environment variable `CUDA_LIB_DIR` to the location of the shared + libraries, and `CUDACXX` to the location of the nvcc compiler. You can customize +-a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "50;60;70") ++a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "35;37;50;60;70") ++ ++To support GPUs older than Compute Capability 5.0, you will need to use an older version of ++the Driver from [Unix Driver Archive](https://www.nvidia.com/en-us/drivers/unix/) (tested with 470) and [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) (tested with cuda V11). When you build Ollama, you will need to set two environment variable to adjust the minimum compute capability Ollama supports via `export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3\" \"-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5\"'"` and the `CMAKE_CUDA_ARCHITECTURES`. To find the Compute Capability of your older GPU, refer to [GPU Compute Capability](https://developer.nvidia.com/cuda-gpus). ++ + + Then generate dependencies: + +diff --git a/docs/gpu.md b/docs/gpu.md +index a6b559f0..66627611 100644 +--- a/docs/gpu.md ++++ b/docs/gpu.md +@@ -28,6 +28,7 @@ Check your compute compatibility to see if your card is supported: + | 5.0 | GeForce GTX | `GTX 750 Ti` `GTX 750` `NVS 810` | + | | Quadro | `K2200` `K1200` `K620` `M1200` `M520` `M5000M` `M4000M` `M3000M` `M2000M` `M1000M` `K620M` `M600M` `M500M` | + ++For building locally to support older GPUs, see [developer.md](./development.md#linux-cuda-nvidia) + + ### GPU Selection + +diff --git a/gpu/amd_linux.go b/gpu/amd_linux.go +index 6b08ac2e..768fb97a 100644 +--- a/gpu/amd_linux.go ++++ b/gpu/amd_linux.go +@@ -159,7 +159,11 @@ func AMDGetGPUInfo() []GpuInfo { + return []GpuInfo{} + } + +- if int(major) < RocmComputeMin { ++ minVer, err := strconv.Atoi(RocmComputeMajorMin) ++ if err != nil { ++ slog.Error("invalid RocmComputeMajorMin setting", "value", RocmComputeMajorMin, "error", err) ++ } ++ if int(major) < minVer { + slog.Warn(fmt.Sprintf("amdgpu too old gfx%d%x%x", major, minor, patch), "gpu", gpuID) + continue + } +diff --git a/gpu/gpu.go b/gpu/gpu.go +index 781e23df..60d68c33 100644 +--- a/gpu/gpu.go ++++ b/gpu/gpu.go +@@ -16,6 +16,7 @@ import ( + "os" + "path/filepath" + "runtime" ++ "strconv" + "strings" + "sync" + "unsafe" +@@ -38,9 +39,11 @@ const ( + var gpuMutex sync.Mutex + + // With our current CUDA compile flags, older than 5.0 will not work properly +-var CudaComputeMin = [2]C.int{5, 0} ++// (string values used to allow ldflags overrides at build time) ++var CudaComputeMajorMin = "5" ++var CudaComputeMinorMin = "0" + +-var RocmComputeMin = 9 ++var RocmComputeMajorMin = "9" + + // TODO find a better way to detect iGPU instead of minimum memory + const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU +@@ -175,11 +178,57 @@ func GetGPUInfo() GpuInfoList { + var memInfo C.mem_info_t + resp := []GpuInfo{} + +- // NVIDIA first +- for i := 0; i < gpuHandles.deviceCount; i++ { +- // TODO once we support CPU compilation variants of GPU libraries refine this... +- if cpuVariant == "" && runtime.GOARCH == "amd64" { +- continue ++ // Load ALL libraries ++ cHandles = initCudaHandles() ++ minMajorVer, err := strconv.Atoi(CudaComputeMajorMin) ++ if err != nil { ++ slog.Error("invalid CudaComputeMajorMin setting", "value", CudaComputeMajorMin, "error", err) ++ } ++ minMinorVer, err := strconv.Atoi(CudaComputeMinorMin) ++ if err != nil { ++ slog.Error("invalid CudaComputeMinorMin setting", "value", CudaComputeMinorMin, "error", err) ++ } ++ ++ // NVIDIA ++ for i := range cHandles.deviceCount { ++ if cHandles.cudart != nil || cHandles.nvcuda != nil { ++ gpuInfo := CudaGPUInfo{ ++ GpuInfo: GpuInfo{ ++ Library: "cuda", ++ }, ++ index: i, ++ } ++ var driverMajor int ++ var driverMinor int ++ if cHandles.cudart != nil { ++ C.cudart_bootstrap(*cHandles.cudart, C.int(i), &memInfo) ++ } else { ++ C.nvcuda_bootstrap(*cHandles.nvcuda, C.int(i), &memInfo) ++ driverMajor = int(cHandles.nvcuda.driver_major) ++ driverMinor = int(cHandles.nvcuda.driver_minor) ++ } ++ if memInfo.err != nil { ++ slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err)) ++ C.free(unsafe.Pointer(memInfo.err)) ++ continue ++ } ++ ++ if int(memInfo.major) < minMajorVer || (int(memInfo.major) == minMajorVer && int(memInfo.minor) < minMinorVer) { ++ slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor)) ++ continue ++ } ++ gpuInfo.TotalMemory = uint64(memInfo.total) ++ gpuInfo.FreeMemory = uint64(memInfo.free) ++ gpuInfo.ID = C.GoString(&memInfo.gpu_id[0]) ++ gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor) ++ gpuInfo.MinimumMemory = cudaMinimumMemory ++ gpuInfo.DependencyPath = depPath ++ gpuInfo.Name = C.GoString(&memInfo.gpu_name[0]) ++ gpuInfo.DriverMajor = driverMajor ++ gpuInfo.DriverMinor = driverMinor ++ ++ // TODO potentially sort on our own algorithm instead of what the underlying GPU library does... ++ cudaGPUs = append(cudaGPUs, gpuInfo) + } + gpuInfo := GpuInfo{ + Library: "cuda", +diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh +index e91c56ed..c03bc25f 100755 +--- a/scripts/build_docker.sh ++++ b/scripts/build_docker.sh +@@ -3,7 +3,7 @@ + set -eu + + export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")} +-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'" ++export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"} + + # We use 2 different image repositories to handle combining architecture images into multiarch manifest + # (The ROCm image is x86 only and is not a multiarch manifest) +diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh +index 27c4ff1f..e7e6d0dd 100755 +--- a/scripts/build_linux.sh ++++ b/scripts/build_linux.sh +@@ -3,7 +3,7 @@ + set -eu + + export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")} +-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'" ++export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"} + + BUILD_ARCH=${BUILD_ARCH:-"amd64 arm64"} + export AMDGPU_TARGETS=${AMDGPU_TARGETS:=""} +-- +2.46.0 + diff --git a/machines/krz01/_configuration.nix b/machines/krz01/_configuration.nix index a1e821d..eb8208f 100644 --- a/machines/krz01/_configuration.nix +++ b/machines/krz01/_configuration.nix @@ -1,4 +1,4 @@ -{ lib, ... }: +{ lib, pkgs, ... }: lib.extra.mkConfig { enabledModules = [ @@ -30,7 +30,23 @@ lib.extra.mkConfig { # We are going to use CUDA here. nixpkgs.config.cudaSupport = true; - + services.ollama = { + enable = true; + package = + (pkgs.ollama.override { + cudaPackages = pkgs.cudaPackages_11; + gcc12 = pkgs.gcc11; + }).overrideAttrs + (old: { + CMAKE_CUDA_ARCHITECTURES = "35"; + ldflags = old.ldflags ++ [ + # K80 is 3.5 + "-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3" + "-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5" + ]; + patches = (old.patches or [ ]) ++ [ ./K80-support.patch ]; + }); + }; users.users.root.hashedPassword = "$y$j9T$eNZQgDN.J5y7KTG2hXgat1$J1i5tjx5dnSZu.C9B7swXi5zMFIkUnmRrnmyLHFAt8/"; }; diff --git a/machines/krz01/microvm-ml01.nix b/machines/krz01/microvm-ml01.nix index a3811ce..b0a8be8 100644 --- a/machines/krz01/microvm-ml01.nix +++ b/machines/krz01/microvm-ml01.nix @@ -1,48 +1,22 @@ _: { microvm.autostart = [ "ml01" ]; microvm.vms.ml01 = { - config = - { config, ... }: - { - nixpkgs.config.cudaSupport = true; - nixpkgs.config.nvidia.acceptLicense = true; - # Tesla K80 is not supported by the latest driver. - hardware.nvidia.package = config.boot.kernelPackages.nvidia_x11_legacy470; - # Don't ask. - services.xserver.videoDrivers = [ "nvidia" ]; - networking.hostName = "ml01"; - services.ollama = { - enable = true; - listenAddress = "0.0.0.0:11434"; - sandbox = true; - acceleration = "cuda"; - }; - - microvm = { - hypervisor = "cloud-hypervisor"; - vcpu = 4; - mem = 4096; - balloonMem = 2048; - devices = [ - # The nVidia Tesla K80 - { - bus = "pci"; - path = "0000:44:00.0"; - } - { - bus = "pci"; - path = "0000:45:00.0"; - } - ]; - shares = [ - { - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - tag = "ro-store"; - proto = "virtiofs"; - } - ]; - }; + config = { + networking.hostName = "ml01"; + microvm = { + hypervisor = "cloud-hypervisor"; + vcpu = 4; + mem = 4096; + balloonMem = 2048; + shares = [ + { + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + tag = "ro-store"; + proto = "virtiofs"; + } + ]; }; + }; }; }