feat(krz01): move the GPU stuff to the host for now
We also add a K80 specific patch for ollama. Signed-off-by: Ryan Lahfa <ryan@dgnum.eu>
This commit is contained in:
parent
8160b2762f
commit
4bedb3f497
3 changed files with 213 additions and 44 deletions
179
machines/krz01/K80-support.patch
Normal file
179
machines/krz01/K80-support.patch
Normal file
|
|
@ -0,0 +1,179 @@
|
||||||
|
From 2abd226ff3093c5a9e18a618fba466853e7ebaf7 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Raito Bezarius <masterancpp@gmail.com>
|
||||||
|
Date: Tue, 8 Oct 2024 18:27:41 +0200
|
||||||
|
Subject: [PATCH] K80 support
|
||||||
|
|
||||||
|
Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
|
||||||
|
---
|
||||||
|
docs/development.md | 6 +++-
|
||||||
|
docs/gpu.md | 1 +
|
||||||
|
gpu/amd_linux.go | 6 +++-
|
||||||
|
gpu/gpu.go | 63 ++++++++++++++++++++++++++++++++++++-----
|
||||||
|
scripts/build_docker.sh | 2 +-
|
||||||
|
scripts/build_linux.sh | 2 +-
|
||||||
|
6 files changed, 69 insertions(+), 11 deletions(-)
|
||||||
|
|
||||||
|
diff --git a/docs/development.md b/docs/development.md
|
||||||
|
index 2f7b9ecf..9da35931 100644
|
||||||
|
--- a/docs/development.md
|
||||||
|
+++ b/docs/development.md
|
||||||
|
@@ -51,7 +51,11 @@ Typically the build scripts will auto-detect CUDA, however, if your Linux distro
|
||||||
|
or installation approach uses unusual paths, you can specify the location by
|
||||||
|
specifying an environment variable `CUDA_LIB_DIR` to the location of the shared
|
||||||
|
libraries, and `CUDACXX` to the location of the nvcc compiler. You can customize
|
||||||
|
-a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "50;60;70")
|
||||||
|
+a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "35;37;50;60;70")
|
||||||
|
+
|
||||||
|
+To support GPUs older than Compute Capability 5.0, you will need to use an older version of
|
||||||
|
+the Driver from [Unix Driver Archive](https://www.nvidia.com/en-us/drivers/unix/) (tested with 470) and [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) (tested with cuda V11). When you build Ollama, you will need to set two environment variable to adjust the minimum compute capability Ollama supports via `export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3\" \"-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5\"'"` and the `CMAKE_CUDA_ARCHITECTURES`. To find the Compute Capability of your older GPU, refer to [GPU Compute Capability](https://developer.nvidia.com/cuda-gpus).
|
||||||
|
+
|
||||||
|
|
||||||
|
Then generate dependencies:
|
||||||
|
|
||||||
|
diff --git a/docs/gpu.md b/docs/gpu.md
|
||||||
|
index a6b559f0..66627611 100644
|
||||||
|
--- a/docs/gpu.md
|
||||||
|
+++ b/docs/gpu.md
|
||||||
|
@@ -28,6 +28,7 @@ Check your compute compatibility to see if your card is supported:
|
||||||
|
| 5.0 | GeForce GTX | `GTX 750 Ti` `GTX 750` `NVS 810` |
|
||||||
|
| | Quadro | `K2200` `K1200` `K620` `M1200` `M520` `M5000M` `M4000M` `M3000M` `M2000M` `M1000M` `K620M` `M600M` `M500M` |
|
||||||
|
|
||||||
|
+For building locally to support older GPUs, see [developer.md](./development.md#linux-cuda-nvidia)
|
||||||
|
|
||||||
|
### GPU Selection
|
||||||
|
|
||||||
|
diff --git a/gpu/amd_linux.go b/gpu/amd_linux.go
|
||||||
|
index 6b08ac2e..768fb97a 100644
|
||||||
|
--- a/gpu/amd_linux.go
|
||||||
|
+++ b/gpu/amd_linux.go
|
||||||
|
@@ -159,7 +159,11 @@ func AMDGetGPUInfo() []GpuInfo {
|
||||||
|
return []GpuInfo{}
|
||||||
|
}
|
||||||
|
|
||||||
|
- if int(major) < RocmComputeMin {
|
||||||
|
+ minVer, err := strconv.Atoi(RocmComputeMajorMin)
|
||||||
|
+ if err != nil {
|
||||||
|
+ slog.Error("invalid RocmComputeMajorMin setting", "value", RocmComputeMajorMin, "error", err)
|
||||||
|
+ }
|
||||||
|
+ if int(major) < minVer {
|
||||||
|
slog.Warn(fmt.Sprintf("amdgpu too old gfx%d%x%x", major, minor, patch), "gpu", gpuID)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
diff --git a/gpu/gpu.go b/gpu/gpu.go
|
||||||
|
index 781e23df..60d68c33 100644
|
||||||
|
--- a/gpu/gpu.go
|
||||||
|
+++ b/gpu/gpu.go
|
||||||
|
@@ -16,6 +16,7 @@ import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
+ "strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"unsafe"
|
||||||
|
@@ -38,9 +39,11 @@ const (
|
||||||
|
var gpuMutex sync.Mutex
|
||||||
|
|
||||||
|
// With our current CUDA compile flags, older than 5.0 will not work properly
|
||||||
|
-var CudaComputeMin = [2]C.int{5, 0}
|
||||||
|
+// (string values used to allow ldflags overrides at build time)
|
||||||
|
+var CudaComputeMajorMin = "5"
|
||||||
|
+var CudaComputeMinorMin = "0"
|
||||||
|
|
||||||
|
-var RocmComputeMin = 9
|
||||||
|
+var RocmComputeMajorMin = "9"
|
||||||
|
|
||||||
|
// TODO find a better way to detect iGPU instead of minimum memory
|
||||||
|
const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU
|
||||||
|
@@ -175,11 +178,57 @@ func GetGPUInfo() GpuInfoList {
|
||||||
|
var memInfo C.mem_info_t
|
||||||
|
resp := []GpuInfo{}
|
||||||
|
|
||||||
|
- // NVIDIA first
|
||||||
|
- for i := 0; i < gpuHandles.deviceCount; i++ {
|
||||||
|
- // TODO once we support CPU compilation variants of GPU libraries refine this...
|
||||||
|
- if cpuVariant == "" && runtime.GOARCH == "amd64" {
|
||||||
|
- continue
|
||||||
|
+ // Load ALL libraries
|
||||||
|
+ cHandles = initCudaHandles()
|
||||||
|
+ minMajorVer, err := strconv.Atoi(CudaComputeMajorMin)
|
||||||
|
+ if err != nil {
|
||||||
|
+ slog.Error("invalid CudaComputeMajorMin setting", "value", CudaComputeMajorMin, "error", err)
|
||||||
|
+ }
|
||||||
|
+ minMinorVer, err := strconv.Atoi(CudaComputeMinorMin)
|
||||||
|
+ if err != nil {
|
||||||
|
+ slog.Error("invalid CudaComputeMinorMin setting", "value", CudaComputeMinorMin, "error", err)
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ // NVIDIA
|
||||||
|
+ for i := range cHandles.deviceCount {
|
||||||
|
+ if cHandles.cudart != nil || cHandles.nvcuda != nil {
|
||||||
|
+ gpuInfo := CudaGPUInfo{
|
||||||
|
+ GpuInfo: GpuInfo{
|
||||||
|
+ Library: "cuda",
|
||||||
|
+ },
|
||||||
|
+ index: i,
|
||||||
|
+ }
|
||||||
|
+ var driverMajor int
|
||||||
|
+ var driverMinor int
|
||||||
|
+ if cHandles.cudart != nil {
|
||||||
|
+ C.cudart_bootstrap(*cHandles.cudart, C.int(i), &memInfo)
|
||||||
|
+ } else {
|
||||||
|
+ C.nvcuda_bootstrap(*cHandles.nvcuda, C.int(i), &memInfo)
|
||||||
|
+ driverMajor = int(cHandles.nvcuda.driver_major)
|
||||||
|
+ driverMinor = int(cHandles.nvcuda.driver_minor)
|
||||||
|
+ }
|
||||||
|
+ if memInfo.err != nil {
|
||||||
|
+ slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
|
||||||
|
+ C.free(unsafe.Pointer(memInfo.err))
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ if int(memInfo.major) < minMajorVer || (int(memInfo.major) == minMajorVer && int(memInfo.minor) < minMinorVer) {
|
||||||
|
+ slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor))
|
||||||
|
+ continue
|
||||||
|
+ }
|
||||||
|
+ gpuInfo.TotalMemory = uint64(memInfo.total)
|
||||||
|
+ gpuInfo.FreeMemory = uint64(memInfo.free)
|
||||||
|
+ gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
|
||||||
|
+ gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor)
|
||||||
|
+ gpuInfo.MinimumMemory = cudaMinimumMemory
|
||||||
|
+ gpuInfo.DependencyPath = depPath
|
||||||
|
+ gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
|
||||||
|
+ gpuInfo.DriverMajor = driverMajor
|
||||||
|
+ gpuInfo.DriverMinor = driverMinor
|
||||||
|
+
|
||||||
|
+ // TODO potentially sort on our own algorithm instead of what the underlying GPU library does...
|
||||||
|
+ cudaGPUs = append(cudaGPUs, gpuInfo)
|
||||||
|
}
|
||||||
|
gpuInfo := GpuInfo{
|
||||||
|
Library: "cuda",
|
||||||
|
diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh
|
||||||
|
index e91c56ed..c03bc25f 100755
|
||||||
|
--- a/scripts/build_docker.sh
|
||||||
|
+++ b/scripts/build_docker.sh
|
||||||
|
@@ -3,7 +3,7 @@
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")}
|
||||||
|
-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
|
||||||
|
+export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"}
|
||||||
|
|
||||||
|
# We use 2 different image repositories to handle combining architecture images into multiarch manifest
|
||||||
|
# (The ROCm image is x86 only and is not a multiarch manifest)
|
||||||
|
diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh
|
||||||
|
index 27c4ff1f..e7e6d0dd 100755
|
||||||
|
--- a/scripts/build_linux.sh
|
||||||
|
+++ b/scripts/build_linux.sh
|
||||||
|
@@ -3,7 +3,7 @@
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")}
|
||||||
|
-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
|
||||||
|
+export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"}
|
||||||
|
|
||||||
|
BUILD_ARCH=${BUILD_ARCH:-"amd64 arm64"}
|
||||||
|
export AMDGPU_TARGETS=${AMDGPU_TARGETS:=""}
|
||||||
|
--
|
||||||
|
2.46.0
|
||||||
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
{ lib, ... }:
|
{ lib, pkgs, ... }:
|
||||||
|
|
||||||
lib.extra.mkConfig {
|
lib.extra.mkConfig {
|
||||||
enabledModules = [
|
enabledModules = [
|
||||||
|
|
@ -30,7 +30,23 @@ lib.extra.mkConfig {
|
||||||
|
|
||||||
# We are going to use CUDA here.
|
# We are going to use CUDA here.
|
||||||
nixpkgs.config.cudaSupport = true;
|
nixpkgs.config.cudaSupport = true;
|
||||||
|
services.ollama = {
|
||||||
|
enable = true;
|
||||||
|
package =
|
||||||
|
(pkgs.ollama.override {
|
||||||
|
cudaPackages = pkgs.cudaPackages_11;
|
||||||
|
gcc12 = pkgs.gcc11;
|
||||||
|
}).overrideAttrs
|
||||||
|
(old: {
|
||||||
|
CMAKE_CUDA_ARCHITECTURES = "35";
|
||||||
|
ldflags = old.ldflags ++ [
|
||||||
|
# K80 is 3.5
|
||||||
|
"-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3"
|
||||||
|
"-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5"
|
||||||
|
];
|
||||||
|
patches = (old.patches or [ ]) ++ [ ./K80-support.patch ];
|
||||||
|
});
|
||||||
|
};
|
||||||
users.users.root.hashedPassword = "$y$j9T$eNZQgDN.J5y7KTG2hXgat1$J1i5tjx5dnSZu.C9B7swXi5zMFIkUnmRrnmyLHFAt8/";
|
users.users.root.hashedPassword = "$y$j9T$eNZQgDN.J5y7KTG2hXgat1$J1i5tjx5dnSZu.C9B7swXi5zMFIkUnmRrnmyLHFAt8/";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,48 +1,22 @@
|
||||||
_: {
|
_: {
|
||||||
microvm.autostart = [ "ml01" ];
|
microvm.autostart = [ "ml01" ];
|
||||||
microvm.vms.ml01 = {
|
microvm.vms.ml01 = {
|
||||||
config =
|
config = {
|
||||||
{ config, ... }:
|
networking.hostName = "ml01";
|
||||||
{
|
microvm = {
|
||||||
nixpkgs.config.cudaSupport = true;
|
hypervisor = "cloud-hypervisor";
|
||||||
nixpkgs.config.nvidia.acceptLicense = true;
|
vcpu = 4;
|
||||||
# Tesla K80 is not supported by the latest driver.
|
mem = 4096;
|
||||||
hardware.nvidia.package = config.boot.kernelPackages.nvidia_x11_legacy470;
|
balloonMem = 2048;
|
||||||
# Don't ask.
|
shares = [
|
||||||
services.xserver.videoDrivers = [ "nvidia" ];
|
{
|
||||||
networking.hostName = "ml01";
|
source = "/nix/store";
|
||||||
services.ollama = {
|
mountPoint = "/nix/.ro-store";
|
||||||
enable = true;
|
tag = "ro-store";
|
||||||
listenAddress = "0.0.0.0:11434";
|
proto = "virtiofs";
|
||||||
sandbox = true;
|
}
|
||||||
acceleration = "cuda";
|
];
|
||||||
};
|
|
||||||
|
|
||||||
microvm = {
|
|
||||||
hypervisor = "cloud-hypervisor";
|
|
||||||
vcpu = 4;
|
|
||||||
mem = 4096;
|
|
||||||
balloonMem = 2048;
|
|
||||||
devices = [
|
|
||||||
# The nVidia Tesla K80
|
|
||||||
{
|
|
||||||
bus = "pci";
|
|
||||||
path = "0000:44:00.0";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
bus = "pci";
|
|
||||||
path = "0000:45:00.0";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
shares = [
|
|
||||||
{
|
|
||||||
source = "/nix/store";
|
|
||||||
mountPoint = "/nix/.ro-store";
|
|
||||||
tag = "ro-store";
|
|
||||||
proto = "virtiofs";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue