DGNum/infrastructure
13
6
Fork 3
Code Issues 20 Pull requests 9 Projects 1 Releases Packages Wiki Activity Actions

feat(krz01): introduce ml01 #141

Merged
thubrecht merged 5 commits from nvidia-fixups into main 2024-10-09 14:52:01 +02:00
Conversation 0 Commits 5 Files changed 9 +572 -4
9 changed files with 572 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

179
machines/krz01/K80-support.patch Normal file
View file

@ -0,0 +1,179 @@
From 2abd226ff3093c5a9e18a618fba466853e7ebaf7 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Tue, 8 Oct 2024 18:27:41 +0200
Subject: [PATCH] K80 support
Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
docs/development.md | 6 +++-
docs/gpu.md | 1 +
gpu/amd_linux.go | 6 +++-
gpu/gpu.go | 63 ++++++++++++++++++++++++++++++++++++-----
scripts/build_docker.sh | 2 +-
scripts/build_linux.sh | 2 +-
6 files changed, 69 insertions(+), 11 deletions(-)
diff --git a/docs/development.md b/docs/development.md
index 2f7b9ecf..9da35931 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -51,7 +51,11 @@ Typically the build scripts will auto-detect CUDA, however, if your Linux distro
or installation approach uses unusual paths, you can specify the location by
specifying an environment variable `CUDA_LIB_DIR` to the location of the shared
libraries, and `CUDACXX` to the location of the nvcc compiler. You can customize
-a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "50;60;70")
+a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "35;37;50;60;70")
+
+To support GPUs older than Compute Capability 5.0, you will need to use an older version of
+the Driver from [Unix Driver Archive](https://www.nvidia.com/en-us/drivers/unix/) (tested with 470) and [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) (tested with cuda V11). When you build Ollama, you will need to set two environment variable to adjust the minimum compute capability Ollama supports via `export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3\" \"-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5\"'"` and the `CMAKE_CUDA_ARCHITECTURES`. To find the Compute Capability of your older GPU, refer to [GPU Compute Capability](https://developer.nvidia.com/cuda-gpus).
+
Then generate dependencies:
diff --git a/docs/gpu.md b/docs/gpu.md
index a6b559f0..66627611 100644
--- a/docs/gpu.md
+++ b/docs/gpu.md
@@ -28,6 +28,7 @@ Check your compute compatibility to see if your card is supported:
| 5.0 | GeForce GTX | `GTX 750 Ti` `GTX 750` `NVS 810` |
| | Quadro | `K2200` `K1200` `K620` `M1200` `M520` `M5000M` `M4000M` `M3000M` `M2000M` `M1000M` `K620M` `M600M` `M500M` |
+For building locally to support older GPUs, see [developer.md](./development.md#linux-cuda-nvidia)
### GPU Selection
diff --git a/gpu/amd_linux.go b/gpu/amd_linux.go
index 6b08ac2e..768fb97a 100644
--- a/gpu/amd_linux.go
+++ b/gpu/amd_linux.go
@@ -159,7 +159,11 @@ func AMDGetGPUInfo() []GpuInfo {
return []GpuInfo{}
}
- if int(major) < RocmComputeMin {
+ minVer, err := strconv.Atoi(RocmComputeMajorMin)
+ if err != nil {
+ slog.Error("invalid RocmComputeMajorMin setting", "value", RocmComputeMajorMin, "error", err)
+ }
+ if int(major) < minVer {
slog.Warn(fmt.Sprintf("amdgpu too old gfx%d%x%x", major, minor, patch), "gpu", gpuID)
continue
}
diff --git a/gpu/gpu.go b/gpu/gpu.go
index 781e23df..60d68c33 100644
--- a/gpu/gpu.go
+++ b/gpu/gpu.go
@@ -16,6 +16,7 @@ import (
"os"
"path/filepath"
"runtime"
+ "strconv"
"strings"
"sync"
"unsafe"
@@ -38,9 +39,11 @@ const (
var gpuMutex sync.Mutex
// With our current CUDA compile flags, older than 5.0 will not work properly
-var CudaComputeMin = [2]C.int{5, 0}
+// (string values used to allow ldflags overrides at build time)
+var CudaComputeMajorMin = "5"
+var CudaComputeMinorMin = "0"
-var RocmComputeMin = 9
+var RocmComputeMajorMin = "9"
// TODO find a better way to detect iGPU instead of minimum memory
const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU
@@ -175,11 +178,57 @@ func GetGPUInfo() GpuInfoList {
var memInfo C.mem_info_t
resp := []GpuInfo{}
- // NVIDIA first
- for i := 0; i < gpuHandles.deviceCount; i++ {
- // TODO once we support CPU compilation variants of GPU libraries refine this...
- if cpuVariant == "" && runtime.GOARCH == "amd64" {
- continue
+ // Load ALL libraries
+ cHandles = initCudaHandles()
+ minMajorVer, err := strconv.Atoi(CudaComputeMajorMin)
+ if err != nil {
+ slog.Error("invalid CudaComputeMajorMin setting", "value", CudaComputeMajorMin, "error", err)
+ }
+ minMinorVer, err := strconv.Atoi(CudaComputeMinorMin)
+ if err != nil {
+ slog.Error("invalid CudaComputeMinorMin setting", "value", CudaComputeMinorMin, "error", err)
+ }
+
+ // NVIDIA
+ for i := range cHandles.deviceCount {
+ if cHandles.cudart != nil || cHandles.nvcuda != nil {
+ gpuInfo := CudaGPUInfo{
+ GpuInfo: GpuInfo{
+ Library: "cuda",
+ },
+ index: i,
+ }
+ var driverMajor int
+ var driverMinor int
+ if cHandles.cudart != nil {
+ C.cudart_bootstrap(*cHandles.cudart, C.int(i), &memInfo)
+ } else {
+ C.nvcuda_bootstrap(*cHandles.nvcuda, C.int(i), &memInfo)
+ driverMajor = int(cHandles.nvcuda.driver_major)
+ driverMinor = int(cHandles.nvcuda.driver_minor)
+ }
+ if memInfo.err != nil {
+ slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
+ C.free(unsafe.Pointer(memInfo.err))
+ continue
+ }
+
+ if int(memInfo.major) < minMajorVer || (int(memInfo.major) == minMajorVer && int(memInfo.minor) < minMinorVer) {
+ slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor))
+ continue
+ }
+ gpuInfo.TotalMemory = uint64(memInfo.total)
+ gpuInfo.FreeMemory = uint64(memInfo.free)
+ gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
+ gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor)
+ gpuInfo.MinimumMemory = cudaMinimumMemory
+ gpuInfo.DependencyPath = depPath
+ gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
+ gpuInfo.DriverMajor = driverMajor
+ gpuInfo.DriverMinor = driverMinor
+
+ // TODO potentially sort on our own algorithm instead of what the underlying GPU library does...
+ cudaGPUs = append(cudaGPUs, gpuInfo)
}
gpuInfo := GpuInfo{
Library: "cuda",
diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh
index e91c56ed..c03bc25f 100755
--- a/scripts/build_docker.sh
+++ b/scripts/build_docker.sh
@@ -3,7 +3,7 @@
set -eu
export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")}
-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
+export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"}
# We use 2 different image repositories to handle combining architecture images into multiarch manifest
# (The ROCm image is x86 only and is not a multiarch manifest)
diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh
index 27c4ff1f..e7e6d0dd 100755
--- a/scripts/build_linux.sh
+++ b/scripts/build_linux.sh
@@ -3,7 +3,7 @@
set -eu
export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")}
-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
+export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"}
BUILD_ARCH=${BUILD_ARCH:-"amd64 arm64"}
export AMDGPU_TARGETS=${AMDGPU_TARGETS:=""}
--
2.46.0

45
machines/krz01/_configuration.nix
View file

@ -1,4 +1,9 @@
{ lib, ... }:
{
config,
lib,
pkgs,
...
}:
lib.extra.mkConfig {
enabledModules = [
@ -10,10 +15,14 @@ lib.extra.mkConfig {
"microvm-router01"
"nvidia-tesla-k80"
"proxmox"
# Machine learning API machine
"microvm-ml01"
];
extraConfig = {
microvm.host.enable = true;
microvm = {
host.enable = true;
};
dgn-hardware = {
useZfs = true;
zfsPools = [
@ -24,6 +33,38 @@ lib.extra.mkConfig {
services.netbird.enable = true;
# We are going to use CUDA here.
nixpkgs.config.cudaSupport = true;
hardware.graphics.enable = true;
environment.systemPackages = [
((pkgs.openai-whisper-cpp.override { cudaPackages = pkgs.cudaPackages_11; }).overrideAttrs (old: {
src = pkgs.fetchFromGitHub {
owner = "ggerganov";
repo = "whisper.cpp";
rev = "v1.7.1";
hash = "sha256-EDFUVjud79ZRCzGbOh9L9NcXfN3ikvsqkVSOME9F9oo=";
};
env = {
WHISPER_CUBLAS = "";
GGML_CUDA = "1";
};
# We only need Compute Capability 3.7.
CUDA_ARCH_FLAGS = [ "sm_37" ];
# We are GPU-only anyway.
patches = (old.patches or [ ]) ++ [
./no-weird-microarch.patch
./all-nvcc-arch.patch
];
}))
];
services.ollama = {
enable = true;
package = pkgs.callPackage ./ollama.nix {
cudaPackages = pkgs.cudaPackages_11;
# We need to thread our nvidia x11 driver for CUDA.
extraLibraries = [ config.hardware.nvidia.package ];
};
};
users.users.root.hashedPassword = "$y$j9T$eNZQgDN.J5y7KTG2hXgat1$J1i5tjx5dnSZu.C9B7swXi5zMFIkUnmRrnmyLHFAt8/";
};

26
machines/krz01/all-nvcc-arch.patch Normal file
View file

@ -0,0 +1,26 @@
From 2278389ef9ac9231349440aa68f9544ddc69cdc7 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Wed, 9 Oct 2024 13:37:08 +0200
Subject: [PATCH] fix: sm_37 for nvcc
Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
Makefile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Makefile b/Makefile
index 2ccb750..70dfd9b 100644
--- a/Makefile
+++ b/Makefile
@@ -537,7 +537,7 @@ endif #GGML_CUDA_NVCC
ifdef CUDA_DOCKER_ARCH
MK_NVCCFLAGS += -Wno-deprecated-gpu-targets -arch=$(CUDA_DOCKER_ARCH)
else ifndef CUDA_POWER_ARCH
- MK_NVCCFLAGS += -arch=native
+ MK_NVCCFLAGS += -arch=sm_37
endif # CUDA_DOCKER_ARCH
ifdef GGML_CUDA_FORCE_DMMV
--
2.46.0

20
machines/krz01/disable-git.patch Normal file
View file

@ -0,0 +1,20 @@
diff --git c/llm/generate/gen_common.sh i/llm/generate/gen_common.sh
index 3825c155..238a74a7 100644
--- c/llm/generate/gen_common.sh
+++ i/llm/generate/gen_common.sh
@@ -69,6 +69,7 @@ git_module_setup() {
}
apply_patches() {
+ return
# apply temporary patches until fix is upstream
for patch in ../patches/*.patch; do
git -c 'user.name=nobody' -c 'user.email=<>' -C ${LLAMACPP_DIR} am ${patch}
@@ -133,6 +134,7 @@ install() {
# Keep the local tree clean after we're done with the build
cleanup() {
+ return
(cd ${LLAMACPP_DIR}/ && git checkout CMakeLists.txt)
if [ -n "$(ls -A ../patches/*.diff)" ]; then

22
machines/krz01/microvm-ml01.nix Normal file
View file

@ -0,0 +1,22 @@
_: {
microvm.autostart = [ "ml01" ];
microvm.vms.ml01 = {
config = {
networking.hostName = "ml01";
microvm = {
hypervisor = "cloud-hypervisor";
vcpu = 4;
mem = 4096;
balloonMem = 2048;
shares = [
{
source = "/nix/store";
mountPoint = "/nix/.ro-store";
tag = "ro-store";
proto = "virtiofs";
}
];
};
};
};
}

34
machines/krz01/no-weird-microarch.patch Normal file
View file

@ -0,0 +1,34 @@
From 51568b61ef63ecd97867562571411082c32751d3 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Wed, 9 Oct 2024 13:36:51 +0200
Subject: [PATCH] fix: avx & f16c in Makefile
Signed-off-by: Raito Bezarius <masterancpp@gmail.com>
---
Makefile | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/Makefile b/Makefile
index 32b7cbb..2ccb750 100644
--- a/Makefile
+++ b/Makefile
@@ -361,12 +361,12 @@ ifndef RISCV
ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64))
# Use all CPU extensions that are available:
- MK_CFLAGS += -march=native -mtune=native
- HOST_CXXFLAGS += -march=native -mtune=native
+ # MK_CFLAGS += -march=native -mtune=native
+ # HOST_CXXFLAGS += -march=native -mtune=native
# Usage AVX-only
- #MK_CFLAGS += -mfma -mf16c -mavx
- #MK_CXXFLAGS += -mfma -mf16c -mavx
+ MK_CFLAGS += -mf16c -mavx
+ MK_CXXFLAGS += -mf16c -mavx
# Usage SSSE3-only (Not is SSE3!)
#MK_CFLAGS += -mssse3
--
2.46.0

5
machines/krz01/nvidia-tesla-k80.nix
View file

@ -1,5 +1,8 @@
{ config, ... }:
{
nixpkgs.config.nvidia.acceptLicense = true;
# Tesla K80 is not supported by the latest driver.
hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages_legacy_470;
hardware.nvidia.package = config.boot.kernelPackages.nvidia_x11_legacy470;
# Don't ask.
services.xserver.videoDrivers = [ "nvidia" ];
}

243
machines/krz01/ollama.nix Normal file
View file

@ -0,0 +1,243 @@
{
lib,
buildGoModule,
fetchFromGitHub,
buildEnv,
linkFarm,
overrideCC,
makeWrapper,
stdenv,
addDriverRunpath,
nix-update-script,
cmake,
gcc11,
clblast,
libdrm,
rocmPackages,
cudaPackages,
darwin,
autoAddDriverRunpath,
extraLibraries ? [ ],
nixosTests,
testers,
ollama,
ollama-rocm,
ollama-cuda,
config,
# one of `[ null false "rocm" "cuda" ]`
acceleration ? null,
}:
assert builtins.elem acceleration [
null
false
"rocm"
"cuda"
];
let
pname = "ollama";
version = "2024-09-10-cc35";
src = fetchFromGitHub {
owner = "aliotard";
repo = "ollama";
rev = "34827c01f7723c7f5f9f5e392fe85f5a4a5d5fc0";
hash = "sha256-xFNuqcW7YWeyCyw5QLBnCHHTSMITR6LJkJT0CXZC+Y8=";
fetchSubmodules = true;
};
vendorHash = "sha256-hSxcREAujhvzHVNwnRTfhi0MKI3s8HNavER2VLz6SYk=";
validateFallback = lib.warnIf (config.rocmSupport && config.cudaSupport) (lib.concatStrings [
"both `nixpkgs.config.rocmSupport` and `nixpkgs.config.cudaSupport` are enabled, "
"but they are mutually exclusive; falling back to cpu"
]) (!(config.rocmSupport && config.cudaSupport));
shouldEnable =
mode: fallback: (acceleration == mode) || (fallback && acceleration == null && validateFallback);
rocmRequested = shouldEnable "rocm" config.rocmSupport;
cudaRequested = shouldEnable "cuda" config.cudaSupport;
enableRocm = rocmRequested && stdenv.isLinux;
enableCuda = cudaRequested && stdenv.isLinux;
rocmLibs = [
rocmPackages.clr
rocmPackages.hipblas
rocmPackages.rocblas
rocmPackages.rocsolver
rocmPackages.rocsparse
rocmPackages.rocm-device-libs
rocmPackages.rocm-smi
];
rocmClang = linkFarm "rocm-clang" { llvm = rocmPackages.llvm.clang; };
rocmPath = buildEnv {
name = "rocm-path";
paths = rocmLibs ++ [ rocmClang ];
};
cudaLibs = [
cudaPackages.cuda_cudart
cudaPackages.libcublas
cudaPackages.cuda_cccl
];
cudaToolkit = buildEnv {
name = "cuda-merged";
paths = map lib.getLib cudaLibs ++ [
(lib.getOutput "static" cudaPackages.cuda_cudart)
(lib.getBin (cudaPackages.cuda_nvcc.__spliced.buildHost or cudaPackages.cuda_nvcc))
];
};
metalFrameworks = with darwin.apple_sdk_11_0.frameworks; [
Accelerate
Metal
MetalKit
MetalPerformanceShaders
];
wrapperOptions =
[
# ollama embeds llama-cpp binaries which actually run the ai models
# these llama-cpp binaries are unaffected by the ollama binary's DT_RUNPATH
# LD_LIBRARY_PATH is temporarily required to use the gpu
# until these llama-cpp binaries can have their runpath patched
"--suffix LD_LIBRARY_PATH : '${addDriverRunpath.driverLink}/lib'"
"--suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath (map lib.getLib extraLibraries)}'"
]
++ lib.optionals enableRocm [
"--suffix LD_LIBRARY_PATH : '${rocmPath}/lib'"
"--set-default HIP_PATH '${rocmPath}'"
]
++ lib.optionals enableCuda [
"--suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath (map lib.getLib cudaLibs)}'"
];
wrapperArgs = builtins.concatStringsSep " " wrapperOptions;
goBuild =
if enableCuda then buildGoModule.override { stdenv = overrideCC stdenv gcc11; } else buildGoModule;
inherit (lib) licenses platforms maintainers;
in
goBuild {
inherit
pname
version
src
vendorHash
;
env =
lib.optionalAttrs enableRocm {
ROCM_PATH = rocmPath;
CLBlast_DIR = "${clblast}/lib/cmake/CLBlast";
}
// lib.optionalAttrs enableCuda { CUDA_LIB_DIR = "${cudaToolkit}/lib"; }
// {
CMAKE_CUDA_ARCHITECTURES = "35;37";
};
nativeBuildInputs =
[ cmake ]
++ lib.optionals enableRocm [ rocmPackages.llvm.bintools ]
++ lib.optionals enableCuda [ cudaPackages.cuda_nvcc ]
++ lib.optionals (enableRocm || enableCuda) [
makeWrapper
autoAddDriverRunpath
]
++ lib.optionals stdenv.isDarwin metalFrameworks;
buildInputs =
lib.optionals enableRocm (rocmLibs ++ [ libdrm ])
++ lib.optionals enableCuda cudaLibs
++ lib.optionals stdenv.isDarwin metalFrameworks;
patches = [
# disable uses of `git` in the `go generate` script
# ollama's build script assumes the source is a git repo, but nix removes the git directory
# this also disables necessary patches contained in `ollama/llm/patches/`
# those patches are applied in `postPatch`
./disable-git.patch
];
postPatch = ''
# replace inaccurate version number with actual release version
substituteInPlace version/version.go --replace-fail 0.0.0 '${version}'
# apply ollama's patches to `llama.cpp` submodule
for diff in llm/patches/*; do
patch -p1 -d llm/llama.cpp < $diff
done
'';
overrideModAttrs = _: _: {
# don't run llama.cpp build in the module fetch phase
preBuild = "";
};
preBuild = ''
# disable uses of `git`, since nix removes the git directory
export OLLAMA_SKIP_PATCHING=true
# build llama.cpp libraries for ollama
go generate ./...
'';
postFixup =
''
# the app doesn't appear functional at the moment, so hide it
mv "$out/bin/app" "$out/bin/.ollama-app"
''
+ lib.optionalString (enableRocm || enableCuda) ''
# expose runtime libraries necessary to use the gpu
wrapProgram "$out/bin/ollama" ${wrapperArgs}
'';
ldflags = [
"-s"
"-w"
"-X=github.com/ollama/ollama/version.Version=${version}"
"-X=github.com/ollama/ollama/server.mode=release"
"-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3"
"-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5"
];
passthru = {
tests =
{
inherit ollama;
version = testers.testVersion {
inherit version;
package = ollama;
};
}
// lib.optionalAttrs stdenv.isLinux {
inherit ollama-rocm ollama-cuda;
service = nixosTests.ollama;
service-cuda = nixosTests.ollama-cuda;
service-rocm = nixosTests.ollama-rocm;
};
updateScript = nix-update-script { };
};
meta = {
description =
"Get up and running with large language models locally"
+ lib.optionalString rocmRequested ", using ROCm for AMD GPU acceleration"
+ lib.optionalString cudaRequested ", using CUDA for NVIDIA GPU acceleration";
homepage = "https://github.com/ollama/ollama";
changelog = "https://github.com/ollama/ollama/releases/tag/v${version}";
license = licenses.mit;
platforms = if (rocmRequested || cudaRequested) then platforms.linux else platforms.unix;
mainProgram = "ollama";
maintainers = with maintainers; [
abysssol
dit7ya
elohmeier
roydubnium
];
};
}

2
meta/nodes.nix
View file

@ -74,7 +74,7 @@
site = "pav01";
stateVersion = "24.05";
nixpkgs = "24.05";
nixpkgs = "unstable";
};
storage01 = {