From f1bd9f9a2b4194a2e152b78c6896b0efdc279d8e Mon Sep 17 00:00:00 2001 From: sinavir Date: Fri, 18 Oct 2024 09:26:45 +0200 Subject: [PATCH] chore(krz01): Put all ollama/whisper stuff in separate modules --- machines/krz01/_configuration.nix | 53 +--- machines/krz01/ollama/K80-support.patch | 179 +++++++++++++ machines/krz01/ollama/all-nvcc-arch.patch | 26 ++ machines/krz01/ollama/default.nix | 20 ++ machines/krz01/ollama/disable-git.patch | 20 ++ .../krz01/ollama/no-weird-microarch.patch | 34 +++ machines/krz01/ollama/package.nix | 243 ++++++++++++++++++ machines/krz01/whisper/all-nvcc-arch.patch | 26 ++ machines/krz01/whisper/default.nix | 25 ++ .../krz01/whisper/no-weird-microarch.patch | 34 +++ 10 files changed, 614 insertions(+), 46 deletions(-) create mode 100644 machines/krz01/ollama/K80-support.patch create mode 100644 machines/krz01/ollama/all-nvcc-arch.patch create mode 100644 machines/krz01/ollama/default.nix create mode 100644 machines/krz01/ollama/disable-git.patch create mode 100644 machines/krz01/ollama/no-weird-microarch.patch create mode 100644 machines/krz01/ollama/package.nix create mode 100644 machines/krz01/whisper/all-nvcc-arch.patch create mode 100644 machines/krz01/whisper/default.nix create mode 100644 machines/krz01/whisper/no-weird-microarch.patch diff --git a/machines/krz01/_configuration.nix b/machines/krz01/_configuration.nix index 21a6f59..e501401 100644 --- a/machines/krz01/_configuration.nix +++ b/machines/krz01/_configuration.nix @@ -1,11 +1,4 @@ -{ - config, - lib, - pkgs, - meta, - name, - ... -}: +{ config, lib, ... }: lib.extra.mkConfig { enabledModules = [ @@ -15,9 +8,11 @@ lib.extra.mkConfig { enabledServices = [ # INFO: This list needs to stay sorted alphabetically # Machine learning API machine - "microvm-ml01" - "microvm-router01" + # "microvm-ml01" + # "microvm-router01" "nvidia-tesla-k80" + "ollama" + "whisper" "proxmox" ]; @@ -33,46 +28,12 @@ lib.extra.mkConfig { ]; }; - services.netbird.enable = true; - # We are going to use CUDA here. nixpkgs.config.cudaSupport = true; hardware.graphics.enable = true; - environment.systemPackages = [ - ((pkgs.openai-whisper-cpp.override { cudaPackages = pkgs.cudaPackages_11; }).overrideAttrs (old: { - src = pkgs.fetchFromGitHub { - owner = "ggerganov"; - repo = "whisper.cpp"; - rev = "v1.7.1"; - hash = "sha256-EDFUVjud79ZRCzGbOh9L9NcXfN3ikvsqkVSOME9F9oo="; - }; - env = { - WHISPER_CUBLAS = ""; - GGML_CUDA = "1"; - }; - # We only need Compute Capability 3.7. - CUDA_ARCH_FLAGS = [ "sm_37" ]; - # We are GPU-only anyway. - patches = (old.patches or [ ]) ++ [ - ./no-weird-microarch.patch - ./all-nvcc-arch.patch - ]; - })) - ]; - services = { - ollama = { - enable = true; - host = meta.network.${name}.netbirdIp; - package = pkgs.callPackage ./ollama.nix { - cudaPackages = pkgs.cudaPackages_11; - # We need to thread our nvidia x11 driver for CUDA. - extraLibraries = [ config.hardware.nvidia.package ]; - }; - }; - }; - - networking.firewall.interfaces.wt0.allowedTCPPorts = [ config.services.ollama.port ]; + services.netbird.enable = true; + networking.firewall.trustedInterfaces = [ "wt0" ]; }; root = ./.; diff --git a/machines/krz01/ollama/K80-support.patch b/machines/krz01/ollama/K80-support.patch new file mode 100644 index 0000000..d8f0c3a --- /dev/null +++ b/machines/krz01/ollama/K80-support.patch @@ -0,0 +1,179 @@ +From 2abd226ff3093c5a9e18a618fba466853e7ebaf7 Mon Sep 17 00:00:00 2001 +From: Raito Bezarius +Date: Tue, 8 Oct 2024 18:27:41 +0200 +Subject: [PATCH] K80 support + +Signed-off-by: Raito Bezarius +--- + docs/development.md | 6 +++- + docs/gpu.md | 1 + + gpu/amd_linux.go | 6 +++- + gpu/gpu.go | 63 ++++++++++++++++++++++++++++++++++++----- + scripts/build_docker.sh | 2 +- + scripts/build_linux.sh | 2 +- + 6 files changed, 69 insertions(+), 11 deletions(-) + +diff --git a/docs/development.md b/docs/development.md +index 2f7b9ecf..9da35931 100644 +--- a/docs/development.md ++++ b/docs/development.md +@@ -51,7 +51,11 @@ Typically the build scripts will auto-detect CUDA, however, if your Linux distro + or installation approach uses unusual paths, you can specify the location by + specifying an environment variable `CUDA_LIB_DIR` to the location of the shared + libraries, and `CUDACXX` to the location of the nvcc compiler. You can customize +-a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "50;60;70") ++a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "35;37;50;60;70") ++ ++To support GPUs older than Compute Capability 5.0, you will need to use an older version of ++the Driver from [Unix Driver Archive](https://www.nvidia.com/en-us/drivers/unix/) (tested with 470) and [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) (tested with cuda V11). When you build Ollama, you will need to set two environment variable to adjust the minimum compute capability Ollama supports via `export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3\" \"-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5\"'"` and the `CMAKE_CUDA_ARCHITECTURES`. To find the Compute Capability of your older GPU, refer to [GPU Compute Capability](https://developer.nvidia.com/cuda-gpus). ++ + + Then generate dependencies: + +diff --git a/docs/gpu.md b/docs/gpu.md +index a6b559f0..66627611 100644 +--- a/docs/gpu.md ++++ b/docs/gpu.md +@@ -28,6 +28,7 @@ Check your compute compatibility to see if your card is supported: + | 5.0 | GeForce GTX | `GTX 750 Ti` `GTX 750` `NVS 810` | + | | Quadro | `K2200` `K1200` `K620` `M1200` `M520` `M5000M` `M4000M` `M3000M` `M2000M` `M1000M` `K620M` `M600M` `M500M` | + ++For building locally to support older GPUs, see [developer.md](./development.md#linux-cuda-nvidia) + + ### GPU Selection + +diff --git a/gpu/amd_linux.go b/gpu/amd_linux.go +index 6b08ac2e..768fb97a 100644 +--- a/gpu/amd_linux.go ++++ b/gpu/amd_linux.go +@@ -159,7 +159,11 @@ func AMDGetGPUInfo() []GpuInfo { + return []GpuInfo{} + } + +- if int(major) < RocmComputeMin { ++ minVer, err := strconv.Atoi(RocmComputeMajorMin) ++ if err != nil { ++ slog.Error("invalid RocmComputeMajorMin setting", "value", RocmComputeMajorMin, "error", err) ++ } ++ if int(major) < minVer { + slog.Warn(fmt.Sprintf("amdgpu too old gfx%d%x%x", major, minor, patch), "gpu", gpuID) + continue + } +diff --git a/gpu/gpu.go b/gpu/gpu.go +index 781e23df..60d68c33 100644 +--- a/gpu/gpu.go ++++ b/gpu/gpu.go +@@ -16,6 +16,7 @@ import ( + "os" + "path/filepath" + "runtime" ++ "strconv" + "strings" + "sync" + "unsafe" +@@ -38,9 +39,11 @@ const ( + var gpuMutex sync.Mutex + + // With our current CUDA compile flags, older than 5.0 will not work properly +-var CudaComputeMin = [2]C.int{5, 0} ++// (string values used to allow ldflags overrides at build time) ++var CudaComputeMajorMin = "5" ++var CudaComputeMinorMin = "0" + +-var RocmComputeMin = 9 ++var RocmComputeMajorMin = "9" + + // TODO find a better way to detect iGPU instead of minimum memory + const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU +@@ -175,11 +178,57 @@ func GetGPUInfo() GpuInfoList { + var memInfo C.mem_info_t + resp := []GpuInfo{} + +- // NVIDIA first +- for i := 0; i < gpuHandles.deviceCount; i++ { +- // TODO once we support CPU compilation variants of GPU libraries refine this... +- if cpuVariant == "" && runtime.GOARCH == "amd64" { +- continue ++ // Load ALL libraries ++ cHandles = initCudaHandles() ++ minMajorVer, err := strconv.Atoi(CudaComputeMajorMin) ++ if err != nil { ++ slog.Error("invalid CudaComputeMajorMin setting", "value", CudaComputeMajorMin, "error", err) ++ } ++ minMinorVer, err := strconv.Atoi(CudaComputeMinorMin) ++ if err != nil { ++ slog.Error("invalid CudaComputeMinorMin setting", "value", CudaComputeMinorMin, "error", err) ++ } ++ ++ // NVIDIA ++ for i := range cHandles.deviceCount { ++ if cHandles.cudart != nil || cHandles.nvcuda != nil { ++ gpuInfo := CudaGPUInfo{ ++ GpuInfo: GpuInfo{ ++ Library: "cuda", ++ }, ++ index: i, ++ } ++ var driverMajor int ++ var driverMinor int ++ if cHandles.cudart != nil { ++ C.cudart_bootstrap(*cHandles.cudart, C.int(i), &memInfo) ++ } else { ++ C.nvcuda_bootstrap(*cHandles.nvcuda, C.int(i), &memInfo) ++ driverMajor = int(cHandles.nvcuda.driver_major) ++ driverMinor = int(cHandles.nvcuda.driver_minor) ++ } ++ if memInfo.err != nil { ++ slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err)) ++ C.free(unsafe.Pointer(memInfo.err)) ++ continue ++ } ++ ++ if int(memInfo.major) < minMajorVer || (int(memInfo.major) == minMajorVer && int(memInfo.minor) < minMinorVer) { ++ slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor)) ++ continue ++ } ++ gpuInfo.TotalMemory = uint64(memInfo.total) ++ gpuInfo.FreeMemory = uint64(memInfo.free) ++ gpuInfo.ID = C.GoString(&memInfo.gpu_id[0]) ++ gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor) ++ gpuInfo.MinimumMemory = cudaMinimumMemory ++ gpuInfo.DependencyPath = depPath ++ gpuInfo.Name = C.GoString(&memInfo.gpu_name[0]) ++ gpuInfo.DriverMajor = driverMajor ++ gpuInfo.DriverMinor = driverMinor ++ ++ // TODO potentially sort on our own algorithm instead of what the underlying GPU library does... ++ cudaGPUs = append(cudaGPUs, gpuInfo) + } + gpuInfo := GpuInfo{ + Library: "cuda", +diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh +index e91c56ed..c03bc25f 100755 +--- a/scripts/build_docker.sh ++++ b/scripts/build_docker.sh +@@ -3,7 +3,7 @@ + set -eu + + export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")} +-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'" ++export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"} + + # We use 2 different image repositories to handle combining architecture images into multiarch manifest + # (The ROCm image is x86 only and is not a multiarch manifest) +diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh +index 27c4ff1f..e7e6d0dd 100755 +--- a/scripts/build_linux.sh ++++ b/scripts/build_linux.sh +@@ -3,7 +3,7 @@ + set -eu + + export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")} +-export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'" ++export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"} + + BUILD_ARCH=${BUILD_ARCH:-"amd64 arm64"} + export AMDGPU_TARGETS=${AMDGPU_TARGETS:=""} +-- +2.46.0 + diff --git a/machines/krz01/ollama/all-nvcc-arch.patch b/machines/krz01/ollama/all-nvcc-arch.patch new file mode 100644 index 0000000..6696836 --- /dev/null +++ b/machines/krz01/ollama/all-nvcc-arch.patch @@ -0,0 +1,26 @@ +From 2278389ef9ac9231349440aa68f9544ddc69cdc7 Mon Sep 17 00:00:00 2001 +From: Raito Bezarius +Date: Wed, 9 Oct 2024 13:37:08 +0200 +Subject: [PATCH] fix: sm_37 for nvcc + +Signed-off-by: Raito Bezarius +--- + Makefile | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Makefile b/Makefile +index 2ccb750..70dfd9b 100644 +--- a/Makefile ++++ b/Makefile +@@ -537,7 +537,7 @@ endif #GGML_CUDA_NVCC + ifdef CUDA_DOCKER_ARCH + MK_NVCCFLAGS += -Wno-deprecated-gpu-targets -arch=$(CUDA_DOCKER_ARCH) + else ifndef CUDA_POWER_ARCH +- MK_NVCCFLAGS += -arch=native ++ MK_NVCCFLAGS += -arch=sm_37 + endif # CUDA_DOCKER_ARCH + + ifdef GGML_CUDA_FORCE_DMMV +-- +2.46.0 + diff --git a/machines/krz01/ollama/default.nix b/machines/krz01/ollama/default.nix new file mode 100644 index 0000000..b0fead5 --- /dev/null +++ b/machines/krz01/ollama/default.nix @@ -0,0 +1,20 @@ +{ + config, + pkgs, + meta, + name, + ... +}: +{ + services = { + ollama = { + enable = true; + host = meta.network.${name}.netbirdIp; + package = pkgs.callPackage ./package.nix { + cudaPackages = pkgs.cudaPackages_11; + # We need to thread our nvidia x11 driver for CUDA. + extraLibraries = [ config.hardware.nvidia.package ]; + }; + }; + }; +} diff --git a/machines/krz01/ollama/disable-git.patch b/machines/krz01/ollama/disable-git.patch new file mode 100644 index 0000000..c305c48 --- /dev/null +++ b/machines/krz01/ollama/disable-git.patch @@ -0,0 +1,20 @@ +diff --git c/llm/generate/gen_common.sh i/llm/generate/gen_common.sh +index 3825c155..238a74a7 100644 +--- c/llm/generate/gen_common.sh ++++ i/llm/generate/gen_common.sh +@@ -69,6 +69,7 @@ git_module_setup() { + } + + apply_patches() { ++ return + # apply temporary patches until fix is upstream + for patch in ../patches/*.patch; do + git -c 'user.name=nobody' -c 'user.email=<>' -C ${LLAMACPP_DIR} am ${patch} +@@ -133,6 +134,7 @@ install() { + + # Keep the local tree clean after we're done with the build + cleanup() { ++ return + (cd ${LLAMACPP_DIR}/ && git checkout CMakeLists.txt) + + if [ -n "$(ls -A ../patches/*.diff)" ]; then diff --git a/machines/krz01/ollama/no-weird-microarch.patch b/machines/krz01/ollama/no-weird-microarch.patch new file mode 100644 index 0000000..7a93b53 --- /dev/null +++ b/machines/krz01/ollama/no-weird-microarch.patch @@ -0,0 +1,34 @@ +From 51568b61ef63ecd97867562571411082c32751d3 Mon Sep 17 00:00:00 2001 +From: Raito Bezarius +Date: Wed, 9 Oct 2024 13:36:51 +0200 +Subject: [PATCH] fix: avx & f16c in Makefile + +Signed-off-by: Raito Bezarius +--- + Makefile | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/Makefile b/Makefile +index 32b7cbb..2ccb750 100644 +--- a/Makefile ++++ b/Makefile +@@ -361,12 +361,12 @@ ifndef RISCV + + ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) + # Use all CPU extensions that are available: +- MK_CFLAGS += -march=native -mtune=native +- HOST_CXXFLAGS += -march=native -mtune=native ++ # MK_CFLAGS += -march=native -mtune=native ++ # HOST_CXXFLAGS += -march=native -mtune=native + + # Usage AVX-only +- #MK_CFLAGS += -mfma -mf16c -mavx +- #MK_CXXFLAGS += -mfma -mf16c -mavx ++ MK_CFLAGS += -mf16c -mavx ++ MK_CXXFLAGS += -mf16c -mavx + + # Usage SSSE3-only (Not is SSE3!) + #MK_CFLAGS += -mssse3 +-- +2.46.0 + diff --git a/machines/krz01/ollama/package.nix b/machines/krz01/ollama/package.nix new file mode 100644 index 0000000..6e252c1 --- /dev/null +++ b/machines/krz01/ollama/package.nix @@ -0,0 +1,243 @@ +{ + lib, + buildGoModule, + fetchFromGitHub, + buildEnv, + linkFarm, + overrideCC, + makeWrapper, + stdenv, + addDriverRunpath, + nix-update-script, + + cmake, + gcc11, + clblast, + libdrm, + rocmPackages, + cudaPackages, + darwin, + autoAddDriverRunpath, + extraLibraries ? [ ], + + nixosTests, + testers, + ollama, + ollama-rocm, + ollama-cuda, + + config, + # one of `[ null false "rocm" "cuda" ]` + acceleration ? null, +}: + +assert builtins.elem acceleration [ + null + false + "rocm" + "cuda" +]; + +let + pname = "ollama"; + version = "2024-09-10-cc35"; + + src = fetchFromGitHub { + owner = "aliotard"; + repo = "ollama"; + rev = "34827c01f7723c7f5f9f5e392fe85f5a4a5d5fc0"; + hash = "sha256-xFNuqcW7YWeyCyw5QLBnCHHTSMITR6LJkJT0CXZC+Y8="; + fetchSubmodules = true; + }; + + vendorHash = "sha256-hSxcREAujhvzHVNwnRTfhi0MKI3s8HNavER2VLz6SYk="; + + validateFallback = lib.warnIf (config.rocmSupport && config.cudaSupport) (lib.concatStrings [ + "both `nixpkgs.config.rocmSupport` and `nixpkgs.config.cudaSupport` are enabled, " + "but they are mutually exclusive; falling back to cpu" + ]) (!(config.rocmSupport && config.cudaSupport)); + shouldEnable = + mode: fallback: (acceleration == mode) || (fallback && acceleration == null && validateFallback); + + rocmRequested = shouldEnable "rocm" config.rocmSupport; + cudaRequested = shouldEnable "cuda" config.cudaSupport; + + enableRocm = rocmRequested && stdenv.isLinux; + enableCuda = cudaRequested && stdenv.isLinux; + + rocmLibs = [ + rocmPackages.clr + rocmPackages.hipblas + rocmPackages.rocblas + rocmPackages.rocsolver + rocmPackages.rocsparse + rocmPackages.rocm-device-libs + rocmPackages.rocm-smi + ]; + rocmClang = linkFarm "rocm-clang" { llvm = rocmPackages.llvm.clang; }; + rocmPath = buildEnv { + name = "rocm-path"; + paths = rocmLibs ++ [ rocmClang ]; + }; + + cudaLibs = [ + cudaPackages.cuda_cudart + cudaPackages.libcublas + cudaPackages.cuda_cccl + ]; + cudaToolkit = buildEnv { + name = "cuda-merged"; + paths = map lib.getLib cudaLibs ++ [ + (lib.getOutput "static" cudaPackages.cuda_cudart) + (lib.getBin (cudaPackages.cuda_nvcc.__spliced.buildHost or cudaPackages.cuda_nvcc)) + ]; + }; + + metalFrameworks = with darwin.apple_sdk_11_0.frameworks; [ + Accelerate + Metal + MetalKit + MetalPerformanceShaders + ]; + + wrapperOptions = + [ + # ollama embeds llama-cpp binaries which actually run the ai models + # these llama-cpp binaries are unaffected by the ollama binary's DT_RUNPATH + # LD_LIBRARY_PATH is temporarily required to use the gpu + # until these llama-cpp binaries can have their runpath patched + "--suffix LD_LIBRARY_PATH : '${addDriverRunpath.driverLink}/lib'" + "--suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath (map lib.getLib extraLibraries)}'" + ] + ++ lib.optionals enableRocm [ + "--suffix LD_LIBRARY_PATH : '${rocmPath}/lib'" + "--set-default HIP_PATH '${rocmPath}'" + ] + ++ lib.optionals enableCuda [ + "--suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath (map lib.getLib cudaLibs)}'" + ]; + wrapperArgs = builtins.concatStringsSep " " wrapperOptions; + + goBuild = + if enableCuda then buildGoModule.override { stdenv = overrideCC stdenv gcc11; } else buildGoModule; + inherit (lib) licenses platforms maintainers; +in +goBuild { + inherit + pname + version + src + vendorHash + ; + + env = + lib.optionalAttrs enableRocm { + ROCM_PATH = rocmPath; + CLBlast_DIR = "${clblast}/lib/cmake/CLBlast"; + } + // lib.optionalAttrs enableCuda { CUDA_LIB_DIR = "${cudaToolkit}/lib"; } + // { + CMAKE_CUDA_ARCHITECTURES = "35;37"; + }; + + nativeBuildInputs = + [ cmake ] + ++ lib.optionals enableRocm [ rocmPackages.llvm.bintools ] + ++ lib.optionals enableCuda [ cudaPackages.cuda_nvcc ] + ++ lib.optionals (enableRocm || enableCuda) [ + makeWrapper + autoAddDriverRunpath + ] + ++ lib.optionals stdenv.isDarwin metalFrameworks; + + buildInputs = + lib.optionals enableRocm (rocmLibs ++ [ libdrm ]) + ++ lib.optionals enableCuda cudaLibs + ++ lib.optionals stdenv.isDarwin metalFrameworks; + + patches = [ + # disable uses of `git` in the `go generate` script + # ollama's build script assumes the source is a git repo, but nix removes the git directory + # this also disables necessary patches contained in `ollama/llm/patches/` + # those patches are applied in `postPatch` + ./disable-git.patch + ]; + + postPatch = '' + # replace inaccurate version number with actual release version + substituteInPlace version/version.go --replace-fail 0.0.0 '${version}' + + # apply ollama's patches to `llama.cpp` submodule + for diff in llm/patches/*; do + patch -p1 -d llm/llama.cpp < $diff + done + ''; + + overrideModAttrs = _: _: { + # don't run llama.cpp build in the module fetch phase + preBuild = ""; + }; + + preBuild = '' + # disable uses of `git`, since nix removes the git directory + export OLLAMA_SKIP_PATCHING=true + # build llama.cpp libraries for ollama + go generate ./... + ''; + + postFixup = + '' + # the app doesn't appear functional at the moment, so hide it + mv "$out/bin/app" "$out/bin/.ollama-app" + '' + + lib.optionalString (enableRocm || enableCuda) '' + # expose runtime libraries necessary to use the gpu + wrapProgram "$out/bin/ollama" ${wrapperArgs} + ''; + + ldflags = [ + "-s" + "-w" + "-X=github.com/ollama/ollama/version.Version=${version}" + "-X=github.com/ollama/ollama/server.mode=release" + "-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3" + "-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5" + ]; + + passthru = { + tests = + { + inherit ollama; + version = testers.testVersion { + inherit version; + package = ollama; + }; + } + // lib.optionalAttrs stdenv.isLinux { + inherit ollama-rocm ollama-cuda; + service = nixosTests.ollama; + service-cuda = nixosTests.ollama-cuda; + service-rocm = nixosTests.ollama-rocm; + }; + + updateScript = nix-update-script { }; + }; + + meta = { + description = + "Get up and running with large language models locally" + + lib.optionalString rocmRequested ", using ROCm for AMD GPU acceleration" + + lib.optionalString cudaRequested ", using CUDA for NVIDIA GPU acceleration"; + homepage = "https://github.com/ollama/ollama"; + changelog = "https://github.com/ollama/ollama/releases/tag/v${version}"; + license = licenses.mit; + platforms = if (rocmRequested || cudaRequested) then platforms.linux else platforms.unix; + mainProgram = "ollama"; + maintainers = with maintainers; [ + abysssol + dit7ya + elohmeier + roydubnium + ]; + }; +} diff --git a/machines/krz01/whisper/all-nvcc-arch.patch b/machines/krz01/whisper/all-nvcc-arch.patch new file mode 100644 index 0000000..6696836 --- /dev/null +++ b/machines/krz01/whisper/all-nvcc-arch.patch @@ -0,0 +1,26 @@ +From 2278389ef9ac9231349440aa68f9544ddc69cdc7 Mon Sep 17 00:00:00 2001 +From: Raito Bezarius +Date: Wed, 9 Oct 2024 13:37:08 +0200 +Subject: [PATCH] fix: sm_37 for nvcc + +Signed-off-by: Raito Bezarius +--- + Makefile | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Makefile b/Makefile +index 2ccb750..70dfd9b 100644 +--- a/Makefile ++++ b/Makefile +@@ -537,7 +537,7 @@ endif #GGML_CUDA_NVCC + ifdef CUDA_DOCKER_ARCH + MK_NVCCFLAGS += -Wno-deprecated-gpu-targets -arch=$(CUDA_DOCKER_ARCH) + else ifndef CUDA_POWER_ARCH +- MK_NVCCFLAGS += -arch=native ++ MK_NVCCFLAGS += -arch=sm_37 + endif # CUDA_DOCKER_ARCH + + ifdef GGML_CUDA_FORCE_DMMV +-- +2.46.0 + diff --git a/machines/krz01/whisper/default.nix b/machines/krz01/whisper/default.nix new file mode 100644 index 0000000..689b60f --- /dev/null +++ b/machines/krz01/whisper/default.nix @@ -0,0 +1,25 @@ +{ pkgs, ... }: +{ + environment.systemPackages = [ + ((pkgs.openai-whisper-cpp.override { cudaPackages = pkgs.cudaPackages_11; }).overrideAttrs (old: { + src = pkgs.fetchFromGitHub { + owner = "ggerganov"; + repo = "whisper.cpp"; + rev = "v1.7.1"; + hash = "sha256-EDFUVjud79ZRCzGbOh9L9NcXfN3ikvsqkVSOME9F9oo="; + }; + env = { + WHISPER_CUBLAS = ""; + GGML_CUDA = "1"; + }; + # We only need Compute Capability 3.7. + CUDA_ARCH_FLAGS = [ "sm_37" ]; + # We are GPU-only anyway. + patches = (old.patches or [ ]) ++ [ + ./no-weird-microarch.patch + ./all-nvcc-arch.patch + ]; + })) + ]; + +} diff --git a/machines/krz01/whisper/no-weird-microarch.patch b/machines/krz01/whisper/no-weird-microarch.patch new file mode 100644 index 0000000..7a93b53 --- /dev/null +++ b/machines/krz01/whisper/no-weird-microarch.patch @@ -0,0 +1,34 @@ +From 51568b61ef63ecd97867562571411082c32751d3 Mon Sep 17 00:00:00 2001 +From: Raito Bezarius +Date: Wed, 9 Oct 2024 13:36:51 +0200 +Subject: [PATCH] fix: avx & f16c in Makefile + +Signed-off-by: Raito Bezarius +--- + Makefile | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/Makefile b/Makefile +index 32b7cbb..2ccb750 100644 +--- a/Makefile ++++ b/Makefile +@@ -361,12 +361,12 @@ ifndef RISCV + + ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) + # Use all CPU extensions that are available: +- MK_CFLAGS += -march=native -mtune=native +- HOST_CXXFLAGS += -march=native -mtune=native ++ # MK_CFLAGS += -march=native -mtune=native ++ # HOST_CXXFLAGS += -march=native -mtune=native + + # Usage AVX-only +- #MK_CFLAGS += -mfma -mf16c -mavx +- #MK_CXXFLAGS += -mfma -mf16c -mavx ++ MK_CFLAGS += -mf16c -mavx ++ MK_CXXFLAGS += -mf16c -mavx + + # Usage SSSE3-only (Not is SSE3!) + #MK_CFLAGS += -mssse3 +-- +2.46.0 +