diff --git a/machines/krz01/K80-support.patch b/machines/krz01/K80-support.patch deleted file mode 100644 index d8f0c3a..0000000 --- a/machines/krz01/K80-support.patch +++ /dev/null @@ -1,179 +0,0 @@ -From 2abd226ff3093c5a9e18a618fba466853e7ebaf7 Mon Sep 17 00:00:00 2001 -From: Raito Bezarius -Date: Tue, 8 Oct 2024 18:27:41 +0200 -Subject: [PATCH] K80 support - -Signed-off-by: Raito Bezarius ---- - docs/development.md | 6 +++- - docs/gpu.md | 1 + - gpu/amd_linux.go | 6 +++- - gpu/gpu.go | 63 ++++++++++++++++++++++++++++++++++++----- - scripts/build_docker.sh | 2 +- - scripts/build_linux.sh | 2 +- - 6 files changed, 69 insertions(+), 11 deletions(-) - -diff --git a/docs/development.md b/docs/development.md -index 2f7b9ecf..9da35931 100644 ---- a/docs/development.md -+++ b/docs/development.md -@@ -51,7 +51,11 @@ Typically the build scripts will auto-detect CUDA, however, if your Linux distro - or installation approach uses unusual paths, you can specify the location by - specifying an environment variable `CUDA_LIB_DIR` to the location of the shared - libraries, and `CUDACXX` to the location of the nvcc compiler. You can customize --a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "50;60;70") -+a set of target CUDA architectures by setting `CMAKE_CUDA_ARCHITECTURES` (e.g. "35;37;50;60;70") -+ -+To support GPUs older than Compute Capability 5.0, you will need to use an older version of -+the Driver from [Unix Driver Archive](https://www.nvidia.com/en-us/drivers/unix/) (tested with 470) and [CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) (tested with cuda V11). When you build Ollama, you will need to set two environment variable to adjust the minimum compute capability Ollama supports via `export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/gpu.CudaComputeMajorMin=3\" \"-X=github.com/ollama/ollama/gpu.CudaComputeMinorMin=5\"'"` and the `CMAKE_CUDA_ARCHITECTURES`. To find the Compute Capability of your older GPU, refer to [GPU Compute Capability](https://developer.nvidia.com/cuda-gpus). -+ - - Then generate dependencies: - -diff --git a/docs/gpu.md b/docs/gpu.md -index a6b559f0..66627611 100644 ---- a/docs/gpu.md -+++ b/docs/gpu.md -@@ -28,6 +28,7 @@ Check your compute compatibility to see if your card is supported: - | 5.0 | GeForce GTX | `GTX 750 Ti` `GTX 750` `NVS 810` | - | | Quadro | `K2200` `K1200` `K620` `M1200` `M520` `M5000M` `M4000M` `M3000M` `M2000M` `M1000M` `K620M` `M600M` `M500M` | - -+For building locally to support older GPUs, see [developer.md](./development.md#linux-cuda-nvidia) - - ### GPU Selection - -diff --git a/gpu/amd_linux.go b/gpu/amd_linux.go -index 6b08ac2e..768fb97a 100644 ---- a/gpu/amd_linux.go -+++ b/gpu/amd_linux.go -@@ -159,7 +159,11 @@ func AMDGetGPUInfo() []GpuInfo { - return []GpuInfo{} - } - -- if int(major) < RocmComputeMin { -+ minVer, err := strconv.Atoi(RocmComputeMajorMin) -+ if err != nil { -+ slog.Error("invalid RocmComputeMajorMin setting", "value", RocmComputeMajorMin, "error", err) -+ } -+ if int(major) < minVer { - slog.Warn(fmt.Sprintf("amdgpu too old gfx%d%x%x", major, minor, patch), "gpu", gpuID) - continue - } -diff --git a/gpu/gpu.go b/gpu/gpu.go -index 781e23df..60d68c33 100644 ---- a/gpu/gpu.go -+++ b/gpu/gpu.go -@@ -16,6 +16,7 @@ import ( - "os" - "path/filepath" - "runtime" -+ "strconv" - "strings" - "sync" - "unsafe" -@@ -38,9 +39,11 @@ const ( - var gpuMutex sync.Mutex - - // With our current CUDA compile flags, older than 5.0 will not work properly --var CudaComputeMin = [2]C.int{5, 0} -+// (string values used to allow ldflags overrides at build time) -+var CudaComputeMajorMin = "5" -+var CudaComputeMinorMin = "0" - --var RocmComputeMin = 9 -+var RocmComputeMajorMin = "9" - - // TODO find a better way to detect iGPU instead of minimum memory - const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU -@@ -175,11 +178,57 @@ func GetGPUInfo() GpuInfoList { - var memInfo C.mem_info_t - resp := []GpuInfo{} - -- // NVIDIA first -- for i := 0; i < gpuHandles.deviceCount; i++ { -- // TODO once we support CPU compilation variants of GPU libraries refine this... -- if cpuVariant == "" && runtime.GOARCH == "amd64" { -- continue -+ // Load ALL libraries -+ cHandles = initCudaHandles() -+ minMajorVer, err := strconv.Atoi(CudaComputeMajorMin) -+ if err != nil { -+ slog.Error("invalid CudaComputeMajorMin setting", "value", CudaComputeMajorMin, "error", err) -+ } -+ minMinorVer, err := strconv.Atoi(CudaComputeMinorMin) -+ if err != nil { -+ slog.Error("invalid CudaComputeMinorMin setting", "value", CudaComputeMinorMin, "error", err) -+ } -+ -+ // NVIDIA -+ for i := range cHandles.deviceCount { -+ if cHandles.cudart != nil || cHandles.nvcuda != nil { -+ gpuInfo := CudaGPUInfo{ -+ GpuInfo: GpuInfo{ -+ Library: "cuda", -+ }, -+ index: i, -+ } -+ var driverMajor int -+ var driverMinor int -+ if cHandles.cudart != nil { -+ C.cudart_bootstrap(*cHandles.cudart, C.int(i), &memInfo) -+ } else { -+ C.nvcuda_bootstrap(*cHandles.nvcuda, C.int(i), &memInfo) -+ driverMajor = int(cHandles.nvcuda.driver_major) -+ driverMinor = int(cHandles.nvcuda.driver_minor) -+ } -+ if memInfo.err != nil { -+ slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err)) -+ C.free(unsafe.Pointer(memInfo.err)) -+ continue -+ } -+ -+ if int(memInfo.major) < minMajorVer || (int(memInfo.major) == minMajorVer && int(memInfo.minor) < minMinorVer) { -+ slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor)) -+ continue -+ } -+ gpuInfo.TotalMemory = uint64(memInfo.total) -+ gpuInfo.FreeMemory = uint64(memInfo.free) -+ gpuInfo.ID = C.GoString(&memInfo.gpu_id[0]) -+ gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor) -+ gpuInfo.MinimumMemory = cudaMinimumMemory -+ gpuInfo.DependencyPath = depPath -+ gpuInfo.Name = C.GoString(&memInfo.gpu_name[0]) -+ gpuInfo.DriverMajor = driverMajor -+ gpuInfo.DriverMinor = driverMinor -+ -+ // TODO potentially sort on our own algorithm instead of what the underlying GPU library does... -+ cudaGPUs = append(cudaGPUs, gpuInfo) - } - gpuInfo := GpuInfo{ - Library: "cuda", -diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh -index e91c56ed..c03bc25f 100755 ---- a/scripts/build_docker.sh -+++ b/scripts/build_docker.sh -@@ -3,7 +3,7 @@ - set -eu - - export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")} --export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'" -+export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"} - - # We use 2 different image repositories to handle combining architecture images into multiarch manifest - # (The ROCm image is x86 only and is not a multiarch manifest) -diff --git a/scripts/build_linux.sh b/scripts/build_linux.sh -index 27c4ff1f..e7e6d0dd 100755 ---- a/scripts/build_linux.sh -+++ b/scripts/build_linux.sh -@@ -3,7 +3,7 @@ - set -eu - - export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")} --export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'" -+export GOFLAGS=${GOFLAGS:-"'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'"} - - BUILD_ARCH=${BUILD_ARCH:-"amd64 arm64"} - export AMDGPU_TARGETS=${AMDGPU_TARGETS:=""} --- -2.46.0 - diff --git a/machines/krz01/all-nvcc-arch.patch b/machines/krz01/all-nvcc-arch.patch deleted file mode 100644 index 6696836..0000000 --- a/machines/krz01/all-nvcc-arch.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 2278389ef9ac9231349440aa68f9544ddc69cdc7 Mon Sep 17 00:00:00 2001 -From: Raito Bezarius -Date: Wed, 9 Oct 2024 13:37:08 +0200 -Subject: [PATCH] fix: sm_37 for nvcc - -Signed-off-by: Raito Bezarius ---- - Makefile | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/Makefile b/Makefile -index 2ccb750..70dfd9b 100644 ---- a/Makefile -+++ b/Makefile -@@ -537,7 +537,7 @@ endif #GGML_CUDA_NVCC - ifdef CUDA_DOCKER_ARCH - MK_NVCCFLAGS += -Wno-deprecated-gpu-targets -arch=$(CUDA_DOCKER_ARCH) - else ifndef CUDA_POWER_ARCH -- MK_NVCCFLAGS += -arch=native -+ MK_NVCCFLAGS += -arch=sm_37 - endif # CUDA_DOCKER_ARCH - - ifdef GGML_CUDA_FORCE_DMMV --- -2.46.0 - diff --git a/machines/krz01/disable-git.patch b/machines/krz01/disable-git.patch deleted file mode 100644 index c305c48..0000000 --- a/machines/krz01/disable-git.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git c/llm/generate/gen_common.sh i/llm/generate/gen_common.sh -index 3825c155..238a74a7 100644 ---- c/llm/generate/gen_common.sh -+++ i/llm/generate/gen_common.sh -@@ -69,6 +69,7 @@ git_module_setup() { - } - - apply_patches() { -+ return - # apply temporary patches until fix is upstream - for patch in ../patches/*.patch; do - git -c 'user.name=nobody' -c 'user.email=<>' -C ${LLAMACPP_DIR} am ${patch} -@@ -133,6 +134,7 @@ install() { - - # Keep the local tree clean after we're done with the build - cleanup() { -+ return - (cd ${LLAMACPP_DIR}/ && git checkout CMakeLists.txt) - - if [ -n "$(ls -A ../patches/*.diff)" ]; then diff --git a/machines/krz01/no-weird-microarch.patch b/machines/krz01/no-weird-microarch.patch deleted file mode 100644 index 7a93b53..0000000 --- a/machines/krz01/no-weird-microarch.patch +++ /dev/null @@ -1,34 +0,0 @@ -From 51568b61ef63ecd97867562571411082c32751d3 Mon Sep 17 00:00:00 2001 -From: Raito Bezarius -Date: Wed, 9 Oct 2024 13:36:51 +0200 -Subject: [PATCH] fix: avx & f16c in Makefile - -Signed-off-by: Raito Bezarius ---- - Makefile | 8 ++++---- - 1 file changed, 4 insertions(+), 4 deletions(-) - -diff --git a/Makefile b/Makefile -index 32b7cbb..2ccb750 100644 ---- a/Makefile -+++ b/Makefile -@@ -361,12 +361,12 @@ ifndef RISCV - - ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) - # Use all CPU extensions that are available: -- MK_CFLAGS += -march=native -mtune=native -- HOST_CXXFLAGS += -march=native -mtune=native -+ # MK_CFLAGS += -march=native -mtune=native -+ # HOST_CXXFLAGS += -march=native -mtune=native - - # Usage AVX-only -- #MK_CFLAGS += -mfma -mf16c -mavx -- #MK_CXXFLAGS += -mfma -mf16c -mavx -+ MK_CFLAGS += -mf16c -mavx -+ MK_CXXFLAGS += -mf16c -mavx - - # Usage SSSE3-only (Not is SSE3!) - #MK_CFLAGS += -mssse3 --- -2.46.0 -