From 492fe550d95489b71e47566e6bb12168a471a5a7 Mon Sep 17 00:00:00 2001 From: sinavir Date: Thu, 17 Oct 2024 15:45:31 +0200 Subject: [PATCH] chore(krz01): Put all ollama/whisper stuff in separate modules --- machines/krz01/_configuration.nix | 47 ++----------------- machines/krz01/{ => ollama}/K80-support.patch | 0 .../krz01/{ => ollama}/all-nvcc-arch.patch | 0 machines/krz01/ollama/default.nix | 20 ++++++++ machines/krz01/{ => ollama}/disable-git.patch | 0 .../{ => ollama}/no-weird-microarch.patch | 0 .../krz01/{ollama.nix => ollama/package.nix} | 0 machines/krz01/whisper/all-nvcc-arch.patch | 26 ++++++++++ machines/krz01/whisper/default.nix | 25 ++++++++++ .../krz01/whisper/no-weird-microarch.patch | 34 ++++++++++++++ 10 files changed, 109 insertions(+), 43 deletions(-) rename machines/krz01/{ => ollama}/K80-support.patch (100%) rename machines/krz01/{ => ollama}/all-nvcc-arch.patch (100%) create mode 100644 machines/krz01/ollama/default.nix rename machines/krz01/{ => ollama}/disable-git.patch (100%) rename machines/krz01/{ => ollama}/no-weird-microarch.patch (100%) rename machines/krz01/{ollama.nix => ollama/package.nix} (100%) create mode 100644 machines/krz01/whisper/all-nvcc-arch.patch create mode 100644 machines/krz01/whisper/default.nix create mode 100644 machines/krz01/whisper/no-weird-microarch.patch diff --git a/machines/krz01/_configuration.nix b/machines/krz01/_configuration.nix index 336de44..e37d0a5 100644 --- a/machines/krz01/_configuration.nix +++ b/machines/krz01/_configuration.nix @@ -1,11 +1,4 @@ -{ - config, - lib, - pkgs, - meta, - name, - ... -}: +{ config, lib, ... }: lib.extra.mkConfig { enabledModules = [ @@ -18,6 +11,8 @@ lib.extra.mkConfig { "microvm-ml01" "microvm-router01" "nvidia-tesla-k80" + "ollama" + "whisper" "proxmox" ]; @@ -33,45 +28,11 @@ lib.extra.mkConfig { ]; }; - services.netbird.enable = true; - # We are going to use CUDA here. nixpkgs.config.cudaSupport = true; hardware.graphics.enable = true; - environment.systemPackages = [ - ((pkgs.openai-whisper-cpp.override { cudaPackages = pkgs.cudaPackages_11; }).overrideAttrs (old: { - src = pkgs.fetchFromGitHub { - owner = "ggerganov"; - repo = "whisper.cpp"; - rev = "v1.7.1"; - hash = "sha256-EDFUVjud79ZRCzGbOh9L9NcXfN3ikvsqkVSOME9F9oo="; - }; - env = { - WHISPER_CUBLAS = ""; - GGML_CUDA = "1"; - }; - # We only need Compute Capability 3.7. - CUDA_ARCH_FLAGS = [ "sm_37" ]; - # We are GPU-only anyway. - patches = (old.patches or [ ]) ++ [ - ./no-weird-microarch.patch - ./all-nvcc-arch.patch - ]; - })) - ]; - - services = { - ollama = { - enable = true; - host = meta.network.${name}.netbirdIp; - package = pkgs.callPackage ./ollama.nix { - cudaPackages = pkgs.cudaPackages_11; - # We need to thread our nvidia x11 driver for CUDA. - extraLibraries = [ config.hardware.nvidia.package ]; - }; - }; - }; + services.netbird.enable = true; networking.firewall.trustedInterfaces = [ "wt0" ]; }; diff --git a/machines/krz01/K80-support.patch b/machines/krz01/ollama/K80-support.patch similarity index 100% rename from machines/krz01/K80-support.patch rename to machines/krz01/ollama/K80-support.patch diff --git a/machines/krz01/all-nvcc-arch.patch b/machines/krz01/ollama/all-nvcc-arch.patch similarity index 100% rename from machines/krz01/all-nvcc-arch.patch rename to machines/krz01/ollama/all-nvcc-arch.patch diff --git a/machines/krz01/ollama/default.nix b/machines/krz01/ollama/default.nix new file mode 100644 index 0000000..b0fead5 --- /dev/null +++ b/machines/krz01/ollama/default.nix @@ -0,0 +1,20 @@ +{ + config, + pkgs, + meta, + name, + ... +}: +{ + services = { + ollama = { + enable = true; + host = meta.network.${name}.netbirdIp; + package = pkgs.callPackage ./package.nix { + cudaPackages = pkgs.cudaPackages_11; + # We need to thread our nvidia x11 driver for CUDA. + extraLibraries = [ config.hardware.nvidia.package ]; + }; + }; + }; +} diff --git a/machines/krz01/disable-git.patch b/machines/krz01/ollama/disable-git.patch similarity index 100% rename from machines/krz01/disable-git.patch rename to machines/krz01/ollama/disable-git.patch diff --git a/machines/krz01/no-weird-microarch.patch b/machines/krz01/ollama/no-weird-microarch.patch similarity index 100% rename from machines/krz01/no-weird-microarch.patch rename to machines/krz01/ollama/no-weird-microarch.patch diff --git a/machines/krz01/ollama.nix b/machines/krz01/ollama/package.nix similarity index 100% rename from machines/krz01/ollama.nix rename to machines/krz01/ollama/package.nix diff --git a/machines/krz01/whisper/all-nvcc-arch.patch b/machines/krz01/whisper/all-nvcc-arch.patch new file mode 100644 index 0000000..6696836 --- /dev/null +++ b/machines/krz01/whisper/all-nvcc-arch.patch @@ -0,0 +1,26 @@ +From 2278389ef9ac9231349440aa68f9544ddc69cdc7 Mon Sep 17 00:00:00 2001 +From: Raito Bezarius +Date: Wed, 9 Oct 2024 13:37:08 +0200 +Subject: [PATCH] fix: sm_37 for nvcc + +Signed-off-by: Raito Bezarius +--- + Makefile | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Makefile b/Makefile +index 2ccb750..70dfd9b 100644 +--- a/Makefile ++++ b/Makefile +@@ -537,7 +537,7 @@ endif #GGML_CUDA_NVCC + ifdef CUDA_DOCKER_ARCH + MK_NVCCFLAGS += -Wno-deprecated-gpu-targets -arch=$(CUDA_DOCKER_ARCH) + else ifndef CUDA_POWER_ARCH +- MK_NVCCFLAGS += -arch=native ++ MK_NVCCFLAGS += -arch=sm_37 + endif # CUDA_DOCKER_ARCH + + ifdef GGML_CUDA_FORCE_DMMV +-- +2.46.0 + diff --git a/machines/krz01/whisper/default.nix b/machines/krz01/whisper/default.nix new file mode 100644 index 0000000..689b60f --- /dev/null +++ b/machines/krz01/whisper/default.nix @@ -0,0 +1,25 @@ +{ pkgs, ... }: +{ + environment.systemPackages = [ + ((pkgs.openai-whisper-cpp.override { cudaPackages = pkgs.cudaPackages_11; }).overrideAttrs (old: { + src = pkgs.fetchFromGitHub { + owner = "ggerganov"; + repo = "whisper.cpp"; + rev = "v1.7.1"; + hash = "sha256-EDFUVjud79ZRCzGbOh9L9NcXfN3ikvsqkVSOME9F9oo="; + }; + env = { + WHISPER_CUBLAS = ""; + GGML_CUDA = "1"; + }; + # We only need Compute Capability 3.7. + CUDA_ARCH_FLAGS = [ "sm_37" ]; + # We are GPU-only anyway. + patches = (old.patches or [ ]) ++ [ + ./no-weird-microarch.patch + ./all-nvcc-arch.patch + ]; + })) + ]; + +} diff --git a/machines/krz01/whisper/no-weird-microarch.patch b/machines/krz01/whisper/no-weird-microarch.patch new file mode 100644 index 0000000..7a93b53 --- /dev/null +++ b/machines/krz01/whisper/no-weird-microarch.patch @@ -0,0 +1,34 @@ +From 51568b61ef63ecd97867562571411082c32751d3 Mon Sep 17 00:00:00 2001 +From: Raito Bezarius +Date: Wed, 9 Oct 2024 13:36:51 +0200 +Subject: [PATCH] fix: avx & f16c in Makefile + +Signed-off-by: Raito Bezarius +--- + Makefile | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/Makefile b/Makefile +index 32b7cbb..2ccb750 100644 +--- a/Makefile ++++ b/Makefile +@@ -361,12 +361,12 @@ ifndef RISCV + + ifeq ($(UNAME_M),$(filter $(UNAME_M),x86_64 i686 amd64)) + # Use all CPU extensions that are available: +- MK_CFLAGS += -march=native -mtune=native +- HOST_CXXFLAGS += -march=native -mtune=native ++ # MK_CFLAGS += -march=native -mtune=native ++ # HOST_CXXFLAGS += -march=native -mtune=native + + # Usage AVX-only +- #MK_CFLAGS += -mfma -mf16c -mavx +- #MK_CXXFLAGS += -mfma -mf16c -mavx ++ MK_CFLAGS += -mf16c -mavx ++ MK_CXXFLAGS += -mf16c -mavx + + # Usage SSSE3-only (Not is SSE3!) + #MK_CFLAGS += -mssse3 +-- +2.46.0 + -- 2.47.0