DGNum/lab-infra
8
1
Fork 1
Code Issues 1 Pull requests 1 Projects Releases Packages Wiki Activity Actions

chore(krz01): Put all ollama/whisper stuff in separate modules

Browse source
This commit is contained in:
sinavir 2024-10-18 09:26:45 +02:00
parent f60b8243cd
commit f1bd9f9a2b
10 changed files with 614 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

53
machines/krz01/_configuration.nix
View file

@ -1,11 +1,4 @@
{
config,
lib,
pkgs,
meta,
name,
...
}:
{ config, lib, ... }:
lib.extra.mkConfig {
enabledModules = [
@ -15,9 +8,11 @@ lib.extra.mkConfig {
enabledServices = [
# INFO: This list needs to stay sorted alphabetically
# Machine learning API machine
"microvm-ml01"
"microvm-router01"
# "microvm-ml01"
# "microvm-router01"
"nvidia-tesla-k80"
"ollama"
"whisper"
"proxmox"
];
@ -33,46 +28,12 @@ lib.extra.mkConfig {
];
};
services.netbird.enable = true;
# We are going to use CUDA here.
nixpkgs.config.cudaSupport = true;
hardware.graphics.enable = true;
environment.systemPackages = [
((pkgs.openai-whisper-cpp.override { cudaPackages = pkgs.cudaPackages_11; }).overrideAttrs (old: {
src = pkgs.fetchFromGitHub {
owner = "ggerganov";
repo = "whisper.cpp";
rev = "v1.7.1";
hash = "sha256-EDFUVjud79ZRCzGbOh9L9NcXfN3ikvsqkVSOME9F9oo=";
};
env = {
WHISPER_CUBLAS = "";
GGML_CUDA = "1";
};
# We only need Compute Capability 3.7.
CUDA_ARCH_FLAGS = [ "sm_37" ];
# We are GPU-only anyway.
patches = (old.patches or [ ]) ++ [
./no-weird-microarch.patch
./all-nvcc-arch.patch
];
}))
];
services = {
ollama = {
enable = true;
host = meta.network.${name}.netbirdIp;
package = pkgs.callPackage ./ollama.nix {
cudaPackages = pkgs.cudaPackages_11;
# We need to thread our nvidia x11 driver for CUDA.
extraLibraries = [ config.hardware.nvidia.package ];
};
};
};
networking.firewall.interfaces.wt0.allowedTCPPorts = [ config.services.ollama.port ];
services.netbird.enable = true;
networking.firewall.trustedInterfaces = [ "wt0" ];
};
root = ./.;