Extract local LLM (Ollama/ROCm) config into modules/local-llm.nix

1 files changed, 35 insertions, 0 deletions

diff --git a/modules/local-llm.nix b/modules/local-llm.nix
new file mode 100644
index 0000000..67d4cac
--- /dev/null
+++ b/modules/local-llm.nix
@@ -0,0 +1,35 @@
+{
+  config,
+  lib,
+  pkgs,
+  inputs,
+  ...
+}:
+
+let
+  unstable = import inputs.nixpkgs-unstable {
+    system = "x86_64-linux";
+    config.allowUnfree = true;
+    config.rocmSupport = true;
+  };
+in
+{
+  services.ollama = {
+    enable = true;
+    package = unstable.ollama;
+    acceleration = "rocm";
+    rocmOverrideGfx = "10.3.0";
+    environmentVariables = {
+      # This forces ROCm to see ONLY the RX 6800.
+      # On most Ryzen 7000 systems, '0' is the dGPU and '1' is the iGPU.
+      HIP_VISIBLE_DEVICES = "0";
+      ROCR_VISIBLE_DEVICES = "0";
+
+      # Keep these for RDNA 2 stability with 26B models
+      HSA_OVERRIDE_GFX_VERSION = "10.3.0";
+      HSA_ENABLE_SDMA = "0";
+    };
+  };
+
+  environment.variables.HSA_OVERRIDE_GFX_VERSION = "10.3.0";
+}