modules/local-llm.nix


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

{
  config,
  lib,
  pkgs,
  inputs,
  ...
}:

let
  unstable = import inputs.nixpkgs-unstable {
    system = "x86_64-linux";
    config.allowUnfree = true;
    config.rocmSupport = true;
  };
in
{
  services.ollama = {
    enable = true;
    package = unstable.ollama;
    acceleration = "rocm";
    rocmOverrideGfx = "10.3.0";
    environmentVariables = {
      # This forces ROCm to see ONLY the RX 6800.
      # On most Ryzen 7000 systems, '0' is the dGPU and '1' is the iGPU.
      HIP_VISIBLE_DEVICES = "0";
      ROCR_VISIBLE_DEVICES = "0";

      # Keep these for RDNA 2 stability with 26B models
      HSA_OVERRIDE_GFX_VERSION = "10.3.0";
      HSA_ENABLE_SDMA = "0";
    };
  };

  environment.variables.HSA_OVERRIDE_GFX_VERSION = "10.3.0";
}