files/llama-cpp.ini

   1 version = 1
   2
   3 ; (Optional) This section provides global settings shared across all presets.
   4 ; If the same key is defined in a specific preset, it will override the value in this global section.
   5 [*]
   6 c = 8192  # works good
   7 ; c = 16400  # works less well
   8 # c = 2048  # works terrible
   9
  10
  11 ; When you ask for too many layers, llama.cpp fails at load time with a CUDA out of memory error. It will not silently fall back. It stops.
  12
  13 ; The fix is to drop ngl by a few and reload. Step down until it loads. If you are right at the edge, shave 2 or 3 layers and try again.
  14
  15 ; Watch your VRAM with nvidia-smi while the model loads. You want a buffer left over, not a card pinned at 100 percent. Other apps, your desktop, and the KV cache all want a slice.
  16
  17 ; 8 was fine, 16 was fine
  18 n-gpu-layers = 16
  19