From 8ac0ce0405c1a53c6d63d7dcd1e4c0ce9fb19443 Mon Sep 17 00:00:00 2001
From: Edward Delaporte <edward@delaporte.us>
Date: Sun, 2 Aug 2026 18:14:06 -0500
Subject: [PATCH 1/1] Lama.cpp WIP

---
 bin/llama-cli.sh    |  6 ++++++
 files/llama-cpp.ini | 15 +++++++++++++--
 2 files changed, 19 insertions(+), 2 deletions(-)
 create mode 100755 bin/llama-cli.sh

diff --git a/bin/llama-cli.sh b/bin/llama-cli.sh
new file mode 100755
index 0000000..747a9f5
--- /dev/null
+++ b/bin/llama-cli.sh
@@ -0,0 +1,6 @@
+# MODEL=https://huggingface.co/TheBloke/Llama-2-7B-GGUF/resolve/main/llama-2-7b.Q4_K_M.gguf
+# MODEL=TheBloke/Llama-2-7B-GGUF/resolve/main/llama-2-7b.Q4_K_M.gguf
+MODEL=TheBloke/Llama-2-7B-GGUF
+
+echo $MODEL
+llama cli --ctx-size 8200 --hf-repo "$MODEL" -p $1
diff --git a/files/llama-cpp.ini b/files/llama-cpp.ini
index 3c5657a..f260af2 100644
--- a/files/llama-cpp.ini
+++ b/files/llama-cpp.ini
@@ -3,6 +3,17 @@ version = 1
 ; (Optional) This section provides global settings shared across all presets.
 ; If the same key is defined in a specific preset, it will override the value in this global section.
 [*]
-c = 8192
-n-gpu-layers = 8
+c = 8192  # works good
+; c = 16400  # works less well
+# c = 2048  # works terrible
+
+
+; When you ask for too many layers, llama.cpp fails at load time with a CUDA out of memory error. It will not silently fall back. It stops.
+
+; The fix is to drop ngl by a few and reload. Step down until it loads. If you are right at the edge, shave 2 or 3 layers and try again.
+
+; Watch your VRAM with nvidia-smi while the model loads. You want a buffer left over, not a card pinned at 100 percent. Other apps, your desktop, and the KV cache all want a slice.
+
+; 8 was fine, 16 was fine
+n-gpu-layers = 16
 
-- 
2.47.3