prain: tune searches the auto CPU threshold and writes --cpu-threads; vLLM patch: fit Gemma 4 in 8 GB (tied lm_head without a copy, rope caches sized by max_model_len)

2026-10-27 23:52:53 +01:00
parent d69fd34492
commit 01dce2ffc6
4 changed files with 75 additions and 3 deletions
@@ -226,15 +226,19 @@ DIMENSIONS = (
    ("prefetch", "mode", ("none", "pregate")),
    ("prefetch", "distance", (1, 2)),
    ("prefetch", "budget", (1, 2, 4)),
-    ("miss", "mode", ("copy", "auto", "cpu")),
+    # One dimension: cpu_max_tokens only matters for auto, and coordinate descent would never
+    # reach auto with a high threshold once auto <= 1 lost to cpu.
+    ("miss", ("mode", "cpu_max_tokens"),
+     (("copy", 1), ("cpu", 1)) + tuple(("auto", n) for n in (1, 2, 4, 8, 16))),
    ("miss", "admit", (0, 1, 2)),
 )
 _PART = {"policy": 0, "prefetch": 1, "miss": 2}


 def _with(variant, table, name, value):
+    names, values = (name, value) if isinstance(name, tuple) else ((name,), (value,))
    parts = list(variant)
-    parts[_PART[table]] = replace(parts[_PART[table]], **{name: value})
+    parts[_PART[table]] = replace(parts[_PART[table]], **dict(zip(names, values)))
    return tuple(parts)