prain: tune searches the auto CPU threshold and writes --cpu-threads; vLLM patch: fit Gemma 4 in 8 GB (tied lm_head without a copy, rope caches sized by max_model_len)

2026-10-27 23:52:53 +01:00
parent d69fd34492
commit 01dce2ffc6
4 changed files with 75 additions and 3 deletions
@@ -81,3 +81,13 @@ class Tune(unittest.TestCase):
        self.assertLessEqual(final.total_ms, default.total_ms)
        self.assertEqual(rows[0][1].total_ms, min(r.total_ms for _, r in rows))
        self.assertGreaterEqual(bound.hit_rate, 0.0)
+
+    def test_tune_searches_the_auto_threshold_and_keeps_the_cpu_threads(self):
+        forwards = forwards_of(synthetic_trace(**SMALL))
+        cost = Cost(expert_mib=64.0, h2d_gbps=2.5, cpu_ms=0.5)  # eGPU: a copy is ~50 CPU routes
+        base = (Policy(), Prefetch(), Miss(cpu_threads=16))
+        best, ram, per, final, bound, rows = tune(forwards, 32, 1.0, 1.0, cost, base=base)
+        self.assertEqual(best[2].cpu_threads, 16)
+        self.assertNotEqual(best[2].mode, "copy")
+        tried = {(v[2].mode, v[2].cpu_max_tokens) for v, _ in rows}
+        self.assertLessEqual({("copy", 1), ("cpu", 1), ("auto", 1), ("auto", 8), ("auto", 16)}, tried)