prain: int8/int4 experts quantized once on the GPU, rounds through vLLM's expert_map, [miss] min_weight skips experts not in VRAM; scripts: tune_moe, quality (answer NLL on identical token ids), bench --context, measure --quant

2026-10-28 22:07:25 +01:00
parent 3ae9bd73a7
commit 708e152549
15 changed files with 703 additions and 120 deletions
@@ -76,3 +76,17 @@ class Cli(TmpCase):
        self.assertEqual((tuned.pack_dir, tuned.trace_path), ("/packs", trace))
        self.assertLessEqual(sum(tuned.placement(layer, 128).cache_slots for layer in range(3)),
                             int(0.2 * 1024 / 11.34) // 3 * 3)
+
+    def test_quantized_experts_check_and_tune(self):
+        hf = self.write("config.json", json.dumps(GEMMA4))
+        out, _ = self.run_cli("check", self.write("q.toml", 'pack_dir = "/packs"\nquant = "int4"\n'), hf)
+        self.assertIn("expert size 3.01 MiB\n", out)
+        trace = os.path.join(self.dir, "trace.jsonl.gz")
+        self.run_cli("synth", trace, "--forwards", "40", "--layers", "3")
+        out, _ = self.run_cli("tune", trace, hf, "--quant", "int4", "--expert-vram-gib", "0.2",
+                              "--ram-gib", "4", "--pack-dir", "/packs", "--top", "3")
+        tuned = load_config(self.write("tuned.toml", out))
+        self.assertEqual(tuned.quant, "int4")
+        slots = sum(tuned.placement(layer, 128).cache_slots for layer in range(3))
+        self.assertGreater(slots, int(0.2 * 1024 / 11.34))  # int4 experts are ~4x smaller
+        self.assertLessEqual(slots, int(0.2 * 1024 / 3.01))