diff --git a/README.md b/README.md index 50ad8d0..5759a5e 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,11 @@ it: `prain serve PLACEMENT MODEL_DIR -- VLLM_FLAGS` validates the placement file and starts `vllm serve` with prain's plugin. prain adds `--enforce-eager --moe-backend triton ---scheduling-policy priority`, and it takes `--max-num-seqs` and `--kv-cache-memory-bytes` from -the file's `[serve]` table (it refuses them after `--`). The flags in `$FLAGS` above: +--scheduling-policy priority --chat-template-content-format string`, and it takes +`--max-num-seqs` and `--kv-cache-memory-bytes` from the file's `[serve]` table (it refuses them +after `--`). vLLM would otherwise pick the `openai` content format for Gemma 4, whose template +then prints a system prompt as a Python list (`[{'type': 'text', 'text': '…\n…'}]`, newlines +escaped); a flag after `--` still overrides it. The flags in `$FLAGS` above: | flag | why | |---|---| diff --git a/prain/cli.py b/prain/cli.py index 8a9b891..1cc75b8 100644 --- a/prain/cli.py +++ b/prain/cli.py @@ -113,6 +113,9 @@ def serve(args): "--scheduling-policy", "priority", "--max-num-seqs", str(config.serve.max_requests), "--kv-cache-memory-bytes", str(int(config.serve.kv_cache_gib * GIB)), + # vLLM detects "openai" for Gemma 4, whose template then prints a system + # prompt as a Python list; a flag in `extra` comes later and wins + "--chat-template-content-format", "string", *extra]) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9285504..e717793 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -55,7 +55,8 @@ class Cli(TmpCase): execvp.assert_called_once_with("vllm", [ "vllm", "serve", "/models/m", "--enforce-eager", "--moe-backend", "triton", "--scheduling-policy", "priority", "--max-num-seqs", "3", - "--kv-cache-memory-bytes", "1073741824", "--port", "8001"]) + "--kv-cache-memory-bytes", "1073741824", "--chat-template-content-format", "string", + "--port", "8001"]) with mock.patch("os.execvp") as execvp, self.assertRaisesRegex( SystemExit, "--max-num-seqs=4 comes from the placement file"): main(["serve", config, "/models/m", "--", "--max-num-seqs=4"])