prain: fix vLLM on ROCm (no CUDA capability families), smoke test and benchmark scripts
This commit is contained in:
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
# Go/no-go: does this vLLM build run Gemma 4 on this GPU? Serves a 2-layer copy of MODEL_DIR with
|
||||
# random weights (port 8000, log in build/smoke.log), then sends a greedy request, the warm-up
|
||||
# shape (continue_final_message, priority) and the draft fields (vllm_xargs, cache_salt).
|
||||
# usage: scripts/smoke.sh MODEL_DIR [vllm serve args...]
|
||||
# Expected: READY, a JSON list of 3 (gibberish) strings, two {"id":"chatcmpl-... lines, STOPPED.
|
||||
# RDNA2 needs --attention-backend TRITON_ATTN: ROCM_ATTN's prefill kernel wants 160 KiB of LDS.
|
||||
cd "$(dirname "$0")/.." && . scripts/env.sh
|
||||
model=$1
|
||||
shift
|
||||
setsid vllm serve "$model" --served-model-name gemma4 --language-model-only --max-model-len 4096 \
|
||||
--max-num-batched-tokens 1024 --max-num-seqs 2 --kv-cache-memory-bytes 805306368 --port 8000 \
|
||||
--enforce-eager --moe-backend triton --scheduling-policy priority --load-format dummy \
|
||||
--hf-overrides '{"text_config": {"num_hidden_layers": 2, "layer_types": ["sliding_attention", "full_attention"]}}' \
|
||||
"$@" > build/smoke.log 2>&1 < /dev/null &
|
||||
server=$!
|
||||
for _ in $(seq 120); do
|
||||
curl -sf localhost:8000/health > /dev/null && break
|
||||
kill -0 $server 2> /dev/null || break
|
||||
sleep 5
|
||||
done
|
||||
curl -sf localhost:8000/health > /dev/null && echo READY || { echo DIED; tail -40 build/smoke.log; exit 1; }
|
||||
python scripts/greedy.py http://127.0.0.1:8000 gemma4 | head -c 200; echo
|
||||
post() { curl -s localhost:8000/v1/chat/completions -H 'Content-Type: application/json' -d "$1" | head -c 60; echo; }
|
||||
post '{"model": "gemma4", "messages": [{"role": "user", "content": "Why is the sky"}], "max_tokens": 1, "add_generation_prompt": false, "continue_final_message": true, "priority": 1}'
|
||||
post '{"model": "gemma4", "messages": [{"role": "user", "content": "Hi"}], "max_tokens": 2, "vllm_xargs": {"prain": "draft"}, "cache_salt": "prain-draft"}'
|
||||
kill -INT -$server
|
||||
while curl -sf localhost:8000/health > /dev/null; do sleep 2; done
|
||||
echo STOPPED
|
||||
Reference in New Issue
Block a user