30 lines
1.9 KiB
Bash
Executable File
30 lines
1.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# Go/no-go: does this vLLM build run Gemma 4 on this GPU? Serves a 2-layer copy of MODEL_DIR with
|
|
# random weights (port 8000, log in build/smoke.log), then sends a greedy request, the warm-up
|
|
# shape (continue_final_message, priority) and the draft fields (vllm_xargs, cache_salt).
|
|
# usage: scripts/smoke.sh MODEL_DIR [vllm serve args...]
|
|
# Expected: READY, a JSON list of 3 (gibberish) strings, two {"id":"chatcmpl-... lines, STOPPED.
|
|
# RDNA2 needs --attention-backend TRITON_ATTN: ROCM_ATTN's prefill kernel wants 160 KiB of LDS.
|
|
cd "$(dirname "$0")/.." && . scripts/env.sh
|
|
model=$1
|
|
shift
|
|
setsid vllm serve "$model" --served-model-name gemma4 --language-model-only --max-model-len 4096 \
|
|
--max-num-batched-tokens 1024 --max-num-seqs 2 --kv-cache-memory-bytes 805306368 --port 8000 \
|
|
--enforce-eager --moe-backend triton --scheduling-policy priority --load-format dummy \
|
|
--hf-overrides '{"text_config": {"num_hidden_layers": 2, "layer_types": ["sliding_attention", "full_attention"]}}' \
|
|
"$@" > build/smoke.log 2>&1 < /dev/null &
|
|
server=$!
|
|
for _ in $(seq 120); do
|
|
curl -sf localhost:8000/health > /dev/null && break
|
|
kill -0 $server 2> /dev/null || break
|
|
sleep 5
|
|
done
|
|
curl -sf localhost:8000/health > /dev/null && echo READY || { echo DIED; tail -40 build/smoke.log; exit 1; }
|
|
python scripts/greedy.py http://127.0.0.1:8000 gemma4 | head -c 200; echo
|
|
post() { curl -s localhost:8000/v1/chat/completions -H 'Content-Type: application/json' -d "$1" | head -c 60; echo; }
|
|
post '{"model": "gemma4", "messages": [{"role": "user", "content": "Why is the sky"}], "max_tokens": 1, "add_generation_prompt": false, "continue_final_message": true, "priority": 1}'
|
|
post '{"model": "gemma4", "messages": [{"role": "user", "content": "Hi"}], "max_tokens": 2, "vllm_xargs": {"prain": "draft"}, "cache_salt": "prain-draft"}'
|
|
kill -INT -$server
|
|
while curl -sf localhost:8000/health > /dev/null; do sleep 2; done
|
|
echo STOPPED
|