fabolous005
a67238f66b
prain web/tui --persona FILE sends the file as the system message ahead of every warm-up and answer; the page shows its name. Three examples in examples/personas (task, how to answer, environment, how to use it). A persona costs 2.0-3.2 s once (241-467 tokens), then at most 0.3 s per conversation from the prefix cache. README rewritten around launching: quick start, the vLLM flags, a quantization table measured today at equal expert VRAM (bf16 0.90 / int8 2.19 / int4 5.19 tok/s; 2.3k-token prompt 69.5 / 32.3 / 20.5 s; 4 of 6 int8 and 0 of 6 int4 answers identical to bf16, one int4 answer wrong), how to switch formats, the VRAM budget, writing personas, the chats and the monitor.
77 lines
5.7 KiB
TOML
77 lines
5.7 KiB
TOML
# prain placement: prithivMLmods/gemma-4-26B-A4B-it-Uncensored-MAX on an 8 GB RX 6600 XT, set up
|
|
# for a coding agent: int4 experts, 16k context, tool calls.
|
|
# Model: 30 MoE layers x 128 experts, top-8 routing, 3.01 MiB per int4 expert (11.34 MiB bf16).
|
|
# VRAM: 4.46 GiB dense weights + 30 x 15 x 3.01 MiB = 1.32 GiB of expert slots + 1.0 GiB KV cache.
|
|
# Serve with --max-model-len 16384 --attention-backend TRITON_ATTN (ROCM_ATTN needs more LDS than
|
|
# RDNA2 has) --enable-auto-tool-choice --tool-call-parser gemma4.
|
|
# RAM tier: pinning host memory took minutes per GiB on the Thunderbolt eGPU host (cause unproven)
|
|
# and pinned copies were only 4% faster than pageable ones, so this file keeps no RAM tier; the
|
|
# kernel page cache holds the pack instead (the int4 pack, 11.3 GiB, fits in free RAM).
|
|
#
|
|
# Tiers: vram = pinned in a VRAM slot for the whole run
|
|
# ram = pinned host memory, copied into a cache slot when a token needs it
|
|
# disk = read from the pack file in pack_dir when a token needs it
|
|
# (the kernel page cache still caches it; nothing is pinned)
|
|
# [layers."<selector>"] sections apply in file order; a later section wins per expert.
|
|
# Layer selectors: "*", "12", "0-9,20". Expert lists: "*", "0-15,32", [1, 2, 3].
|
|
# Check what a file costs: prain check <this file> <model config.json>
|
|
# Compare variants offline: prain sim <trace> <this file> <model config.json>
|
|
# prain tune <trace> <model config.json> --quant int4 --expert-vram-gib 1.4 ...
|
|
# Relative paths start at this file's directory; "~" works too.
|
|
|
|
pack_dir = "../models/packs/gemma4-26b-a4b" # bf16 pack (42.5 GiB) written on the first run,
|
|
quant = "int4" # then quantized once into an int4 pack (11.3 GiB):
|
|
# bf16 | int8 (5.69 MiB/expert) | int4 (3.01 MiB)
|
|
default_tier = "disk" # runs on any RAM size; `prain tune` does better
|
|
cache_slots = 15 # 30 x 15 x 3.01 MiB = 1.32 GiB of VRAM. The same VRAM
|
|
# holds 8 int8 or 4 bf16 experts per layer: change
|
|
# quant and cache_slots together
|
|
stats_path = "../models/packs/gemma4-26b-a4b/stats.json" # prain plan and the web monitor read it
|
|
stats_every = 8 # MoE forward passes between stats/trace writes (~3 ms each)
|
|
trace_path = "../models/packs/gemma4-26b-a4b/trace.jsonl.gz" # routing trace for sim/tune
|
|
|
|
[policy] # which expert a cache slot gives up
|
|
eviction = "gdsf" # lru | lfu (decayed frequency) | gdsf (frequency x refetch cost)
|
|
heat = "count" # what makes an expert hot: count (tokens) | weight (router mass) |
|
|
# recency (router mass, later tokens weigh more: good for warm-ups)
|
|
lfu_half_life = 64 # forwards until an LFU score halves
|
|
disk_cost = 4.0 # gdsf: a disk expert costs this many RAM experts to refetch
|
|
recency_half_life = 32 # heat = "recency": tokens until a token's weight halves
|
|
scan_tokens = 64 # a forward with this many tokens (a prompt) is a scan:
|
|
scan_keep = 0 # it may not evict the scan_keep most valuable cached experts,
|
|
promote = 2 # and only its `promote` hottest experts get full priority
|
|
|
|
[prefetch] # copy experts before their layer runs, on a side stream
|
|
mode = "none" # none | pregate (run layer L+distance's router on layer L's input)
|
|
distance = 1 # 1 or 2 layers ahead
|
|
budget = 2 # experts per layer and forward
|
|
min_prob = 0.05 # skip guesses below this router probability
|
|
|
|
[miss] # an expert that is not in VRAM
|
|
mode = "copy" # copy | cpu (always compute it on the CPU) | auto (CPU if it has
|
|
cpu_max_tokens = 1 # at most cpu_max_tokens tokens, else copy). Measured on the Thunderbolt
|
|
# eGPU host, per int4 expert: a copy ~1.3 ms + disk read, the CPU
|
|
# 0.8-1.3 ms per token. `prain tune` picked copy for int4 (a copied
|
|
# expert stays for later tokens) and auto <= 2 tokens for bf16 (copy 4.8 ms)
|
|
cpu_threads = 16 # CPU threads of the vLLM worker (sets OMP_NUM_THREADS)
|
|
admit = 0 # cpu/auto: copy this many CPU-run RAM-tier experts into VRAM afterwards
|
|
min_weight = 1.0 # share of each token's router weight that must be computed. Below 1 a token
|
|
# skips experts not in VRAM once it has that share: faster, dumber.
|
|
# Measured here: 0.6 decodes 14% faster (answer NLL 0.11 -> 0.19), 0.4
|
|
# 21% (0.40, visible slips)
|
|
|
|
[serve] # parallel requests (`prain serve` passes these to vLLM)
|
|
max_requests = 2 # requests vLLM runs at once (--max-num-seqs); warm-ups count too
|
|
kv_cache_gib = 1.0 # KV cache VRAM shared by them (--kv-cache-memory-bytes). vLLM logs
|
|
# "Maximum concurrency for 16384 tokens per request: 1.11x" with 1.0 GiB.
|
|
# Gemma 4 needs ~20 KiB per token of context plus ~0.6 GiB for its
|
|
# sliding-window layers; every GiB here is 340 int4 expert slots less.
|
|
|
|
[draft] # answers that arrive while others use the GPU (`prain web --config`)
|
|
full_requests = 1 # answers that run with all their experts; later ones run at once as
|
|
# drafts on the experts already in VRAM (= max_requests: never draft)
|
|
min_weight = 0.5 # a draft token still copies its heaviest missing experts until this
|
|
# share of its router weight is computed (0 = only what is in VRAM)
|
|
refine = "regenerate" # regenerate: once a full slot frees, the draft is cut short and the
|
|
# answer generated again with all experts | none: keep the draft
|