fabolous005/prain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

prain: README: expert formats, dense-layer fix, min_weight trade-off, 16k agent setup; example config int4, 16k, tools

Browse Source
This commit is contained in:
fabolous005
2026-10-28 22:07:25 +01:00
parent 216cd76a77
commit bb42ea3802
2 changed files with 126 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/gemma4-26b-a4b-rx6600xt.toml
+29 -18
View File
@@ -1,9 +1,12 @@
# prain placement: prithivMLmods/gemma-4-26B-A4B-it-Uncensored-MAX on an 8 GB RX 6600 XT
# Model: 30 MoE layers x 128 experts, top-8 routing, 11.34 MiB per expert (BF16).
# VRAM, measured: 6.22 GiB weights (4.46 dense + 5 slots x 30 layers) + 0.75 GiB KV cache: 7.9 of
# 8 GiB in use. Serve with --attention-backend TRITON_ATTN (ROCM_ATTN needs more LDS than RDNA2 has).
# RAM tier: pinning is very slow behind a strict IOMMU (Thunderbolt eGPU: minutes per GiB), so this
# file keeps no RAM tier; the kernel page cache holds the hot part of the pack instead.
# prain placement: prithivMLmods/gemma-4-26B-A4B-it-Uncensored-MAX on an 8 GB RX 6600 XT, set up
# for a coding agent: int4 experts, 16k context, tool calls.
# Model: 30 MoE layers x 128 experts, top-8 routing, 3.01 MiB per int4 expert (11.34 MiB bf16).
# VRAM: 4.46 GiB dense weights + 30 x 15 x 3.01 MiB = 1.32 GiB of expert slots + 1.0 GiB KV cache.
# Serve with --max-model-len 16384 --attention-backend TRITON_ATTN (ROCM_ATTN needs more LDS than
# RDNA2 has) --enable-auto-tool-choice --tool-call-parser gemma4.
# RAM tier: pinning host memory took minutes per GiB on the Thunderbolt eGPU host (cause unproven)
# and pinned copies were only 4% faster than pageable ones, so this file keeps no RAM tier; the
# kernel page cache holds the pack instead (the int4 pack, 11.3 GiB, fits in free RAM).
#
# Tiers: vram = pinned in a VRAM slot for the whole run
# ram = pinned host memory, copied into a cache slot when a token needs it
@@ -13,19 +16,21 @@
# Layer selectors: "*", "12", "0-9,20". Expert lists: "*", "0-15,32", [1, 2, 3].
# Check what a file costs: prain check <this file> <model config.json>
# Compare variants offline: prain sim <trace> <this file> <model config.json>
# prain tune <trace> <model config.json> --expert-vram-gib 1.7 ...
# prain tune <trace> <model config.json> --quant int4 --expert-vram-gib 1.4 ...
# Relative paths start at this file's directory; "~" works too.
pack_dir = "../models/packs/gemma4-26b-a4b" # 42.5 GiB, written on the first run
pack_dir = "../models/packs/gemma4-26b-a4b" # bf16 pack (42.5 GiB) written on the first run,
quant = "int4" # then quantized once into an int4 pack (11.3 GiB):
# bf16 | int8 (5.69 MiB/expert) | int4 (3.01 MiB)
default_tier = "disk" # runs on any RAM size; `prain tune` does better
cache_slots = 5 # 30 x 5 x 11.34 MiB = 1.66 GiB of VRAM
cache_slots = 15 # 30 x 15 x 3.01 MiB = 1.32 GiB of VRAM
stats_path = "../models/packs/gemma4-26b-a4b/stats.json"
stats_every = 64 # MoE forward passes between stats/trace writes
trace_path = "../models/packs/gemma4-26b-a4b/trace.jsonl.gz" # routing trace for sim/tune
[policy] # which expert a cache slot gives up
eviction = "lfu" # lru | lfu (decayed frequency) | gdsf (frequency x refetch cost)
heat = "weight" # what makes an expert hot: count (tokens) | weight (router mass) |
eviction = "gdsf" # lru | lfu (decayed frequency) | gdsf (frequency x refetch cost)
heat = "count" # what makes an expert hot: count (tokens) | weight (router mass) |
# recency (router mass, later tokens weigh more: good for warm-ups)
lfu_half_life = 64 # forwards until an LFU score halves
disk_cost = 4.0 # gdsf: a disk expert costs this many RAM experts to refetch
@@ -41,18 +46,24 @@ budget = 2 # experts per layer and forward
min_prob = 0.05 # skip guesses below this router probability
[miss] # an expert that is not in VRAM
mode = "auto" # copy | cpu (always compute it on the CPU) | auto (CPU if it has
cpu_max_tokens = 2 # at most cpu_max_tokens tokens, else copy). Measured on the Thunderbolt
# eGPU host: a copy costs ~4.6 ms + disk read, the CPU ~2.5 ms per token
mode = "copy" # copy | cpu (always compute it on the CPU) | auto (CPU if it has
cpu_max_tokens = 1 # at most cpu_max_tokens tokens, else copy). Measured on the Thunderbolt
# eGPU host, per int4 expert: a copy ~1.3 ms + disk read, the CPU
# 0.8-1.3 ms per token. `prain tune` picked copy for int4 (a copied
# expert stays for later tokens) and auto <= 2 tokens for bf16 (copy 4.8 ms)
cpu_threads = 16 # CPU threads of the vLLM worker (sets OMP_NUM_THREADS)
admit = 0 # cpu/auto: copy this many CPU-run RAM-tier experts into VRAM afterwards
min_weight = 1.0 # share of each token's router weight that must be computed. Below 1 a token
# skips experts not in VRAM once it has that share: faster, dumber.
# Measured here: 0.6 decodes 14% faster (answer NLL 0.11 -> 0.19), 0.4
# 21% (0.40, visible slips)
[serve] # parallel requests (`prain serve` passes these to vLLM)
max_requests = 2 # requests vLLM runs at once (--max-num-seqs); warm-ups count too
kv_cache_gib = 0.75 # KV cache VRAM shared by them (--kv-cache-memory-bytes). vLLM logs
# "Maximum concurrency for 4096 tokens per request: N.NNx" at start;
# below max_requests, long requests wait or get preempted. Every GiB
# here is 90 expert cache slots less.
kv_cache_gib = 1.0 # KV cache VRAM shared by them (--kv-cache-memory-bytes). vLLM logs
# "Maximum concurrency for 16384 tokens per request: 1.11x" with 1.0 GiB.
# Gemma 4 needs ~20 KiB per token of context plus ~0.6 GiB for its
# sliding-window layers; every GiB here is 340 int4 expert slots less.
[draft] # answers that arrive while others use the GPU (`prain web --config`)
full_requests = 1 # answers that run with all their experts; later ones run at once as