64 lines
4.5 KiB
TOML
64 lines
4.5 KiB
TOML
# prain placement: prithivMLmods/gemma-4-26B-A4B-it-Uncensored-MAX on an 8 GB RX 6600 XT
|
|
# Model: 30 MoE layers x 128 experts, top-8 routing, 11.34 MiB per expert (BF16).
|
|
# VRAM, measured: 6.22 GiB weights (4.46 dense + 5 slots x 30 layers) + 0.75 GiB KV cache: 7.9 of
|
|
# 8 GiB in use. Serve with --attention-backend TRITON_ATTN (ROCM_ATTN needs more LDS than RDNA2 has).
|
|
# RAM tier: pinning is very slow behind a strict IOMMU (Thunderbolt eGPU: minutes per GiB), so this
|
|
# file keeps no RAM tier; the kernel page cache holds the hot part of the pack instead.
|
|
#
|
|
# Tiers: vram = pinned in a VRAM slot for the whole run
|
|
# ram = pinned host memory, copied into a cache slot when a token needs it
|
|
# disk = read from the pack file in pack_dir when a token needs it
|
|
# (the kernel page cache still caches it; nothing is pinned)
|
|
# [layers."<selector>"] sections apply in file order; a later section wins per expert.
|
|
# Layer selectors: "*", "12", "0-9,20". Expert lists: "*", "0-15,32", [1, 2, 3].
|
|
# Check what a file costs: prain check <this file> <model config.json>
|
|
# Compare variants offline: prain sim <trace> <this file> <model config.json>
|
|
# prain tune <trace> <model config.json> --expert-vram-gib 1.7 ...
|
|
# Relative paths start at this file's directory; "~" works too.
|
|
|
|
pack_dir = "../models/packs/gemma4-26b-a4b" # 42.5 GiB, written on the first run
|
|
default_tier = "disk" # runs on any RAM size; `prain tune` does better
|
|
cache_slots = 5 # 30 x 5 x 11.34 MiB = 1.66 GiB of VRAM
|
|
stats_path = "../models/packs/gemma4-26b-a4b/stats.json"
|
|
stats_every = 64 # MoE forward passes between stats/trace writes
|
|
trace_path = "../models/packs/gemma4-26b-a4b/trace.jsonl.gz" # routing trace for sim/tune
|
|
|
|
[policy] # which expert a cache slot gives up
|
|
eviction = "lfu" # lru | lfu (decayed frequency) | gdsf (frequency x refetch cost)
|
|
heat = "weight" # what makes an expert hot: count (tokens) | weight (router mass) |
|
|
# recency (router mass, later tokens weigh more: good for warm-ups)
|
|
lfu_half_life = 64 # forwards until an LFU score halves
|
|
disk_cost = 4.0 # gdsf: a disk expert costs this many RAM experts to refetch
|
|
recency_half_life = 32 # heat = "recency": tokens until a token's weight halves
|
|
scan_tokens = 64 # a forward with this many tokens (a prompt) is a scan:
|
|
scan_keep = 0 # it may not evict the scan_keep most valuable cached experts,
|
|
promote = 2 # and only its `promote` hottest experts get full priority
|
|
|
|
[prefetch] # copy experts before their layer runs, on a side stream
|
|
mode = "none" # none | pregate (run layer L+distance's router on layer L's input)
|
|
distance = 1 # 1 or 2 layers ahead
|
|
budget = 2 # experts per layer and forward
|
|
min_prob = 0.05 # skip guesses below this router probability
|
|
|
|
[miss] # an expert that is not in VRAM
|
|
mode = "auto" # copy | cpu (always compute it on the CPU) | auto (CPU if it has
|
|
cpu_max_tokens = 2 # at most cpu_max_tokens tokens, else copy). Measured on the Thunderbolt
|
|
# eGPU host: a copy costs ~4.6 ms + disk read, the CPU ~2.5 ms per token
|
|
cpu_threads = 16 # CPU threads of the vLLM worker (sets OMP_NUM_THREADS)
|
|
admit = 0 # cpu/auto: copy this many CPU-run RAM-tier experts into VRAM afterwards
|
|
|
|
[serve] # parallel requests (`prain serve` passes these to vLLM)
|
|
max_requests = 2 # requests vLLM runs at once (--max-num-seqs); warm-ups count too
|
|
kv_cache_gib = 0.75 # KV cache VRAM shared by them (--kv-cache-memory-bytes). vLLM logs
|
|
# "Maximum concurrency for 4096 tokens per request: N.NNx" at start;
|
|
# below max_requests, long requests wait or get preempted. Every GiB
|
|
# here is 90 expert cache slots less.
|
|
|
|
[draft] # answers that arrive while others use the GPU (`prain web --config`)
|
|
full_requests = 1 # answers that run with all their experts; later ones run at once as
|
|
# drafts on the experts already in VRAM (= max_requests: never draft)
|
|
min_weight = 0.5 # a draft token still copies its heaviest missing experts until this
|
|
# share of its router weight is computed (0 = only what is in VRAM)
|
|
refine = "regenerate" # regenerate: once a full slot frees, the draft is cut short and the
|
|
# answer generated again with all experts | none: keep the draft
|