diff --git a/examples/gemma4-26b-a4b-rx6600xt.toml b/examples/gemma4-26b-a4b-rx6600xt.toml new file mode 100644 index 0000000..33f1c4e --- /dev/null +++ b/examples/gemma4-26b-a4b-rx6600xt.toml @@ -0,0 +1,59 @@ +# prain placement: prithivMLmods/gemma-4-26B-A4B-it-Uncensored-MAX on an 8 GB RX 6600 XT +# Model: 30 MoE layers x 128 experts, top-8 routing, 11.34 MiB per expert (BF16). +# VRAM: 8 GiB - 4.45 GiB dense weights - 0.75 GiB KV cache - ~0.9 GiB runtime = ~1.7 GiB experts. +# +# Tiers: vram = pinned in a VRAM slot for the whole run +# ram = pinned host memory, copied into a cache slot when a token needs it +# disk = read from the pack file in pack_dir when a token needs it +# (the kernel page cache still caches it; nothing is pinned) +# [layers.""] sections apply in file order; a later section wins per expert. +# Layer selectors: "*", "12", "0-9,20". Expert lists: "*", "0-15,32", [1, 2, 3]. +# Check what a file costs: prain check +# Compare variants offline: prain sim +# prain tune --expert-vram-gib 1.7 ... +# Relative paths start at this file's directory; "~" works too. + +pack_dir = "../models/packs/gemma4-26b-a4b" # 42.5 GiB, written on the first run +default_tier = "disk" # runs on any RAM size; `prain tune` does better +cache_slots = 5 # 30 x 5 x 11.34 MiB = 1.66 GiB of VRAM +stats_path = "../models/packs/gemma4-26b-a4b/stats.json" +stats_every = 64 # MoE forward passes between stats/trace writes +trace_path = "../models/packs/gemma4-26b-a4b/trace.jsonl.gz" # routing trace for sim/tune + +[policy] # which expert a cache slot gives up +eviction = "lfu" # lru | lfu (decayed frequency) | gdsf (frequency x refetch cost) +heat = "weight" # what makes an expert hot: count (tokens) | weight (router mass) | + # recency (router mass, later tokens weigh more: good for warm-ups) +lfu_half_life = 64 # forwards until an LFU score halves +disk_cost = 4.0 # gdsf: a disk expert costs this many RAM experts to refetch +recency_half_life = 32 # heat = "recency": tokens until a token's weight halves +scan_tokens = 64 # a forward with this many tokens (a prompt) is a scan: +scan_keep = 0 # it may not evict the scan_keep most valuable cached experts, +promote = 2 # and only its `promote` hottest experts get full priority + +[prefetch] # copy experts before their layer runs, on a side stream +mode = "none" # none | pregate (run layer L+distance's router on layer L's input) +distance = 1 # 1 or 2 layers ahead +budget = 2 # experts per layer and forward +min_prob = 0.05 # skip guesses below this router probability + +[miss] # an expert that is not in VRAM +mode = "copy" # copy | cpu (always compute it on the CPU) | auto (CPU if it has +cpu_max_tokens = 1 # at most cpu_max_tokens tokens, else copy) +cpu_threads = 4 # CPU threads of the vLLM worker (sets OMP_NUM_THREADS) +admit = 1 # cpu/auto: copy this many CPU-run experts into VRAM afterwards + +[serve] # parallel requests (`prain serve` passes these to vLLM) +max_requests = 2 # requests vLLM runs at once (--max-num-seqs); warm-ups count too +kv_cache_gib = 0.75 # KV cache VRAM shared by them (--kv-cache-memory-bytes). vLLM logs + # "Maximum concurrency for 4096 tokens per request: N.NNx" at start; + # below max_requests, long requests wait or get preempted. Every GiB + # here is 90 expert cache slots less. + +[draft] # answers that arrive while others use the GPU (`prain web --config`) +full_requests = 1 # answers that run with all their experts; later ones run at once as + # drafts on the experts already in VRAM (= max_requests: never draft) +min_weight = 0.5 # a draft token still copies its heaviest missing experts until this + # share of its router weight is computed (0 = only what is in VRAM) +refine = "regenerate" # regenerate: once a full slot frees, the draft is cut short and the + # answer generated again with all experts | none: keep the draft