prain: RX 6600 XT example config

2026-10-27 13:52:26 +01:00
parent 1cf3c18a82
commit a49ff1f3b2
1 changed files with 59 additions and 0 deletions
@@ -0,0 +1,59 @@
+# prain placement: prithivMLmods/gemma-4-26B-A4B-it-Uncensored-MAX on an 8 GB RX 6600 XT
+# Model: 30 MoE layers x 128 experts, top-8 routing, 11.34 MiB per expert (BF16).
+# VRAM: 8 GiB - 4.45 GiB dense weights - 0.75 GiB KV cache - ~0.9 GiB runtime = ~1.7 GiB experts.
+#
+# Tiers:  vram = pinned in a VRAM slot for the whole run
+#         ram  = pinned host memory, copied into a cache slot when a token needs it
+#         disk = read from the pack file in pack_dir when a token needs it
+#                (the kernel page cache still caches it; nothing is pinned)
+# [layers."<selector>"] sections apply in file order; a later section wins per expert.
+# Layer selectors: "*", "12", "0-9,20".   Expert lists: "*", "0-15,32", [1, 2, 3].
+# Check what a file costs:   prain check <this file> <model config.json>
+# Compare variants offline:  prain sim <trace> <this file> <model config.json>
+#                            prain tune <trace> <model config.json> --expert-vram-gib 1.7 ...
+# Relative paths start at this file's directory; "~" works too.
+
+pack_dir = "../models/packs/gemma4-26b-a4b"  # 42.5 GiB, written on the first run
+default_tier = "disk"                    # runs on any RAM size; `prain tune` does better
+cache_slots = 5                          # 30 x 5 x 11.34 MiB = 1.66 GiB of VRAM
+stats_path = "../models/packs/gemma4-26b-a4b/stats.json"
+stats_every = 64                         # MoE forward passes between stats/trace writes
+trace_path = "../models/packs/gemma4-26b-a4b/trace.jsonl.gz"  # routing trace for sim/tune
+
+[policy]              # which expert a cache slot gives up
+eviction = "lfu"      # lru | lfu (decayed frequency) | gdsf (frequency x refetch cost)
+heat = "weight"       # what makes an expert hot: count (tokens) | weight (router mass) |
+                      # recency (router mass, later tokens weigh more: good for warm-ups)
+lfu_half_life = 64    # forwards until an LFU score halves
+disk_cost = 4.0       # gdsf: a disk expert costs this many RAM experts to refetch
+recency_half_life = 32  # heat = "recency": tokens until a token's weight halves
+scan_tokens = 64      # a forward with this many tokens (a prompt) is a scan:
+scan_keep = 0         #   it may not evict the scan_keep most valuable cached experts,
+promote = 2           #   and only its `promote` hottest experts get full priority
+
+[prefetch]            # copy experts before their layer runs, on a side stream
+mode = "none"         # none | pregate (run layer L+distance's router on layer L's input)
+distance = 1          # 1 or 2 layers ahead
+budget = 2            # experts per layer and forward
+min_prob = 0.05       # skip guesses below this router probability
+
+[miss]                # an expert that is not in VRAM
+mode = "copy"         # copy | cpu (always compute it on the CPU) | auto (CPU if it has
+cpu_max_tokens = 1    #   at most cpu_max_tokens tokens, else copy)
+cpu_threads = 4       # CPU threads of the vLLM worker (sets OMP_NUM_THREADS)
+admit = 1             # cpu/auto: copy this many CPU-run experts into VRAM afterwards
+
+[serve]               # parallel requests (`prain serve` passes these to vLLM)
+max_requests = 2      # requests vLLM runs at once (--max-num-seqs); warm-ups count too
+kv_cache_gib = 0.75   # KV cache VRAM shared by them (--kv-cache-memory-bytes). vLLM logs
+                      #   "Maximum concurrency for 4096 tokens per request: N.NNx" at start;
+                      #   below max_requests, long requests wait or get preempted. Every GiB
+                      #   here is 90 expert cache slots less.
+
+[draft]               # answers that arrive while others use the GPU (`prain web --config`)
+full_requests = 1     # answers that run with all their experts; later ones run at once as
+                      #   drafts on the experts already in VRAM (= max_requests: never draft)
+min_weight = 0.5      # a draft token still copies its heaviest missing experts until this
+                      #   share of its router weight is computed (0 = only what is in VRAM)
+refine = "regenerate" # regenerate: once a full slot frees, the draft is cut short and the
+                      #   answer generated again with all experts | none: keep the draft