prain: web UI with a model monitor
A clear chat page (markdown, code blocks with copy, stop, new chat, kept in the browser, drafts shown as such) and a monitor sidebar, a sheet on phones: tokens/s, a live map of the 30 x 128 experts (in VRAM, used now, copied in), VRAM hit rate, copies, context, prompt cache, GPU VRAM/load/temperature/power/fan. GET /status merges vLLM /metrics, amdgpu sysfs and the stats file, which now lists the experts in VRAM. /warm answers 202 when a warm-up starts. README: the int4 kernels, their A/B, 2048-token chunks as a trade, prompt logprobs limit.
This commit is contained in:
@@ -24,8 +24,8 @@ quant = "int4" # then quantized once into an int4 pack
|
||||
# bf16 | int8 (5.69 MiB/expert) | int4 (3.01 MiB)
|
||||
default_tier = "disk" # runs on any RAM size; `prain tune` does better
|
||||
cache_slots = 15 # 30 x 15 x 3.01 MiB = 1.32 GiB of VRAM
|
||||
stats_path = "../models/packs/gemma4-26b-a4b/stats.json"
|
||||
stats_every = 64 # MoE forward passes between stats/trace writes
|
||||
stats_path = "../models/packs/gemma4-26b-a4b/stats.json" # prain plan and the web monitor read it
|
||||
stats_every = 8 # MoE forward passes between stats/trace writes (~3 ms each)
|
||||
trace_path = "../models/packs/gemma4-26b-a4b/trace.jsonl.gz" # routing trace for sim/tune
|
||||
|
||||
[policy] # which expert a cache slot gives up
|
||||
|
||||
Reference in New Issue
Block a user