fabolous005/prain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

prain: web UI with a model monitor

Browse Source 
A clear chat page (markdown, code blocks with copy, stop, new chat, kept in the browser,
drafts shown as such) and a monitor sidebar, a sheet on phones: tokens/s, a live map of the
30 x 128 experts (in VRAM, used now, copied in), VRAM hit rate, copies, context, prompt cache,
GPU VRAM/load/temperature/power/fan. GET /status merges vLLM /metrics, amdgpu sysfs and the
stats file, which now lists the experts in VRAM. /warm answers 202 when a warm-up starts.
README: the int4 kernels, their A/B, 2048-token chunks as a trade, prompt logprobs limit.
This commit is contained in:
fabolous005
2026-10-29 14:20:23 +01:00
parent d1ccc5c294
commit a14e3ade0b
11 changed files with 977 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/helpers.py
+3
View File
@@ -41,10 +41,13 @@ class StubVllm:
self.delay = 0.0
self.pause = 0.0
self.fail = None
self.metrics = None # Prometheus text served at GET /metrics
stub = self
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/metrics" and stub.metrics is not None:
return self.send(200, "text/plain", stub.metrics)
self.send(200, "application/json", {"data": [{"id": "stub-model"}]})
def do_POST(self):
tests/test_policy.py
+4
View File
@@ -119,3 +119,7 @@ class Prefetch(TmpCase):
stats = json.load(f)["layers"]["7"]
self.assertEqual((stats["uses"], stats["prefetch_wasted"], stats["misses"]),
([1, 1, 1, 0], 1, 2))
self.assertEqual(stats["vram"], [1, 2]) # the experts in VRAM now, for the web monitor
write_stats(path, {7: SlotCache([5, 3], 8, 2, Policy(eviction="lfu"))})
with open(path) as f:
self.assertEqual(json.load(f)["layers"]["7"]["vram"], [3, 5]) # pinned ones too
tests/test_web.py
+61 -5
View File
@@ -1,4 +1,5 @@
import json
import os
import threading
import unittest
import urllib.error
@@ -7,7 +8,7 @@ import urllib.request
import prain.chat as chat
from prain.config import Draft, Serve
from prain.web import make_server
from tests.helpers import StubVllm, wait_for
from tests.helpers import StubVllm, TmpCase, wait_for
def events(response):
@@ -38,7 +39,9 @@ class Web(unittest.TestCase):
self.url = f"http://127.0.0.1:{self.server.server_port}"
def warm(self, session, draft):
chat.post(self.url, "/warm", {"session": session, "messages": [], "draft": draft}).read()
with chat.post(self.url, "/warm", {"session": session, "messages": [],
"draft": draft}) as response:
return response.status
def ask(self, session, text):
return chat.post(self.url, "/chat", {"session": session,
@@ -46,7 +49,9 @@ class Web(unittest.TestCase):
def test_page_warm_up_and_chat(self):
with urllib.request.urlopen(self.url) as response:
self.assertIn(b'<textarea id="draft"', response.read())
page = response.read()
self.assertIn(b'<textarea id="draft"', page)
self.assertIn(b'"/status"', page) # the monitor polls it
self.warm("s1", "Why is ")
wait_for(lambda: self.stub.bodies)
self.assertEqual(self.stub.bodies[0]["messages"], [{"role": "user", "content": "Why is"}])
@@ -67,9 +72,9 @@ class Web(unittest.TestCase):
def test_no_warm_up_while_another_answer_streams(self):
self.server.streams[0] = 1 # someone else's answer is streaming
self.warm("bob", "typing along")
self.assertEqual(self.warm("bob", "typing along"), 204) # 204: nothing warms up
self.server.streams[0] = 0
self.warm("bob", "now")
self.assertEqual(self.warm("bob", "now"), 202) # 202: a warm-up starts
wait_for(lambda: self.stub.bodies)
self.assertEqual([b["messages"][-1]["content"] for b in self.stub.bodies], ["now"])
@@ -136,3 +141,54 @@ class Web(unittest.TestCase):
self.assertEqual(events(r), ["Hel", "lo", "!", "<done>"])
thread.join()
self.assertEqual(sum("vllm_xargs" in b for b in self.stub.bodies), 1)
class Status(TmpCase):
def setUp(self):
super().setUp()
self.stub = StubVllm()
self.addCleanup(self.stub.close)
def status(self, **options):
server = make_server(self.stub.url, 0, **options)
threading.Thread(target=server.serve_forever, daemon=True).start()
self.addCleanup(server.server_close)
self.addCleanup(server.shutdown)
with urllib.request.urlopen(f"http://127.0.0.1:{server.server_port}/status") as r:
return json.load(r)
def test_status_reports_vllm_the_gpu_and_the_experts(self):
self.stub.metrics = (
"# HELP vllm:num_requests_running Number of requests running.\n"
'vllm:num_requests_running{engine="0",model_name="m"} 1.0\n'
'vllm:num_requests_waiting{engine="0",model_name="m"} 0.0\n'
'vllm:num_requests_waiting_by_reason{engine="0",reason="capacity"} 5.0\n'
'vllm:kv_cache_usage_perc{engine="0",model_name="m"} 0.25\n'
'vllm:prefix_cache_queries_total{engine="0",model_name="m"} 40.0\n'
'vllm:prefix_cache_hits_total{engine="0",model_name="m"} 30.0\n'
'vllm:prompt_tokens_total{engine="0",model_name="m"} 100.0\n'
'vllm:generation_tokens_total{engine="0",model_name="m"} 7.0\n')
os.makedirs(os.path.join(self.dir, "card0", "device")) # an iGPU: no VRAM files
gpu = os.path.join(self.dir, "card1", "device")
os.makedirs(os.path.join(gpu, "hwmon", "hwmon4"))
for name, value in {"mem_info_vram_total": 8573157376, "mem_info_vram_used": 8000000000,
"gpu_busy_percent": 96, "hwmon/hwmon4/temp1_input": 61000,
"hwmon/hwmon4/power1_average": 120500000,
"hwmon/hwmon4/fan1_input": 1800}.items():
self.write(os.path.join("card1", "device", name), f"{value}\n")
stats = {"layers": {"0": {"uses": [3, 0], "vram": [0], "hits": 2, "misses": 1}}}
stats_path = self.write("stats.json", json.dumps(stats))
got = self.status(stats_path=stats_path, sysfs=self.dir)
self.assertEqual(got["model"], "stub-model")
self.assertEqual(got["vllm"], {"running": 1.0, "waiting": 0.0, "kv_usage": 0.25,
"prefix_queries": 40.0, "prefix_hits": 30.0,
"prompt_tokens": 100.0, "generation_tokens": 7.0})
self.assertEqual(got["gpu"], {"vram_used": 8000000000, "vram_total": 8573157376,
"busy": 96, "temp": 61.0, "power": 120.5, "fan": 1800.0})
self.assertEqual(got["experts"], stats)
self.assertEqual(got["streams"], 0)
def test_status_without_metrics_gpu_or_stats(self):
got = self.status(sysfs=self.dir) # the stub answers /metrics with JSON, not metrics
self.assertEqual((got["gpu"], got["experts"]), (None, None))
self.assertEqual(got["vllm"]["running"], 0.0)
tests/test_web_page.py
+49
View File
@@ -0,0 +1,49 @@
import json
import shutil
import subprocess
import unittest
from pathlib import Path
PAGE = (Path(__file__).parent.parent / "prain" / "web.html").read_text()
@unittest.skipUnless(shutil.which("node"), "needs node")
class Markdown(unittest.TestCase):
"""The page's md() (model output -> innerHTML) run in node."""
def md(self, text):
script = PAGE[PAGE.index("<script>") + 8:PAGE.index("/* ---------- thread")]
script = script.split("/* ---------- markdown", 1)[1].split("*/", 1)[1]
out = subprocess.run(["node", "-e", script + f"\nprocess.stdout.write(md({json.dumps(text)}))"],
capture_output=True, text=True, check=True)
return out.stdout
def test_html_in_model_output_is_text(self):
html = self.md('<img src=x onerror="alert(1)"> **<b>hi</b>** `<i>` [x](javascript:alert(1))')
self.assertNotIn("<img", html)
self.assertNotIn("<b>", html)
self.assertNotIn('href="javascript', html)
self.assertIn("&lt;img src=x onerror=&quot;alert(1)&quot;&gt;", html)
self.assertIn("<strong>&lt;b&gt;hi&lt;/b&gt;</strong>", html)
def test_blocks(self):
html = self.md("# Title\n\nSome *very* **bold** text with `code` and "
"[a link](https://x.org/?a=1&b=2).\n\n```python\nif a < b:\n pass\n```\n"
"3. three\n4. four\n\n- one\n more\n- two\n\n| a | b |\n|---|---|\n| 1 | 2 |\n"
"\n> quoted\n\n---\nend")
for part in ("<h3>Title</h3>", "<em>very</em>", "<strong>bold</strong>", "<code>code</code>",
'<a href="https://x.org/?a=1&amp;b=2" target="_blank" rel="noopener noreferrer">',
'<span>python</span>', "<pre><code>if a &lt; b:\n pass</code></pre>",
'<ol start="3"><li>three</li><li>four</li></ol>',
"<ul><li>one more</li><li>two</li></ul>",
"<th>a</th><th>b</th></tr></thead><tbody><tr><td>1</td><td>2</td>",
"<blockquote><p>quoted</p></blockquote>", "<hr>", "<p>end</p>"):
self.assertIn(part, html)
def test_an_open_fence_while_streaming_is_code(self):
self.assertIn("<pre><code>x = 1\n# not a heading</code></pre>",
self.md("Here:\n```\nx = 1\n# not a heading"))
if __name__ == "__main__":
unittest.main()