prain: web UI with a model monitor

A clear chat page (markdown, code blocks with copy, stop, new chat, kept in the browser,
drafts shown as such) and a monitor sidebar, a sheet on phones: tokens/s, a live map of the
30 x 128 experts (in VRAM, used now, copied in), VRAM hit rate, copies, context, prompt cache,
GPU VRAM/load/temperature/power/fan. GET /status merges vLLM /metrics, amdgpu sysfs and the
stats file, which now lists the experts in VRAM. /warm answers 202 when a warm-up starts.
README: the int4 kernels, their A/B, 2048-token chunks as a trade, prompt logprobs limit.

tests/test_web.py

@@ -1,4 +1,5 @@
 import json
+import os
 import threading
 import unittest
 import urllib.error
@@ -7,7 +8,7 @@ import urllib.request
 import prain.chat as chat
 from prain.config import Draft, Serve
 from prain.web import make_server
-from tests.helpers import StubVllm, wait_for
+from tests.helpers import StubVllm, TmpCase, wait_for
 
 
 def events(response):
@@ -38,7 +39,9 @@ class Web(unittest.TestCase):
         self.url = f"http://127.0.0.1:{self.server.server_port}"
 
     def warm(self, session, draft):
-        chat.post(self.url, "/warm", {"session": session, "messages": [], "draft": draft}).read()
+        with chat.post(self.url, "/warm", {"session": session, "messages": [],
+                                           "draft": draft}) as response:
+            return response.status
 
     def ask(self, session, text):
         return chat.post(self.url, "/chat", {"session": session,
@@ -46,7 +49,9 @@ class Web(unittest.TestCase):
 
     def test_page_warm_up_and_chat(self):
         with urllib.request.urlopen(self.url) as response:
-            self.assertIn(b'<textarea id="draft"', response.read())
+            page = response.read()
+        self.assertIn(b'<textarea id="draft"', page)
+        self.assertIn(b'"/status"', page)  # the monitor polls it
         self.warm("s1", "Why is ")
         wait_for(lambda: self.stub.bodies)
         self.assertEqual(self.stub.bodies[0]["messages"], [{"role": "user", "content": "Why is"}])
@@ -67,9 +72,9 @@ class Web(unittest.TestCase):
 
     def test_no_warm_up_while_another_answer_streams(self):
         self.server.streams[0] = 1  # someone else's answer is streaming
-        self.warm("bob", "typing along")
+        self.assertEqual(self.warm("bob", "typing along"), 204)  # 204: nothing warms up
         self.server.streams[0] = 0
-        self.warm("bob", "now")
+        self.assertEqual(self.warm("bob", "now"), 202)  # 202: a warm-up starts
         wait_for(lambda: self.stub.bodies)
         self.assertEqual([b["messages"][-1]["content"] for b in self.stub.bodies], ["now"])
 
@@ -136,3 +141,54 @@ class Web(unittest.TestCase):
             self.assertEqual(events(r), ["Hel", "lo", "!", "<done>"])
         thread.join()
         self.assertEqual(sum("vllm_xargs" in b for b in self.stub.bodies), 1)
+
+
+class Status(TmpCase):
+    def setUp(self):
+        super().setUp()
+        self.stub = StubVllm()
+        self.addCleanup(self.stub.close)
+
+    def status(self, **options):
+        server = make_server(self.stub.url, 0, **options)
+        threading.Thread(target=server.serve_forever, daemon=True).start()
+        self.addCleanup(server.server_close)
+        self.addCleanup(server.shutdown)
+        with urllib.request.urlopen(f"http://127.0.0.1:{server.server_port}/status") as r:
+            return json.load(r)
+
+    def test_status_reports_vllm_the_gpu_and_the_experts(self):
+        self.stub.metrics = (
+            "# HELP vllm:num_requests_running Number of requests running.\n"
+            'vllm:num_requests_running{engine="0",model_name="m"} 1.0\n'
+            'vllm:num_requests_waiting{engine="0",model_name="m"} 0.0\n'
+            'vllm:num_requests_waiting_by_reason{engine="0",reason="capacity"} 5.0\n'
+            'vllm:kv_cache_usage_perc{engine="0",model_name="m"} 0.25\n'
+            'vllm:prefix_cache_queries_total{engine="0",model_name="m"} 40.0\n'
+            'vllm:prefix_cache_hits_total{engine="0",model_name="m"} 30.0\n'
+            'vllm:prompt_tokens_total{engine="0",model_name="m"} 100.0\n'
+            'vllm:generation_tokens_total{engine="0",model_name="m"} 7.0\n')
+        os.makedirs(os.path.join(self.dir, "card0", "device"))  # an iGPU: no VRAM files
+        gpu = os.path.join(self.dir, "card1", "device")
+        os.makedirs(os.path.join(gpu, "hwmon", "hwmon4"))
+        for name, value in {"mem_info_vram_total": 8573157376, "mem_info_vram_used": 8000000000,
+                            "gpu_busy_percent": 96, "hwmon/hwmon4/temp1_input": 61000,
+                            "hwmon/hwmon4/power1_average": 120500000,
+                            "hwmon/hwmon4/fan1_input": 1800}.items():
+            self.write(os.path.join("card1", "device", name), f"{value}\n")
+        stats = {"layers": {"0": {"uses": [3, 0], "vram": [0], "hits": 2, "misses": 1}}}
+        stats_path = self.write("stats.json", json.dumps(stats))
+        got = self.status(stats_path=stats_path, sysfs=self.dir)
+        self.assertEqual(got["model"], "stub-model")
+        self.assertEqual(got["vllm"], {"running": 1.0, "waiting": 0.0, "kv_usage": 0.25,
+                                       "prefix_queries": 40.0, "prefix_hits": 30.0,
+                                       "prompt_tokens": 100.0, "generation_tokens": 7.0})
+        self.assertEqual(got["gpu"], {"vram_used": 8000000000, "vram_total": 8573157376,
+                                      "busy": 96, "temp": 61.0, "power": 120.5, "fan": 1800.0})
+        self.assertEqual(got["experts"], stats)
+        self.assertEqual(got["streams"], 0)
+
+    def test_status_without_metrics_gpu_or_stats(self):
+        got = self.status(sysfs=self.dir)  # the stub answers /metrics with JSON, not metrics
+        self.assertEqual((got["gpu"], got["experts"]), (None, None))
+        self.assertEqual(got["vllm"]["running"], 0.0)