prain: personas for the chats; README on quantization levels and personas
prain web/tui --persona FILE sends the file as the system message ahead of every warm-up and answer; the page shows its name. Three examples in examples/personas (task, how to answer, environment, how to use it). A persona costs 2.0-3.2 s once (241-467 tokens), then at most 0.3 s per conversation from the prefix cache. README rewritten around launching: quick start, the vLLM flags, a quantization table measured today at equal expert VRAM (bf16 0.90 / int8 2.19 / int4 5.19 tok/s; 2.3k-token prompt 69.5 / 32.3 / 20.5 s; 4 of 6 int8 and 0 of 6 int4 answers identical to bf16, one int4 answer wrong), how to switch formats, the VRAM budget, writing personas, the chats and the monitor.
This commit is contained in:
@@ -68,6 +68,19 @@ class Cli(TmpCase):
|
||||
main(["web", "--host", "0.0.0.0"])
|
||||
self.assertEqual(web.call_args.kwargs["host"], "0.0.0.0")
|
||||
|
||||
def test_a_persona_file_reaches_web_and_tui(self):
|
||||
path = self.write("gentoo-admin.md", "You look after a Gentoo box.\n\n")
|
||||
with mock.patch("prain.web.serve") as web:
|
||||
main(["web", "--persona", path])
|
||||
self.assertEqual((web.call_args.kwargs["system"], web.call_args.kwargs["persona"]),
|
||||
("You look after a Gentoo box.", "gentoo-admin"))
|
||||
main(["web"])
|
||||
self.assertEqual((web.call_args.kwargs["system"], web.call_args.kwargs["persona"]),
|
||||
(None, None))
|
||||
with mock.patch("prain.tui.main") as tui:
|
||||
main(["tui", "--persona", path])
|
||||
self.assertEqual(tui.call_args.kwargs["system"], "You look after a Gentoo box.")
|
||||
|
||||
def test_synth_sim_and_tune(self):
|
||||
hf = self.write("config.json", json.dumps(GEMMA4))
|
||||
trace = os.path.join(self.dir, "trace.jsonl.gz")
|
||||
|
||||
@@ -30,6 +30,17 @@ class Tui(unittest.TestCase):
|
||||
self.assertEqual(len(self.stub.bodies), 2)
|
||||
self.assertIn("tok/s", chat.status)
|
||||
|
||||
def test_a_persona_leads_every_request_but_stays_off_screen(self):
|
||||
chat = Chat(self.stub.url, "stub-model", system="Be brief.")
|
||||
chat.type("Why")
|
||||
wait_for(lambda: self.stub.bodies and not chat.warmer.busy)
|
||||
chat.send()
|
||||
wait_for(lambda: chat.poll() or chat.reply is None)
|
||||
system = {"role": "system", "content": "Be brief."}
|
||||
self.assertEqual([b["messages"][0] for b in self.stub.bodies], [system, system])
|
||||
self.assertEqual(chat.messages, [{"role": "user", "content": "Why"},
|
||||
{"role": "assistant", "content": "Hello!"}])
|
||||
|
||||
def test_warmup_off_and_cancel_on_send(self):
|
||||
off = Chat(self.stub.url, "stub-model", warmup="off")
|
||||
off.type("hello")
|
||||
|
||||
@@ -124,6 +124,19 @@ class Web(unittest.TestCase):
|
||||
"<done>"])
|
||||
thread.join()
|
||||
|
||||
def test_a_persona_leads_every_warm_up_and_answer(self):
|
||||
self.start(system="You answer in German.", persona="german")
|
||||
self.warm("s1", "Wie spät")
|
||||
wait_for(lambda: self.stub.bodies)
|
||||
with self.ask("s1", "Wie spät ist es?") as r:
|
||||
r.read()
|
||||
system = {"role": "system", "content": "You answer in German."}
|
||||
self.assertEqual([b["messages"][0] for b in self.stub.bodies], [system, system])
|
||||
self.assertEqual(self.stub.bodies[1]["messages"][1:],
|
||||
[{"role": "user", "content": "Wie spät ist es?"}])
|
||||
with urllib.request.urlopen(self.url + "/status") as r:
|
||||
self.assertEqual(json.load(r)["persona"], "german")
|
||||
|
||||
def test_refine_none_keeps_the_draft_and_more_full_slots_mean_no_draft(self):
|
||||
self.start(draft=Draft(refine="none"))
|
||||
self.stub.delay = lambda body: 0.0 if "vllm_xargs" in body else 0.5
|
||||
|
||||
Reference in New Issue
Block a user