fabolous005
a14e3ade0b
A clear chat page (markdown, code blocks with copy, stop, new chat, kept in the browser, drafts shown as such) and a monitor sidebar, a sheet on phones: tokens/s, a live map of the 30 x 128 experts (in VRAM, used now, copied in), VRAM hit rate, copies, context, prompt cache, GPU VRAM/load/temperature/power/fan. GET /status merges vLLM /metrics, amdgpu sysfs and the stats file, which now lists the experts in VRAM. /warm answers 202 when a warm-up starts. README: the int4 kernels, their A/B, 2048-token chunks as a trade, prompt logprobs limit.
23 lines
522 B
TOML
23 lines
522 B
TOML
[project]
|
|
name = "prain"
|
|
version = "0.1.0"
|
|
description = "Per-expert VRAM/RAM/disk placement for MoE models served by vLLM"
|
|
requires-python = ">=3.11"
|
|
dependencies = [] # torch and vllm come from the gfx1030 vLLM image
|
|
|
|
[project.scripts]
|
|
prain = "prain.cli:main"
|
|
|
|
[project.entry-points."vllm.general_plugins"]
|
|
prain = "prain.plugin:register"
|
|
|
|
[build-system]
|
|
requires = ["setuptools>=64"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[tool.setuptools]
|
|
packages = ["prain"]
|
|
|
|
[tool.setuptools.package-data]
|
|
prain = ["web.html"]
|