mstrasser's picture
Jeff v1.2 adapters demo
8ef35b0 verified
Raw History Blame Contribute Delete
8.88 kB
"""Jeff demo Space: one Jeff v1.2 base model loaded once, several LoRA adapters beside it, one request at a time.
Pick an adapter, edit its example request (a situation and questions with named options), and see the probability of
each option and how long the model call took.
Configuration (environment variables):
JEFF_BASE_REPO Hugging Face repo of the base model (default mstrasser/Jeff-Qwen3.5-0.8B)
JEFF_BASE_REVISION its revision (default v1.2)
JEFF_ADAPTER_REVISION revision of every adapter repo (default main)
JEFF_ADAPTERS_LOAD comma-separated adapter names to load (default: every adapter in adapters.json)
JEFF_BASE_DIR a local base checkpoint folder instead of JEFF_BASE_REPO (for local tests)
JEFF_ADAPTERS_DIR a local folder with one sub-folder per adapter, named like the adapter, instead of the repos
JEFF_DEVICE cuda, mps or cpu (default: cuda when present, otherwise cpu)
JEFF_LORA_PRECISION model (default, the base model's own dtype) or float32, as for jeff-serve
"""
from __future__ import annotations
import json
import os
import threading
import time
from pathlib import Path
from typing import Any, cast
import gradio as gr
import pandas as pd
from huggingface_hub import snapshot_download
from jeff.lora import AdapterSet
from jeff.model import answer
from jeff.models import device_from_environment, load_decision_model
from jeff.orders import predict_orders
try:
import spaces
gpu = spaces.GPU
except ImportError: # running outside Hugging Face Spaces (a local test): ZeroGPU's decorator does not exist there
def gpu(function): # type: ignore[no-untyped-def]
"""Local stand-in for spaces.GPU: on a normal machine the model already sits on its device."""
return function
BASE_ONLY = "none (Jeff v1.2 base only)"
HERE = Path(__file__).parent
CATALOGUE = json.loads((HERE / "adapters.json").read_text())
EXAMPLES = {entry["name"]: entry for entry in CATALOGUE["adapters"]}
def wanted_adapters() -> list[str]:
value = os.getenv("JEFF_ADAPTERS_LOAD")
names = [name.strip() for name in value.split(",") if name.strip()] if value else list(EXAMPLES)
unknown = [name for name in names if name not in EXAMPLES]
if unknown:
raise ValueError(f"JEFF_ADAPTERS_LOAD names adapters that adapters.json does not list: {unknown}")
return names
def base_folder() -> Path:
local = os.getenv("JEFF_BASE_DIR")
if local:
return Path(local)
repo = os.getenv("JEFF_BASE_REPO", CATALOGUE["base_repo"])
return Path(snapshot_download(repo, revision=os.getenv("JEFF_BASE_REVISION", CATALOGUE["base_revision"])))
def adapter_folder(name: str) -> Path:
local = os.getenv("JEFF_ADAPTERS_DIR")
if local:
folder = Path(local) / name
if not folder.is_dir():
raise FileNotFoundError(f"Adapter {name!r}: {folder} does not exist (JEFF_ADAPTERS_DIR={local}); "
"set JEFF_ADAPTERS_LOAD to the adapters you have")
return folder
repo = CATALOGUE["repo_prefix"] + name
try:
return Path(snapshot_download(repo, revision=os.getenv("JEFF_ADAPTER_REVISION", "main")))
except Exception as error:
raise RuntimeError(f"Could not download adapter {name!r} from {repo}; set JEFF_ADAPTERS_LOAD to the adapters "
"that exist") from error
BASE = base_folder()
MODEL = load_decision_model(checkpoint=BASE, device=device_from_environment())
MODEL.eval()
BASE_MAX_OPTIONS = int(json.loads((BASE / "decision_config.json").read_text())["max_options"])
LOADED = wanted_adapters()
ADAPTERS = AdapterSet(MODEL, BASE, {name: adapter_folder(name) for name in LOADED}, base_max_options=BASE_MAX_OPTIONS,
precision=os.getenv("JEFF_LORA_PRECISION", "model")) # as jeff-serve: the model's own dtype
LOCK = threading.Lock()
def check_request(request: Any) -> dict[str, Any]:
if not isinstance(request, dict) or set(request) - {"state", "questions"} or "state" not in request:
raise gr.Error('The request must be a JSON object with "state" and "questions" (and nothing else).')
questions = request.get("questions")
if not isinstance(questions, dict) or not questions:
raise gr.Error('"questions" must be an object with at least one named question.')
for key, question in questions.items():
if not isinstance(question, dict) or question.get("type") not in ("choice", "score", "noul"):
raise gr.Error(f"Question {key!r} needs a type: choice, score or noul.")
return cast(dict[str, Any], request)
@gpu
def decide(name: str, request: dict[str, Any], orders: int) -> tuple[dict[str, Any], float]:
"""Answer every question of the request with the chosen adapter (or the base). Returns the answers and the
milliseconds spent in the model call itself (not in parsing or drawing)."""
adapter = None if name == BASE_ONLY else name
rows = [{"state": request["state"], "question": question} for question in request["questions"].values()]
with LOCK:
ADAPTERS.use(adapter)
limit = ADAPTERS.heads[adapter].max_options or BASE_MAX_OPTIONS
for key, question in request["questions"].items():
count = len(question.get("criteria") or {}) if question["type"] == "choice" else 0
if count > limit:
raise gr.Error(f"Question {key!r} has {count} options; this model handles at most {limit}.")
started = time.perf_counter()
values = predict_orders(MODEL, rows, orders) # type: ignore[arg-type]
milliseconds = (time.perf_counter() - started) * 1000
answers = {key: answer(question, probabilities)
for (key, question), probabilities in zip(request["questions"].items(), values, strict=True)}
return {"model": adapter or "jeff", "answers": answers, "usage": {"orders": orders}}, milliseconds
def chart_rows(result: dict[str, Any]) -> pd.DataFrame:
"""The bars for one answer: yes and no for a yes/no question, every level for a score, the 15 most likely options
for a choice."""
if result["type"] == "noul":
pairs = [("yes", result["noul"]), ("no", 1 - result["noul"])]
elif result["type"] == "score":
pairs = [(f"{index}: {level}", result["probabilities"][index]) for index, level in result["legend"].items()]
else:
pairs = sorted(result["probabilities"].items(), key=lambda pair: -pair[1])[:15]
return pd.DataFrame({"option": [label for label, _ in pairs], "probability": [round(value, 4) for _, value in pairs]})
def run(name: str, text: str, twice: bool) -> tuple[pd.DataFrame, str, dict[str, Any]]:
try:
request = json.loads(text)
except json.JSONDecodeError as error:
raise gr.Error(f"The request is not valid JSON: {error}") from error
request = check_request(request)
response, milliseconds = decide(name, request, 2 if twice else 1)
first = next(iter(request["questions"]))
table = chart_rows(response["answers"][first])
timing = (f"**{milliseconds:.0f} ms** for the model call ({len(request['questions'])} question(s)"
f"{', each answered twice' if twice else ''}; parsing and drawing not included). Chart: question `{first}`"
f"{' (top 15 options)' if len(table) == 15 else ''}.")
return table, timing, response
def example(name: str) -> str:
return json.dumps(EXAMPLES[name]["request"], indent=2, ensure_ascii=False)
with gr.Blocks(title="Jeff: fast, calibrated decisions") as demo:
gr.Markdown("# Jeff v1.2 with swappable adapters\nOne 0.8B base model, loaded once. Each request picks a small "
"LoRA adapter (or none). Jeff returns a calibrated probability for every option in one forward pass; "
"it does not generate text. Times shown are for this Space's GPU (an NVIDIA L4), one request at a "
"time.")
with gr.Row():
choice = gr.Dropdown([*LOADED, BASE_ONLY], value=LOADED[0] if LOADED else BASE_ONLY, label="Adapter")
twice = gr.Checkbox(False, label="Answer twice (second time with the options reversed, then average)")
editor = gr.Code(example(LOADED[0]) if LOADED else json.dumps({"state": "", "questions": {}}), language="json",
label="Request: the situation (state) and the questions with their options")
button = gr.Button("Run", variant="primary")
timing = gr.Markdown()
chart = gr.BarPlot(x="option", y="probability", label="Probability per option (first question)",
y_lim=[0, 1], sort=None)
output = gr.JSON(label="Full answer")
choice.change(lambda name: example(name) if name != BASE_ONLY else gr.update(), choice, editor)
button.click(run, [choice, editor, twice], [chart, timing, output])
if __name__ == "__main__":
demo.launch()