"""Jeff demo Space: one Jeff v1.2 base model loaded once, several LoRA adapters beside it, one request at a time. Pick an adapter, edit its example request (a situation and questions with named options), and see the probability of each option and how long the model call took. Configuration (environment variables): JEFF_BASE_REPO Hugging Face repo of the base model (default mstrasser/Jeff-Qwen3.5-0.8B) JEFF_BASE_REVISION its revision (default v1.2) JEFF_ADAPTER_REVISION revision of every adapter repo (default main) JEFF_ADAPTERS_LOAD comma-separated adapter names to load (default: every adapter in adapters.json) JEFF_BASE_DIR a local base checkpoint folder instead of JEFF_BASE_REPO (for local tests) JEFF_ADAPTERS_DIR a local folder with one sub-folder per adapter, named like the adapter, instead of the repos JEFF_DEVICE cuda, mps or cpu (default: cuda when present, otherwise cpu) JEFF_LORA_PRECISION model (default, the base model's own dtype) or float32, as for jeff-serve """ from __future__ import annotations import json import os import threading import time from pathlib import Path from typing import Any, cast import gradio as gr import pandas as pd from huggingface_hub import snapshot_download from jeff.lora import AdapterSet from jeff.model import answer from jeff.models import device_from_environment, load_decision_model from jeff.orders import predict_orders try: import spaces gpu = spaces.GPU except ImportError: # running outside Hugging Face Spaces (a local test): ZeroGPU's decorator does not exist there def gpu(function): # type: ignore[no-untyped-def] """Local stand-in for spaces.GPU: on a normal machine the model already sits on its device.""" return function BASE_ONLY = "none (Jeff v1.2 base only)" HERE = Path(__file__).parent CATALOGUE = json.loads((HERE / "adapters.json").read_text()) EXAMPLES = {entry["name"]: entry for entry in CATALOGUE["adapters"]} def wanted_adapters() -> list[str]: value = os.getenv("JEFF_ADAPTERS_LOAD") names = [name.strip() for name in value.split(",") if name.strip()] if value else list(EXAMPLES) unknown = [name for name in names if name not in EXAMPLES] if unknown: raise ValueError(f"JEFF_ADAPTERS_LOAD names adapters that adapters.json does not list: {unknown}") return names def base_folder() -> Path: local = os.getenv("JEFF_BASE_DIR") if local: return Path(local) repo = os.getenv("JEFF_BASE_REPO", CATALOGUE["base_repo"]) return Path(snapshot_download(repo, revision=os.getenv("JEFF_BASE_REVISION", CATALOGUE["base_revision"]))) def adapter_folder(name: str) -> Path: local = os.getenv("JEFF_ADAPTERS_DIR") if local: folder = Path(local) / name if not folder.is_dir(): raise FileNotFoundError(f"Adapter {name!r}: {folder} does not exist (JEFF_ADAPTERS_DIR={local}); " "set JEFF_ADAPTERS_LOAD to the adapters you have") return folder repo = CATALOGUE["repo_prefix"] + name try: return Path(snapshot_download(repo, revision=os.getenv("JEFF_ADAPTER_REVISION", "main"))) except Exception as error: raise RuntimeError(f"Could not download adapter {name!r} from {repo}; set JEFF_ADAPTERS_LOAD to the adapters " "that exist") from error BASE = base_folder() MODEL = load_decision_model(checkpoint=BASE, device=device_from_environment()) MODEL.eval() BASE_MAX_OPTIONS = int(json.loads((BASE / "decision_config.json").read_text())["max_options"]) LOADED = wanted_adapters() ADAPTERS = AdapterSet(MODEL, BASE, {name: adapter_folder(name) for name in LOADED}, base_max_options=BASE_MAX_OPTIONS, precision=os.getenv("JEFF_LORA_PRECISION", "model")) # as jeff-serve: the model's own dtype LOCK = threading.Lock() def check_request(request: Any) -> dict[str, Any]: if not isinstance(request, dict) or set(request) - {"state", "questions"} or "state" not in request: raise gr.Error('The request must be a JSON object with "state" and "questions" (and nothing else).') questions = request.get("questions") if not isinstance(questions, dict) or not questions: raise gr.Error('"questions" must be an object with at least one named question.') for key, question in questions.items(): if not isinstance(question, dict) or question.get("type") not in ("choice", "score", "noul"): raise gr.Error(f"Question {key!r} needs a type: choice, score or noul.") return cast(dict[str, Any], request) @gpu def decide(name: str, request: dict[str, Any], orders: int) -> tuple[dict[str, Any], float]: """Answer every question of the request with the chosen adapter (or the base). Returns the answers and the milliseconds spent in the model call itself (not in parsing or drawing).""" adapter = None if name == BASE_ONLY else name rows = [{"state": request["state"], "question": question} for question in request["questions"].values()] with LOCK: ADAPTERS.use(adapter) limit = ADAPTERS.heads[adapter].max_options or BASE_MAX_OPTIONS for key, question in request["questions"].items(): count = len(question.get("criteria") or {}) if question["type"] == "choice" else 0 if count > limit: raise gr.Error(f"Question {key!r} has {count} options; this model handles at most {limit}.") started = time.perf_counter() values = predict_orders(MODEL, rows, orders) # type: ignore[arg-type] milliseconds = (time.perf_counter() - started) * 1000 answers = {key: answer(question, probabilities) for (key, question), probabilities in zip(request["questions"].items(), values, strict=True)} return {"model": adapter or "jeff", "answers": answers, "usage": {"orders": orders}}, milliseconds def chart_rows(result: dict[str, Any]) -> pd.DataFrame: """The bars for one answer: yes and no for a yes/no question, every level for a score, the 15 most likely options for a choice.""" if result["type"] == "noul": pairs = [("yes", result["noul"]), ("no", 1 - result["noul"])] elif result["type"] == "score": pairs = [(f"{index}: {level}", result["probabilities"][index]) for index, level in result["legend"].items()] else: pairs = sorted(result["probabilities"].items(), key=lambda pair: -pair[1])[:15] return pd.DataFrame({"option": [label for label, _ in pairs], "probability": [round(value, 4) for _, value in pairs]}) def run(name: str, text: str, twice: bool) -> tuple[pd.DataFrame, str, dict[str, Any]]: try: request = json.loads(text) except json.JSONDecodeError as error: raise gr.Error(f"The request is not valid JSON: {error}") from error request = check_request(request) response, milliseconds = decide(name, request, 2 if twice else 1) first = next(iter(request["questions"])) table = chart_rows(response["answers"][first]) timing = (f"**{milliseconds:.0f} ms** for the model call ({len(request['questions'])} question(s)" f"{', each answered twice' if twice else ''}; parsing and drawing not included). Chart: question `{first}`" f"{' (top 15 options)' if len(table) == 15 else ''}.") return table, timing, response def example(name: str) -> str: return json.dumps(EXAMPLES[name]["request"], indent=2, ensure_ascii=False) with gr.Blocks(title="Jeff: fast, calibrated decisions") as demo: gr.Markdown("# Jeff v1.2 with swappable adapters\nOne 0.8B base model, loaded once. Each request picks a small " "LoRA adapter (or none). Jeff returns a calibrated probability for every option in one forward pass; " "it does not generate text. Times shown are for this Space's GPU (an NVIDIA L4), one request at a " "time.") with gr.Row(): choice = gr.Dropdown([*LOADED, BASE_ONLY], value=LOADED[0] if LOADED else BASE_ONLY, label="Adapter") twice = gr.Checkbox(False, label="Answer twice (second time with the options reversed, then average)") editor = gr.Code(example(LOADED[0]) if LOADED else json.dumps({"state": "", "questions": {}}), language="json", label="Request: the situation (state) and the questions with their options") button = gr.Button("Run", variant="primary") timing = gr.Markdown() chart = gr.BarPlot(x="option", y="probability", label="Probability per option (first question)", y_lim=[0, 1], sort=None) output = gr.JSON(label="Full answer") choice.change(lambda name: example(name) if name != BASE_ONLY else gr.update(), choice, editor) button.click(run, [choice, editor, twice], [chart, timing, output]) if __name__ == "__main__": demo.launch()