Download app.py from mstrasser/Jeff-adapters-demo: direct link, hf CLI and curl.
- Browser
- Download file 8.88 kB
-
https://huggingface.co/spaces/mstrasser/Jeff-adapters-demo/resolve/main/app.py
- Command line
-
hf download hf://spaces/mstrasser/Jeff-adapters-demo/app.py
-
curl -L -o app.py https://huggingface.co/spaces/mstrasser/Jeff-adapters-demo/resolve/main/app.py
8.88 kB
| """Jeff demo Space: one Jeff v1.2 base model loaded once, several LoRA adapters beside it, one request at a time. | |
| Pick an adapter, edit its example request (a situation and questions with named options), and see the probability of | |
| each option and how long the model call took. | |
| Configuration (environment variables): | |
| JEFF_BASE_REPO Hugging Face repo of the base model (default mstrasser/Jeff-Qwen3.5-0.8B) | |
| JEFF_BASE_REVISION its revision (default v1.2) | |
| JEFF_ADAPTER_REVISION revision of every adapter repo (default main) | |
| JEFF_ADAPTERS_LOAD comma-separated adapter names to load (default: every adapter in adapters.json) | |
| JEFF_BASE_DIR a local base checkpoint folder instead of JEFF_BASE_REPO (for local tests) | |
| JEFF_ADAPTERS_DIR a local folder with one sub-folder per adapter, named like the adapter, instead of the repos | |
| JEFF_DEVICE cuda, mps or cpu (default: cuda when present, otherwise cpu) | |
| JEFF_LORA_PRECISION model (default, the base model's own dtype) or float32, as for jeff-serve | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import threading | |
| import time | |
| from pathlib import Path | |
| from typing import Any, cast | |
| import gradio as gr | |
| import pandas as pd | |
| from huggingface_hub import snapshot_download | |
| from jeff.lora import AdapterSet | |
| from jeff.model import answer | |
| from jeff.models import device_from_environment, load_decision_model | |
| from jeff.orders import predict_orders | |
| try: | |
| import spaces | |
| gpu = spaces.GPU | |
| except ImportError: # running outside Hugging Face Spaces (a local test): ZeroGPU's decorator does not exist there | |
| def gpu(function): # type: ignore[no-untyped-def] | |
| """Local stand-in for spaces.GPU: on a normal machine the model already sits on its device.""" | |
| return function | |
| BASE_ONLY = "none (Jeff v1.2 base only)" | |
| HERE = Path(__file__).parent | |
| CATALOGUE = json.loads((HERE / "adapters.json").read_text()) | |
| EXAMPLES = {entry["name"]: entry for entry in CATALOGUE["adapters"]} | |
| def wanted_adapters() -> list[str]: | |
| value = os.getenv("JEFF_ADAPTERS_LOAD") | |
| names = [name.strip() for name in value.split(",") if name.strip()] if value else list(EXAMPLES) | |
| unknown = [name for name in names if name not in EXAMPLES] | |
| if unknown: | |
| raise ValueError(f"JEFF_ADAPTERS_LOAD names adapters that adapters.json does not list: {unknown}") | |
| return names | |
| def base_folder() -> Path: | |
| local = os.getenv("JEFF_BASE_DIR") | |
| if local: | |
| return Path(local) | |
| repo = os.getenv("JEFF_BASE_REPO", CATALOGUE["base_repo"]) | |
| return Path(snapshot_download(repo, revision=os.getenv("JEFF_BASE_REVISION", CATALOGUE["base_revision"]))) | |
| def adapter_folder(name: str) -> Path: | |
| local = os.getenv("JEFF_ADAPTERS_DIR") | |
| if local: | |
| folder = Path(local) / name | |
| if not folder.is_dir(): | |
| raise FileNotFoundError(f"Adapter {name!r}: {folder} does not exist (JEFF_ADAPTERS_DIR={local}); " | |
| "set JEFF_ADAPTERS_LOAD to the adapters you have") | |
| return folder | |
| repo = CATALOGUE["repo_prefix"] + name | |
| try: | |
| return Path(snapshot_download(repo, revision=os.getenv("JEFF_ADAPTER_REVISION", "main"))) | |
| except Exception as error: | |
| raise RuntimeError(f"Could not download adapter {name!r} from {repo}; set JEFF_ADAPTERS_LOAD to the adapters " | |
| "that exist") from error | |
| BASE = base_folder() | |
| MODEL = load_decision_model(checkpoint=BASE, device=device_from_environment()) | |
| MODEL.eval() | |
| BASE_MAX_OPTIONS = int(json.loads((BASE / "decision_config.json").read_text())["max_options"]) | |
| LOADED = wanted_adapters() | |
| ADAPTERS = AdapterSet(MODEL, BASE, {name: adapter_folder(name) for name in LOADED}, base_max_options=BASE_MAX_OPTIONS, | |
| precision=os.getenv("JEFF_LORA_PRECISION", "model")) # as jeff-serve: the model's own dtype | |
| LOCK = threading.Lock() | |
| def check_request(request: Any) -> dict[str, Any]: | |
| if not isinstance(request, dict) or set(request) - {"state", "questions"} or "state" not in request: | |
| raise gr.Error('The request must be a JSON object with "state" and "questions" (and nothing else).') | |
| questions = request.get("questions") | |
| if not isinstance(questions, dict) or not questions: | |
| raise gr.Error('"questions" must be an object with at least one named question.') | |
| for key, question in questions.items(): | |
| if not isinstance(question, dict) or question.get("type") not in ("choice", "score", "noul"): | |
| raise gr.Error(f"Question {key!r} needs a type: choice, score or noul.") | |
| return cast(dict[str, Any], request) | |
| def decide(name: str, request: dict[str, Any], orders: int) -> tuple[dict[str, Any], float]: | |
| """Answer every question of the request with the chosen adapter (or the base). Returns the answers and the | |
| milliseconds spent in the model call itself (not in parsing or drawing).""" | |
| adapter = None if name == BASE_ONLY else name | |
| rows = [{"state": request["state"], "question": question} for question in request["questions"].values()] | |
| with LOCK: | |
| ADAPTERS.use(adapter) | |
| limit = ADAPTERS.heads[adapter].max_options or BASE_MAX_OPTIONS | |
| for key, question in request["questions"].items(): | |
| count = len(question.get("criteria") or {}) if question["type"] == "choice" else 0 | |
| if count > limit: | |
| raise gr.Error(f"Question {key!r} has {count} options; this model handles at most {limit}.") | |
| started = time.perf_counter() | |
| values = predict_orders(MODEL, rows, orders) # type: ignore[arg-type] | |
| milliseconds = (time.perf_counter() - started) * 1000 | |
| answers = {key: answer(question, probabilities) | |
| for (key, question), probabilities in zip(request["questions"].items(), values, strict=True)} | |
| return {"model": adapter or "jeff", "answers": answers, "usage": {"orders": orders}}, milliseconds | |
| def chart_rows(result: dict[str, Any]) -> pd.DataFrame: | |
| """The bars for one answer: yes and no for a yes/no question, every level for a score, the 15 most likely options | |
| for a choice.""" | |
| if result["type"] == "noul": | |
| pairs = [("yes", result["noul"]), ("no", 1 - result["noul"])] | |
| elif result["type"] == "score": | |
| pairs = [(f"{index}: {level}", result["probabilities"][index]) for index, level in result["legend"].items()] | |
| else: | |
| pairs = sorted(result["probabilities"].items(), key=lambda pair: -pair[1])[:15] | |
| return pd.DataFrame({"option": [label for label, _ in pairs], "probability": [round(value, 4) for _, value in pairs]}) | |
| def run(name: str, text: str, twice: bool) -> tuple[pd.DataFrame, str, dict[str, Any]]: | |
| try: | |
| request = json.loads(text) | |
| except json.JSONDecodeError as error: | |
| raise gr.Error(f"The request is not valid JSON: {error}") from error | |
| request = check_request(request) | |
| response, milliseconds = decide(name, request, 2 if twice else 1) | |
| first = next(iter(request["questions"])) | |
| table = chart_rows(response["answers"][first]) | |
| timing = (f"**{milliseconds:.0f} ms** for the model call ({len(request['questions'])} question(s)" | |
| f"{', each answered twice' if twice else ''}; parsing and drawing not included). Chart: question `{first}`" | |
| f"{' (top 15 options)' if len(table) == 15 else ''}.") | |
| return table, timing, response | |
| def example(name: str) -> str: | |
| return json.dumps(EXAMPLES[name]["request"], indent=2, ensure_ascii=False) | |
| with gr.Blocks(title="Jeff: fast, calibrated decisions") as demo: | |
| gr.Markdown("# Jeff v1.2 with swappable adapters\nOne 0.8B base model, loaded once. Each request picks a small " | |
| "LoRA adapter (or none). Jeff returns a calibrated probability for every option in one forward pass; " | |
| "it does not generate text. Times shown are for this Space's GPU (an NVIDIA L4), one request at a " | |
| "time.") | |
| with gr.Row(): | |
| choice = gr.Dropdown([*LOADED, BASE_ONLY], value=LOADED[0] if LOADED else BASE_ONLY, label="Adapter") | |
| twice = gr.Checkbox(False, label="Answer twice (second time with the options reversed, then average)") | |
| editor = gr.Code(example(LOADED[0]) if LOADED else json.dumps({"state": "", "questions": {}}), language="json", | |
| label="Request: the situation (state) and the questions with their options") | |
| button = gr.Button("Run", variant="primary") | |
| timing = gr.Markdown() | |
| chart = gr.BarPlot(x="option", y="probability", label="Probability per option (first question)", | |
| y_lim=[0, 1], sort=None) | |
| output = gr.JSON(label="Full answer") | |
| choice.change(lambda name: example(name) if name != BASE_ONLY else gr.update(), choice, editor) | |
| button.click(run, [choice, editor, twice], [chart, timing, output]) | |
| if __name__ == "__main__": | |
| demo.launch() | |