| """ |
| evaluate.py |
| |
| Reproduce the demo test accuracy on the hand-crafted 18-query set. |
| |
| Run: |
| python evaluate.py |
| """ |
|
|
| from hv_router import HVRouter |
|
|
|
|
| DEMO_QUERIES = [ |
| ("what is the weather in tokyo", "weather"), |
| ("remind me to water the plants", "reminder"), |
| ("play something relaxing", "music"), |
| ("how do i get to the train station", "directions"), |
| ("book a table for four", "restaurant"), |
| ("show me my recent purchases", "shopping"), |
| ("what happened in the news today", "news"), |
| ("draft an email to sarah", "email"), |
| ("what is on my calendar tomorrow", "calendar"), |
| ("how many calories did i eat today", "health"), |
| ("transfer fifty dollars to checking", "finance"), |
| ("tell me a joke", "out_of_scope"), |
| ("what did i buy last week", "shopping"), |
| ("send a message to the team", "email"), |
| ("how many steps did i take", "health"), |
| ("what is the capital of france", "out_of_scope"), |
| ("who are you", "out_of_scope"), |
| ("sing me a song", "out_of_scope"), |
| ] |
|
|
|
|
| def main(model_path="router_model"): |
| router = HVRouter.load(model_path) |
|
|
| print(f"Model: {len(router.domain_labels)} domains, " |
| f"{len(router.vocab_index)} vocabulary") |
| print() |
|
|
| correct = 0 |
| print(f"{'query':<40} {'predicted':<14} {'expected':<14} " |
| f"{'peak':>6} ok") |
| print("-" * 80) |
|
|
| for phrase, expected in DEMO_QUERIES: |
| vote, peak = router.scores_and_peak(phrase) |
| pred = router.route(phrase) |
| ok = pred == expected |
| if ok: |
| correct += 1 |
| mark = "OK" if ok else "ERR" |
| print(f"{phrase:<40} {pred:<14} {expected:<14} " |
| f"{peak:>6.2f} {mark}") |
|
|
| print() |
| n = len(DEMO_QUERIES) |
| print(f"Accuracy: {correct}/{n} = {correct/n:.1%}") |
|
|
|
|
| if __name__ == "__main__": |
| import sys |
| path = sys.argv[1] if len(sys.argv) > 1 else "router_model" |
| main(path) |
