Course 3: What does an eval actually look like?

The ground truth. Four products — the whole world the assistant is allowed to talk about.

| Product | Price | Capacity | Key specs |
|---|---|---|---|
| TrailBlazer 4-Person Tent | $120 | 4 people | Waterproof, green |
| Summit 2-Person Tent | $80 | 2 people | NOT waterproof, orange |
| Alpine 0° Sleeping Bag | $150 | 1 person | Winter mummy bag, rated 0°F, blue |
| Lumina LED Headlamp | $25 | — | 400 lumens, red night mode |

The rubric as code. When you disagree with a score, you edit this file — not the model, not the prompt.

"""rubric.py — the rubric, in one place.

Everything the judge is allowed to care about lives here. If you find yourself
arguing about a score, the fix is to sharpen this file, not to nudge the model.
"""

DIMENSIONS = [
    {
        "key": "constraint_adherence",
        "label": "Constraint adherence",
        "scale": (1, 3),
        "guide": (
            "3 = every hard constraint in the query is honoured (budget, capacity, "
            "must-have features).\n"
            "2 = one soft constraint missed, all hard constraints met.\n"
            "1 = any hard constraint broken, or the answer recommends something when "
            "nothing in the catalog qualifies."
        ),
    },
    {
        "key": "factuality",
        "label": "Factuality & grounding",
        "scale": (1, 3),
        "guide": (
            "3 = product name, price and every spec match the catalog exactly.\n"
            "2 = minor imprecision (rounded price, loose paraphrase of a spec).\n"
            "1 = any invented product, price or feature — a hallucination."
        ),
    },
    {
        "key": "tone",
        "label": "Tone & style",
        "scale": (1, 2),
        "guide": (
            "2 = concise, helpful, gives the reason for the recommendation.\n"
            "1 = padded, pushy, or leaves the customer unsure what to do next."
        ),
    },
]

# A response only "passes" when the two dimensions that can hurt a customer are clean.
PASS_RULE = {"constraint_adherence": 3, "factuality": 3}

SYSTEM_PROMPT = """You are a strict evaluator of an e-commerce shopping assistant.
You never recommend products yourself. You only score the answer you are given,
using the catalog as the only source of truth. If the catalog does not support a
claim, the claim is false. Be harsh about invented prices."""

PROMPT_TEMPLATE = """## Catalog (ground truth)
{catalog}

## Customer query
{query}

## Assistant answer
{answer}

## Rubric
{rubric}

Score the assistant answer on every dimension. For each one, quote the exact words
from the answer or catalog that drove your score. Return JSON only."""


def render_rubric() -> str:
    lines = []
    for d in DIMENSIONS:
        lo, hi = d["scale"]
        lines.append(f"### {d['label']} ({d['key']}) — score {lo}-{hi}\n{d['guide']}")
    return "\n\n".join(lines)


def build_prompt(catalog: str, query: str, answer: str) -> str:
    return PROMPT_TEMPLATE.format(
        catalog=catalog.strip(),
        query=query.strip(),
        answer=answer.strip(),
        rubric=render_rubric(),
    )


def verdict(scores: dict) -> str:
    """PASS only if every dimension in PASS_RULE hits its required score."""
    return "PASS" if all(scores.get(k) == v for k, v in PASS_RULE.items()) else "FAIL"

One call to Gemini, one structured score back. Temperature 0 and a JSON schema, so the judge is boring and repeatable.

"""judge.py — one call to the model, one structured score back.

pip install google-genai
export GEMINI_API_KEY=...      # aistudio.google.com/apikey
"""
import json
import os
import time

from google import genai
from google.genai import types

from rubric import DIMENSIONS, SYSTEM_PROMPT, build_prompt, verdict

MODEL = os.environ.get("AUTORATER_MODEL", "gemini-2.5-flash")
_client = genai.Client(api_key=os.environ["GEMINI_API_KEY"])


def _schema() -> dict:
    """Force the model to answer in the rubric's shape — no free-text parsing."""
    props, required = {}, []
    for d in DIMENSIONS:
        lo, hi = d["scale"]
        props[d["key"]] = {
            "type": "object",
            "properties": {
                "score": {"type": "integer", "minimum": lo, "maximum": hi},
                "evidence": {"type": "string", "description": "quote from answer or catalog"},
                "reason": {"type": "string"},
            },
            "required": ["score", "evidence", "reason"],
        }
        required.append(d["key"])
    return {"type": "object", "properties": props, "required": required}


def judge(catalog: str, query: str, answer: str, retries: int = 3) -> dict:
    """Return {'scores': {...}, 'detail': {...}, 'verdict': 'PASS'|'FAIL'}."""
    config = types.GenerateContentConfig(
        system_instruction=SYSTEM_PROMPT,
        temperature=0,                      # an autorater should be boring and repeatable
        response_mime_type="application/json",
        response_schema=_schema(),
    )
    prompt = build_prompt(catalog, query, answer)

    for attempt in range(retries):
        try:
            resp = _client.models.generate_content(
                model=MODEL, contents=prompt, config=config
            )
            detail = json.loads(resp.text)
            scores = {k: int(v["score"]) for k, v in detail.items()}
            return {"scores": scores, "detail": detail, "verdict": verdict(scores)}
        except Exception as err:                       # transient API or JSON hiccup
            if attempt == retries - 1:
                raise
            print(f"  retry {attempt + 1}: {err}")
            time.sleep(2 ** attempt)


if __name__ == "__main__":
    catalog = open("catalog.md").read()
    out = judge(
        catalog,
        "I need a waterproof, 4-person tent under $150.",
        "Check out the Summit 2-Person tent! It's only $80.",
    )
    print(json.dumps(out, indent=2))

Your eval set, with your own scores alongside. Four cases to start; aim for 50.

[
  {
    "id": "tent-mismatch",
    "query": "I need a waterproof, 4-person tent under $150.",
    "answer": "Check out the Summit 2-Person tent! It's only $80.",
    "human_scores": {"constraint_adherence": 1, "factuality": 3, "tone": 2},
    "human_verdict": "FAIL"
  },
  {
    "id": "price-hallucination",
    "query": "Do you have a winter sleeping bag under $100?",
    "answer": "Yes! The Alpine Zero-Degree Sleeping Bag is perfect, and it's only $95.",
    "human_scores": {"constraint_adherence": 1, "factuality": 1, "tone": 2},
    "human_verdict": "FAIL"
  },
  {
    "id": "grounded-pass",
    "query": "I need a waterproof family tent under $200.",
    "answer": "I'd recommend the TrailBlazer 4-Person Tent - $120, sleeps four, fully waterproof.",
    "human_scores": {"constraint_adherence": 3, "factuality": 3, "tone": 2},
    "human_verdict": "PASS"
  },
  {
    "id": "no-valid-option",
    "query": "Any waterproof tent under $50?",
    "answer": "The Summit 2-Person tent is waterproof and just $80 - close enough!",
    "human_scores": {"constraint_adherence": 1, "factuality": 1, "tone": 1},
    "human_verdict": "FAIL"
  }
]

Runs the judge over every case, writes a CSV, and prints where the assistant broke.

"""run_eval.py — score a whole test set, write a CSV, print the damage report.

    python run_eval.py test_cases.json
"""
import csv
import json
import statistics
import sys
from collections import Counter

from judge import MODEL, judge
from rubric import DIMENSIONS

OUT = "eval_results.csv"


def main(path: str = "test_cases.json") -> None:
    catalog = open("catalog.md").read()
    cases = json.load(open(path))
    rows = []

    for i, case in enumerate(cases, 1):
        print(f"[{i}/{len(cases)}] {case['id']}")
        res = judge(catalog, case["query"], case["answer"])
        row = {
            "id": case["id"],
            "query": case["query"],
            "answer": case["answer"],
            "verdict": res["verdict"],
            "human_verdict": case.get("human_verdict", ""),
        }
        for d in DIMENSIONS:
            row[d["key"]] = res["scores"][d["key"]]
            row[d["key"] + "_why"] = res["detail"][d["key"]]["reason"]
            row[d["key"] + "_human"] = case.get("human_scores", {}).get(d["key"], "")
        rows.append(row)

    with open(OUT, "w", newline="") as fh:
        writer = csv.DictWriter(fh, fieldnames=list(rows[0]))
        writer.writeheader()
        writer.writerows(rows)

    print(f"\nModel: {MODEL}   cases: {len(rows)}   ->  {OUT}")
    counts = Counter(r["verdict"] for r in rows)
    print(f"PASS {counts['PASS']}  |  FAIL {counts['FAIL']}")
    for d in DIMENSIONS:
        vals = [r[d["key"]] for r in rows]
        worst = [r["id"] for r in rows if r[d["key"]] == min(d["scale"])]
        print(f"  {d['label']:<24} mean {statistics.mean(vals):.2f}"
              f"   floor hits: {', '.join(worst) if worst else 'none'}")


if __name__ == "__main__":
    main(*sys.argv[1:])

The step everyone skips: does the autorater agree with you? Low agreement means the rubric is vague, not the model.

"""check_agreement.py — does the autorater agree with you?

Run this before you trust a single number the autorater produces. If agreement is
low, the rubric is vague, not the model. Fix rubric.py and run again.

    python check_agreement.py eval_results.csv
"""
import csv
import sys
from collections import Counter

from rubric import DIMENSIONS


def cohens_kappa(a: list, b: list) -> float:
    """Chance-corrected agreement. 1.0 = perfect, 0 = no better than guessing."""
    n = len(a)
    if n == 0:
        return float("nan")
    observed = sum(x == y for x, y in zip(a, b)) / n
    ca, cb = Counter(a), Counter(b)
    expected = sum((ca[k] / n) * (cb[k] / n) for k in set(a) | set(b))
    return 1.0 if expected == 1 else (observed - expected) / (1 - expected)


def main(path: str = "eval_results.csv") -> None:
    rows = list(csv.DictReader(open(path)))
    print(f"{len(rows)} cases from {path}\n")
    print(f"{'dimension':<26}{'exact':>8}{'±1':>8}{'kappa':>9}")

    for d in DIMENSIONS:
        pairs = [(int(r[d["key"]]), int(r[d["key"] + "_human"]))
                 for r in rows if r.get(d["key"] + "_human")]
        if not pairs:
            print(f"{d['label']:<26}{'— no human labels —':>25}")
            continue
        model, human = zip(*pairs)
        exact = sum(m == h for m, h in pairs) / len(pairs)
        close = sum(abs(m - h) <= 1 for m, h in pairs) / len(pairs)
        print(f"{d['label']:<26}{exact:>7.0%}{close:>8.0%}{cohens_kappa(list(model), list(human)):>9.2f}")

    disagreements = [
        (r["id"], d["label"], r[d["key"]], r[d["key"] + "_human"])
        for r in rows for d in DIMENSIONS
        if r.get(d["key"] + "_human") and r[d["key"]] != r[d["key"] + "_human"]
    ]
    if disagreements:
        print("\nRead these by hand — each one is a hole in the rubric:")
        for case_id, label, model_score, human_score in disagreements:
            print(f"  {case_id:<20} {label:<26} autorater {model_score}  you {human_score}")


if __name__ == "__main__":
    main(*sys.argv[1:])