Agent Benchmarking

from pathlib import Path
import sys

ROOT = (
    Path.cwd().resolve().parent
    if Path.cwd().name == "notebooks"
    else Path.cwd().resolve()
)
if str(ROOT) not in sys.path:
    sys.path.insert(0, str(ROOT))

from shipit_agent import Agent
from shipit_agent.deep import AgentBenchmark, TestCase
from examples.run_multi_tool_agent import build_llm_from_env

llm = build_llm_from_env("bedrock")

# Create a knowledge agent to benchmark
agent = Agent(
    llm=llm,
    prompt="You are a helpful programming assistant. Answer questions concisely and accurately.",
    name="code-helper",
)

# Define benchmark test cases
benchmark = AgentBenchmark(
    name="Programming Knowledge Eval",
    cases=[
        TestCase(
            input="What is Python's GIL?",
            expected_contains=["global interpreter lock", "thread"],
        ),
        TestCase(
            input="Explain what a REST API is",
            expected_contains=["http", "resource"],
            expected_not_contains=["graphql"],  # should not confuse with GraphQL
        ),
        TestCase(
            input="What are Python decorators?",
            expected_contains=["function", "wrap"],
        ),
        TestCase(
            input="Explain the difference between a list and a tuple in Python",
            expected_contains=["mutable", "immutable"],
        ),
        TestCase(
            input="What is Docker?",
            expected_contains=["container"],
        ),
    ],
)

print(f"Benchmark: {benchmark.name}")
print(f"Test cases: {len(benchmark.cases)}")

# Run all test cases against the agent
report = benchmark.run(agent)

# Print the summary report
print(report.summary())

# Detailed inspection of each test case
for i, r in enumerate(report.results, 1):
    status = "PASS" if r.passed else "FAIL"
    print(f"\n{'='*60}")
    print(f"Test {i}: [{status}] {r.test_case.input[:50]}...")
    print(f"Output preview: {r.output[:150]}...")
    if r.failures:
        print("Failures:")
        for f in r.failures:
            print(f"  - {f}")

# Export as dict for logging/dashboards
print(f"\n{'='*60}")
print("Exportable report:")
import json

print(json.dumps(report.to_dict(), indent=2)[:500])