The Judge class uses an LLM to evaluate outputs with human-like reasoning. This is the recommended approach for most evaluations.
Diagram
Quick Start
python
from praisonaiagents.eval import Judge
# Evaluate any output
result = Judge().run(
output="The capital of France is Paris.",
expected="Paris is the capital of France."
)
print(f"Score: {result.score}/10")
print(f"Reasoning: {result.reasoning}")
With Agent
python
from praisonaiagents import Agent
from praisonaiagents.eval import Judge
# Create your agent
agent = Agent(name="assistant", instructions="Answer questions")
# Evaluate agent output
result = Judge().run(
agent=agent,
input="What is 2+2?",
expected="4"
)
print(f"Score: {result.score}/10")
Custom Criteria
python
from praisonaiagents.eval import Judge
# Define your own evaluation criteria
result = Judge(criteria="Response is helpful, accurate, and concise").run(
output="Here's a detailed explanation of quantum physics..."
)
print(f"Score: {result.score}/10")
print(f"Passed: {result.passed}")
Judge Types
AccuracyJudge
Compares output against expected output.
python
from praisonaiagents.eval import AccuracyJudge
judge = AccuracyJudge()
result = judge.run(
output="Paris",
expected="Paris",
input="What is the capital of France?"
)
# Score: 10/10 - Perfect match
CriteriaJudge
Evaluates against custom criteria.
python
from praisonaiagents.eval import CriteriaJudge
judge = CriteriaJudge(criteria="Response is professional and helpful")
result = judge.run(output="Hello! How can I assist you today?")
# Score: 9/10 - Professional and helpful
RecipeJudge
Evaluates multi-agent workflow outputs.
python
from praisonaiagents.eval import RecipeJudge
judge = RecipeJudge()
result = judge.run(
output=workflow_output,
expected="Complete research report"
)
Judge Registry
Register and retrieve custom judges:
python
from praisonaiagents.eval import add_judge, get_judge, list_judges
# Register a custom judge
add_judge("my_judge", MyCustomJudge)
# List all judges
print(list_judges()) # ['accuracy', 'criteria', 'recipe', 'my_judge']
# Get a judge by name
judge = get_judge("my_judge")
Evaluation Types
Diagram
Accuracy Evaluation
Compare agent output against expected output using LLM-as-judge.
Diagram
Basic
python
from praisonaiagents.eval import AccuracyEvaluator
evaluator = AccuracyEvaluator(
agent=my_agent,
input_text="What is 2+2?",
expected_output="4"
)
result = evaluator.run(print_summary=True)
print(f"Average Score: {result.avg_score}/10")
Multiple Iterations
python
from praisonaiagents.eval import AccuracyEvaluator
# Run 5 times to check consistency
evaluator = AccuracyEvaluator(
agent=my_agent,
input_text="Explain AI briefly",
expected_output="AI is...",
num_iterations=5
)
result = evaluator.run(print_summary=True)
print(f"Min: {result.min_score}, Max: {result.max_score}")
print(f"Std Dev: {result.std_dev_score}")
Evaluate Output
python
from praisonaiagents.eval import AccuracyEvaluator
# Evaluate pre-generated output (no agent needed)
evaluator = AccuracyEvaluator(
input_text="What is Python?",
expected_output="A programming language"
)
result = evaluator.evaluate_output(
output="Python is a high-level programming language",
print_summary=True
)
from praisonaiagents.eval import PerformanceEvaluator
def my_function():
# Your code to benchmark
return expensive_operation()
evaluator = PerformanceEvaluator(
func=my_function,
num_iterations=50,
warmup_runs=5
)
result = evaluator.run(print_summary=True)
print(f"P95 Time: {result.p95_run_time:.3f}s")
Performance Metrics Available
Metric
Description
avg_run_time
Average execution time
min_run_time
Fastest execution
max_run_time
Slowest execution
std_dev_run_time
Standard deviation
median_run_time
Median execution time
p95_run_time
95th percentile
avg_memory_usage
Average memory (MB)
Reliability Evaluation
Verify that expected tools are called.
Diagram
Expected Tools
python
from praisonaiagents.eval import ReliabilityEvaluator
evaluator = ReliabilityEvaluator(
agent=my_agent,
input_text="Search for AI news",
expected_tools=["search_web", "summarize"]
)
result = evaluator.run(print_summary=True)
print(f"Status: {result.status}") # PASSED or FAILED
print(f"Pass Rate: {result.pass_rate}%")
Forbidden Tools
python
from praisonaiagents.eval import ReliabilityEvaluator
# Ensure certain tools are NOT called
evaluator = ReliabilityEvaluator(
agent=my_agent,
input_text="Answer from memory only",
forbidden_tools=["search_web", "database_query"]
)
result = evaluator.run(print_summary=True)
Pre-recorded
python
from praisonaiagents.eval import ReliabilityEvaluator
# Evaluate without running agent
evaluator = ReliabilityEvaluator(
agent=my_agent,
expected_tools=["search_web"]
)
result = evaluator.evaluate_tool_calls(
actual_tools=["search_web", "summarize"],
print_summary=True
)
Criteria Evaluation
Evaluate against custom criteria with numeric or binary scoring.
Diagram
Numeric Scoring
python
from praisonaiagents.eval import CriteriaEvaluator
evaluator = CriteriaEvaluator(
criteria="Response is helpful, accurate, and professional",
agent=my_agent,
input_text="How do I reset my password?",
scoring_type="numeric",
threshold=7.0
)
result = evaluator.run(print_summary=True)
print(f"Score: {result.avg_score}/10")
print(f"Passed: {result.all_passed}")
Binary Scoring
python
from praisonaiagents.eval import CriteriaEvaluator
evaluator = CriteriaEvaluator(
criteria="Response contains no harmful content",
agent=my_agent,
input_text="Tell me about safety",
scoring_type="binary"
)
result = evaluator.run(print_summary=True)
print(f"Passed: {result.all_passed}")
With Callback
python
from praisonaiagents.eval import CriteriaEvaluator
def on_fail(score):
print(f"⚠️ Failed: {score.reasoning}")
# Send alert, log, etc.
evaluator = CriteriaEvaluator(
criteria="Response is under 100 words",
agent=my_agent,
input_text="Explain quantum physics",
on_fail=on_fail
)
result = evaluator.run()
Evaluation Flow
Diagram
Async Evaluation
All evaluators support async execution:
python
import asyncio
from praisonaiagents.eval import AccuracyEvaluator
async def evaluate():
evaluator = AccuracyEvaluator(
agent=my_agent,
input_text="Hello",
expected_output="Hi there!"
)
result = await evaluator.run_async(print_summary=True)
return result
result = asyncio.run(evaluate())
Saving Results
Save evaluation results for later analysis:
python
from praisonaiagents.eval import AccuracyEvaluator
evaluator = AccuracyEvaluator(
agent=my_agent,
input_text="Test",
expected_output="Expected",
save_results_path="./eval_results/accuracy_{timestamp}.json"
)
result = evaluator.run()
# Results automatically saved to file
Evaluation Packages
Run multiple test cases as a batch:
Diagrampython
from praisonaiagents.eval import EvalPackage, EvalCase, Judge
# Define test cases
cases = [
EvalCase(name="math", input="2+2", expected="4"),
EvalCase(name="geography", input="Capital of France?", expected="Paris"),
EvalCase(name="greeting", input="Hello", expected="Hi"),
]
# Create package
package = EvalPackage(
name="Math and Geography Tests",
cases=cases
)
# Run cases with Judge
judge = Judge()
for case in package.cases:
result = judge.run(
agent=my_agent,
input=case.input,
expected=case.expected
)
print(f"{case.name}: {result.score}/10")