Skip to content
Overview History Stats Security
SkillBundle Find AI coding assistant skills, watch the ones you depend on, and see what changed.
© 2026 SkillBundle
Skill metadata from skills.sh and public GitHub repositories.
npx skills add wshobson/agents --skill llm-evaluation llm-evaluation | SkillBundle
Implement comprehensive evaluation strategies for LLM applications using automated metrics, human feedback, and benchmarking. Use when testing LLM performance, measuring AI application quality, or establishing evaluation frameworks.
npx skills add wshobson/agents --skill llm-evaluation
import
dataclass
from typing import Callable
import numpy as np
@dataclass
class Metric :
name: str
fn: Callable
@ staticmethod
def accuracy ():
return Metric( "accuracy" , calculate_accuracy)
@ staticmethod
def bleu ():
return Metric( "bleu" , calculate_bleu)
@ staticmethod
def bertscore ():
return Metric( "bertscore" , calculate_bertscore)
@ staticmethod
def custom (name: str , fn: Callable):
return Metric(name, fn)
class EvaluationSuite :
def __init__ (self, metrics: list[Metric]):
self .metrics = metrics
async def evaluate (self, model, test_cases: list[ dict ]) -> dict :
results = {m.name: [] for m in self .metrics}
for test in test_cases:
prediction = await model.predict(test[ "input" ])
for metric in self .metrics:
score = metric.fn(
prediction = prediction,
reference = test.get( "expected" ),
context = test.get( "context" )
)
results[metric.name].append(score)
return {
"metrics" : {k: np.mean(v) for k, v in results.items()},
"raw_scores" : results
}
# Usage
suite = EvaluationSuite([
Metric.accuracy(),
Metric.bleu(),
Metric.bertscore(),
Metric.custom( "groundedness" , check_groundedness)
])
test_cases = [
{
"input" : "What is the capital of France?" ,
"expected" : "Paris" ,
"context" : "France is a country in Europe. Paris is its capital."
},
]
results = await suite.evaluate( model = your_model, test_cases = test_cases)