LLM Evaluation in 2026: Why Your Benchmark Scores Don't Matter Anymore

# Context (from a research paper)
context = """
The study found that 67% of participants showed improvement
after 8 weeks of treatment. The control group showed 12% improvement.
"""

# LLM Output 1 (Low Faithfulness)
output_1 = """
The treatment was highly effective, with most patients showing
significant improvement within just 2 weeks.
"""
# Faithfulness Score: 0.3 - Hallucinated timeline (2 weeks vs 8 weeks)

# LLM Output 2 (High Faithfulness)
output_2 = """
67% of treatment group participants improved after 8 weeks,
compared to 12% in the control group.
"""
# Faithfulness Score: 0.95 - Accurate, grounded in context

question = "How do I reduce hallucinations in my RAG system?"

# Low Informativeness
response_low = """
To reduce hallucinations, you should improve your system and use
better retrieval methods.
"""

# High Informativeness
response_high = """
To reduce hallucinations in RAG systems:

1. Improve retrieval quality:
   - Use hybrid search (keyword + semantic)
   - Increase chunk overlap to 20-50 tokens
   - Implement reranking with a cross-encoder

2. Constrain generation:
   - Add "only use provided context" to system prompt
   - Set temperature to 0.1-0.3 for factual tasks
   - Use citation mechanisms to link claims to sources

3. Add verification:
   - Implement SelfCheckGPT for consistency
   - Use an LLM judge to verify faithfulness
   - Track hallucination metrics in production

Research shows hybrid RAG can reduce errors by 35-60%.
"""

query = "What is the return policy for electronics?"

# Low Relevance Retrieved Context
retrieved_docs = [
    "Our electronics department features the latest smartphones...",
    "Customer service hours are Monday-Friday 9am-5pm...",
    "Electronics require special recycling procedures..."
]
# Context Relevance: 0.2

# High Relevance Retrieved Context
retrieved_docs = [
    "Electronics can be returned within 30 days with original packaging...",
    "Defective electronics are covered by manufacturer warranty...",
    "Return process: 1) Bring item and receipt 2) Visit customer service..."
]
# Context Relevance: 0.9

# For tasks with ground truth
def accuracy_score(predicted, ground_truth):
    """
    Simple exact match for closed-ended questions
    """
    return 1.0 if predicted.strip().lower() == ground_truth.strip().lower() else 0.0

# For subjective tasks, use LLM-as-judge
def llm_judge_accuracy(question, predicted, reference):
    """
    Use a strong LLM to judge correctness
    """
    prompt = f"""
    Compare the predicted answer to the reference answer.
    Question: {question}
    Predicted: {predicted}
    Reference: {reference}

    Score accuracy from 0-1, where:
    0 = Completely wrong
    0.5 = Partially correct
    1.0 = Fully correct

    Return only the numeric score.
    """
    return call_llm_judge(prompt)

# Installation
# pip install deepeval

from deepeval import assert_test
from deepeval.test_case import LLMTestCase
from deepeval.metrics import FaithfulnessMetric, AnswerRelevancyMetric

def test_rag_response():
    """Test RAG system response quality"""

    # Define test case
    test_case = LLMTestCase(
        input="What is the capital of France?",
        actual_output="Paris is the capital of France, located in the north-central region.",
        retrieval_context=[
            "Paris is the capital and largest city of France.",
            "The city of Paris is located in north-central France."
        ]
    )

    # Define metrics
    faithfulness = FaithfulnessMetric(threshold=0.7)
    relevancy = AnswerRelevancyMetric(threshold=0.7)

    # Assert (fails test if below threshold)
    assert_test(test_case, [faithfulness, relevancy])

# Run with: pytest test_rag.py

from deepeval.metrics import BaseMetric
from deepeval.test_case import LLMTestCase

class CitationMetric(BaseMetric):
    """Custom metric to check if response includes citations"""

    def __init__(self, threshold: float = 0.5):
        self.threshold = threshold

    def measure(self, test_case: LLMTestCase):
        # Check if output contains citation markers like [1], [2]
        import re
        citations = re.findall(r'\[\d+\]', test_case.actual_output)

        # Calculate score based on citation density
        words = len(test_case.actual_output.split())
        citation_density = len(citations) / (words / 100)  # Citations per 100 words

        self.score = min(citation_density, 1.0)
        self.success = self.score >= self.threshold

        return self.score

# Usage
test_case = LLMTestCase(
    input="Explain quantum computing",
    actual_output="Quantum computing uses qubits [1]. Unlike classical bits [2]...",
)

metric = CitationMetric(threshold=0.5)
metric.measure(test_case)
print(f"Citation Score: {metric.score}")

# Installation
# pip install ragas

from ragas import evaluate
from ragas.metrics import (
    faithfulness,
    answer_relevancy,
    context_relevancy,
    context_recall
)
from datasets import Dataset

# Your RAG system outputs
data = {
    'question': [
        'What is the capital of France?',
        'Who wrote Romeo and Juliet?'
    ],
    'answer': [
        'Paris is the capital of France.',
        'William Shakespeare wrote Romeo and Juliet in the 1590s.'
    ],
    'contexts': [
        ['Paris is the capital and most populous city of France.'],
        ['Romeo and Juliet is a tragedy written by William Shakespeare early in his career.']
    ],
    'ground_truth': [
        'Paris',
        'William Shakespeare'
    ]
}

dataset = Dataset.from_dict(data)

# Evaluate
results = evaluate(
    dataset,
    metrics=[
        faithfulness,
        answer_relevancy,
        context_relevancy,
        context_recall
    ]
)

print(results)
# Output:
# {
#   'faithfulness': 0.95,
#   'answer_relevancy': 0.92,
#   'context_relevancy': 0.88,
#   'context_recall': 0.91
# }

# Installation
# pip install langsmith langchain

from langsmith import Client
from langchain.chat_models import ChatOpenAI
from langchain.prompts import ChatPromptTemplate

# Initialize LangSmith
client = Client()

# Create a chain
llm = ChatOpenAI(temperature=0)
prompt = ChatPromptTemplate.from_messages([
    ("system", "You are a helpful assistant."),
    ("user", "{question}")
])
chain = prompt | llm

# Run with automatic tracing
result = chain.invoke(
    {"question": "What is the capital of France?"},
    config={"run_name": "capital_question"}
)

# Evaluate with LangSmith
from langsmith.evaluation import evaluate

def correctness_evaluator(run, example):
    """Custom evaluator function"""
    predicted = run.outputs["output"]
    expected = example.outputs["answer"]
    return {"score": 1.0 if expected.lower() in predicted.lower() else 0.0}

# Run evaluation on a dataset
evaluate(
    lambda x: chain.invoke(x),
    data="my_evaluation_dataset",
    evaluators=[correctness_evaluator],
    experiment_prefix="rag_eval_v1"
)

pip install deepeval

import pytest
from deepeval import assert_test
from deepeval.test_case import LLMTestCase
from deepeval.metrics import (
    FaithfulnessMetric,
    AnswerRelevancyMetric,
    HallucinationMetric
)

# Simulate your LLM system
def my_rag_system(question):
    """
    Your RAG system here. This is a mock for demonstration.
    In practice, replace with actual RAG pipeline.
    """
    if "capital" in question.lower() and "france" in question.lower():
        return {
            "answer": "Paris is the capital of France.",
            "context": ["Paris is the capital and most populous city of France."]
        }
    return {"answer": "I don't know.", "context": []}

@pytest.mark.parametrize(
    "question,expected_keywords",
    [
        ("What is the capital of France?", ["Paris", "capital"]),
        ("Who is the president of France?", ["president"]),
    ]
)
def test_rag_quality(question, expected_keywords):
    """Test that RAG responses are faithful and relevant"""

    # Get response from your system
    response = my_rag_system(question)

    # Create test case
    test_case = LLMTestCase(
        input=question,
        actual_output=response["answer"],
        retrieval_context=response["context"]
    )

    # Define metrics with thresholds
    faithfulness = FaithfulnessMetric(
        threshold=0.7,
        model="gpt-3.5-turbo"  # LLM to use for evaluation
    )

    relevancy = AnswerRelevancyMetric(
        threshold=0.7,
        model="gpt-3.5-turbo"
    )

    hallucination = HallucinationMetric(
        threshold=0.3  # Lower is better for hallucination
    )

    # Run evaluation
    assert_test(test_case, [faithfulness, relevancy, hallucination])

    # Check if expected keywords are present
    for keyword in expected_keywords:
        assert keyword.lower() in response["answer"].lower(), \
            f"Expected keyword '{keyword}' not found in response"

# Run with: pytest test_llm.py -v

# Run all tests
pytest test_llm.py -v

# Run with detailed output
deepeval test run test_llm.py

# Generate HTML report
deepeval test run test_llm.py --output report.html

# .github/workflows/llm-tests.yml
name: LLM Evaluation Tests

on: [push, pull_request]

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Install dependencies
        run: |
          pip install deepeval pytest
          pip install -r requirements.txt

      - name: Run LLM evaluation tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          pytest test_llm.py -v --tb=short

      - name: Generate evaluation report
        if: always()
        run: |
          deepeval test run test_llm.py --output evaluation-report.html

      - name: Upload report
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: evaluation-report
          path: evaluation-report.html

# Traditional Software
def divide(a, b):
    return a / b

result = divide(10, 0)
# CRASH: ZeroDivisionError - Clear, immediate, debuggable

# LLM Application
def answer_question(question):
    return llm.generate(question)

result = answer_question("What is 2+2?")
# Returns: "2+2 equals 5"
# NO ERROR - Silent failure, plausible but wrong

from langsmith import Client
from deepeval.metrics import FaithfulnessMetric
import time

class ProductionRAGSystem:
    """RAG system with built-in observability"""

    def __init__(self):
        self.langsmith = Client()  # For tracing
        self.metrics = {
            'faithfulness': FaithfulnessMetric(threshold=0.7)
        }
        self.cost_tracker = CostTracker()
        self.latency_tracker = LatencyTracker()

    def query(self, question: str, user_id: str):
        """Process query with full observability"""

        # Start trace
        with self.langsmith.trace(
            name="rag_query",
            metadata={"user_id": user_id}
        ) as trace:
            start_time = time.time()

            # Retrieve context
            context = self.retrieve(question)
            trace.log("Retrieved chunks", {"count": len(context)})

            # Generate response
            response = self.generate(question, context)

            # Track latency
            latency = time.time() - start_time
            self.latency_tracker.record(latency)

            # Track cost
            cost = self.cost_tracker.calculate(response.tokens)

            # Evaluate quality (async in production)
            quality_score = self.evaluate_async(question, response, context)

            # Log metrics
            trace.log("Metrics", {
                "latency_ms": latency * 1000,
                "cost_usd": cost,
                "quality_score": quality_score,
                "tokens": response.tokens
            })

            # Alert if quality drops
            if quality_score < 0.6:
                self.alert("Low quality response detected", {
                    "question": question,
                    "score": quality_score
                })

            return response

class CostTracker:
    """Track token usage and costs"""

    def __init__(self):
        self.costs = {
            "gpt-4": {"input": 0.03, "output": 0.06},  # per 1K tokens
            "gpt-3.5-turbo": {"input": 0.001, "output": 0.002}
        }

    def calculate(self, tokens, model="gpt-4"):
        """Calculate cost for token usage"""
        input_cost = (tokens.input / 1000) * self.costs[model]["input"]
        output_cost = (tokens.output / 1000) * self.costs[model]["output"]
        total = input_cost + output_cost

        # Log to monitoring system
        self.log_to_datadog(total, tokens)

        return total

# 1. Latency Distribution
latency_p50 = 120ms   # Median response time
latency_p95 = 450ms   # 95th percentile
latency_p99 = 890ms   # 99th percentile (catch outliers)

# 2. Cost Metrics
daily_cost = $127.34
cost_per_request = $0.023
token_efficiency = 0.78  # Ratio of useful to total tokens

# 3. Quality Metrics
hallucination_rate = 0.03  # 3% of responses
faithfulness_score = 0.89
user_feedback_positive = 0.92

# 4. System Health
error_rate = 0.001  # 0.1% errors
timeout_rate = 0.005  # 0.5% timeouts
cache_hit_rate = 0.67  # 67% served from cache

# alert_config.py
ALERT_THRESHOLDS = {
    "quality": {
        "faithfulness_score": {
            "min": 0.7,
            "severity": "high"
        },
        "hallucination_rate": {
            "max": 0.05,  # Alert if >5% hallucinations
            "severity": "critical"
        }
    },
    "performance": {
        "latency_p95": {
            "max": 500,  # Alert if p95 >500ms
            "severity": "medium"
        }
    },
    "cost": {
        "hourly_spend": {
            "max": 10.0,  # Alert if >$10/hour
            "severity": "high"
        }
    }
}

def check_alerts(metrics):
    """Check if metrics exceed thresholds"""
    alerts = []

    if metrics['faithfulness'] < ALERT_THRESHOLDS['quality']['faithfulness_score']['min']:
        alerts.append({
            "type": "quality",
            "message": f"Faithfulness dropped to {metrics['faithfulness']}",
            "severity": "high"
        })

    if metrics['hourly_cost'] > ALERT_THRESHOLDS['cost']['hourly_spend']['max']:
        alerts.append({
            "type": "cost",
            "message": f"Hourly cost ${metrics['hourly_cost']:.2f} exceeds threshold",
            "severity": "high"
        })

    return alerts

from deepeval.metrics import HallucinationMetric, FaithfulnessMetric

# Test on 100 real user queries
test_cases = load_production_queries(limit=100)

hallucination_metric = HallucinationMetric(threshold=0.3)
faithfulness_metric = FaithfulnessMetric(threshold=0.7)

results = []
for case in test_cases:
    hallucination_score = hallucination_metric.measure(case)
    faithfulness_score = faithfulness_metric.measure(case)
    results.append({
        'hallucination': hallucination_score,
        'faithfulness': faithfulness_score
    })

# Analysis
avg_hallucination = sum(r['hallucination'] for r in results) / len(results)
avg_faithfulness = sum(r['faithfulness'] for r in results) / len(results)

print(f"Hallucination Rate: {avg_hallucination:.2%}")  # 23% ❌
print(f"Faithfulness Score: {avg_faithfulness:.2f}")   # 0.54 ❌

# 1. Improved retrieval with hybrid search
from langchain.retrievers import EnsembleRetriever
from langchain.vectorstores import FAISS
from langchain.retrievers import BM25Retriever

# Combine semantic + keyword search
vector_retriever = FAISS.from_documents(docs, embeddings)
keyword_retriever = BM25Retriever.from_documents(docs)

ensemble = EnsembleRetriever(
    retrievers=[vector_retriever, keyword_retriever],
    weights=[0.6, 0.4]  # 60% semantic, 40% keyword
)

# 2. Constrained generation prompt
prompt = """
You are a financial document assistant.

CRITICAL RULES:
1. ONLY use information explicitly stated in the context below
2. If the answer is not in the context, say "Information not found"
3. Include citation numbers [1], [2] for each claim
4. Never infer, estimate, or extrapolate numbers

Context:
{context}

Question: {question}

Answer with citations:
"""

# 3. Post-generation verification
def verify_response(response, context):
    """Verify each claim in response against context"""
    verifier = HallucinationMetric()
    score = verifier.measure(response, context)

    if score > 0.1:  # >10% hallucination
        return "I cannot provide a confident answer based on the available documents."

    return response

# Don't just test one metric
evaluation_suite = [
    FaithfulnessMetric(threshold=0.7),      # Anti-hallucination
    AnswerRelevancyMetric(threshold=0.7),   # Usefulness
    ContextRelevancyMetric(threshold=0.6),  # Retrieval quality
    LatencyMetric(max_latency=500),         # Performance
    CostMetric(max_cost_per_query=0.05)     # Economics
]

# Bad: Synthetic test data
test_data = [
    "What is X?",
    "How does Y work?",
    "Explain Z"
]

# Good: Real production queries
test_data = load_production_queries(
    date_range="last_30_days",
    sample_size=1000,
    include_edge_cases=True
)

# Track versions for reproducibility
evaluation_run = {
    "timestamp": "2026-02-15T10:30:00Z",
    "model": "gpt-4-0125-preview",
    "prompt_version": "v2.3",
    "retrieval_config": {
        "chunk_size": 512,
        "overlap": 50,
        "top_k": 5
    },
    "metrics_version": "deepeval==0.20.1",
    "results": results
}

# Save for traceability
save_evaluation_run(evaluation_run)

# Catch degradation before deployment
def test_no_regression():
    """Ensure new version doesn't degrade performance"""

    # Baseline from production
    baseline_scores = load_baseline_scores()

    # Current version
    current_scores = run_evaluation()

    # Check each metric
    for metric in baseline_scores:
        current = current_scores[metric]
        baseline = baseline_scores[metric]

        # Allow 5% degradation, but flag it
        threshold = baseline * 0.95

        assert current >= threshold, \
            f"{metric} regressed: {current:.3f} < {baseline:.3f}"

# Problem: Using GPT-4 to evaluate GPT-4
evaluator_model = "gpt-4"
test_model = "gpt-4"

# Solution: Use different models or add human validation
evaluator_model = "claude-3-opus"  # Different family
test_model = "gpt-4"

# And sample for human review
if random.random() < 0.1:  # 10% sample
    flag_for_human_review(test_case)

# Don't forget: Evaluation itself costs money!

# Bad: Evaluate every response in production
for response in all_responses:
    evaluate(response)  # Doubles your API costs!

# Good: Sample + async + batching
if random.random() < 0.05:  # 5% sample
    async_evaluate_batch(response)  # Batch for efficiency

# Bad: One threshold for all scenarios
faithfulness_threshold = 0.7  # Always

# Good: Context-aware thresholds
def get_threshold(query_type):
    if query_type == "financial_data":
        return 0.95  # High stakes = high threshold
    elif query_type == "general_info":
        return 0.7   # Lower stakes = lower threshold
    elif query_type == "creative_writing":
        return 0.5   # Creativity valued over faithfulness