uv add gaussia
uv add langchain-openai  # Or your preferred LLM provider

Batch(qa_id="q1", query="...", assistant="...", ground_truth_assistant="...", weight=0.5),
Batch(qa_id="q2", query="...", assistant="...", ground_truth_assistant="...", weight=0.3),
Batch(qa_id="q3", query="...", assistant="...", ground_truth_assistant="...", weight=0.2),

class ConversationalMetric(BaseMetric):
    session_id: str
    assistant_id: str
    n_interactions: int
    conversational_memory: ConversationalScore
    conversational_language: ConversationalScore
    conversational_quality_maxim: ConversationalScore
    conversational_quantity_maxim: ConversationalScore
    conversational_relation_maxim: ConversationalScore
    conversational_manner_maxim: ConversationalScore
    conversational_sensibleness: ConversationalScore
    interactions: list[ConversationalInteraction]

class ConversationalScore(BaseModel):
    mean: float           # Session-level weighted mean
    ci_low: float | None  # Lower credible bound — Bayesian mode only
    ci_high: float | None # Upper credible bound — Bayesian mode only

class ConversationalInteraction(BaseModel):
    qa_id: str
    memory: float
    language: float
    quality_maxim: float
    quantity_maxim: float
    relation_maxim: float
    manner_maxim: float
    sensibleness: float

High (8-10): "The capital of France is Paris." (Verifiable fact)
Low  (0-4):  "France doesn't have a capital." (False)

High (8-10): "Paris is the capital of France."
Low  (0-4):  "Paris." (Too brief) or a 3-paragraph essay (Too much)

High (8-10): Q: "What's your return policy?" A: "Returns accepted within 30 days."
Low  (0-4):  Q: "What's your return policy?" A: "Our company was founded in 1998."

High (8-10): "You can return items within 30 days at any store location."
Low  (0-4):  "So basically, if you want to, you could possibly maybe return the thing..."

import os
from gaussia.metrics.conversational import Conversational
from gaussia.statistical import BayesianMode
from gaussia.core.retriever import Retriever
from gaussia.schemas.common import Dataset, Batch
from langchain_openai import ChatOpenAI

class ConversationalRetriever(Retriever):
    def load_dataset(self) -> list[Dataset]:
        return [
            Dataset(
                session_id="conv-eval-001",
                assistant_id="support-bot",
                language="english",
                context="You are a helpful, professional customer service assistant.",
                conversation=[
                    Batch(
                        qa_id="q1",
                        query="Hi, I need help with my order.",
                        assistant="Hello! I'd be happy to help. Could you share your order number?",
                        ground_truth_assistant="Greet and ask for order number.",
                        observation="Opening interaction - should be professional and helpful",
                    ),
                    Batch(
                        qa_id="q2",
                        query="It's ORDER-12345. I haven't received it yet.",
                        assistant="Thank you! ORDER-12345 was shipped Monday, expected Friday.",
                        ground_truth_assistant="Find order, provide shipping status and ETA.",
                    ),
                    Batch(
                        qa_id="q3",
                        query="Can you change the delivery address?",
                        assistant="For security, please confirm your email address first.",
                        ground_truth_assistant="Offer to help, verify identity first.",
                    ),
                ]
            )
        ]

judge = ChatOpenAI(model="gpt-4o-mini", api_key=os.getenv("OPENAI_API_KEY"), temperature=0.0)

metrics = Conversational.run(
    ConversationalRetriever,
    model=judge,
    statistical_mode=BayesianMode(mc_samples=5000, ci_level=0.95),
    use_structured_output=True,
    verbose=True,
)

for metric in metrics:
    print(f"Session: {metric.session_id}  ({metric.n_interactions} interactions)")
    print()

    dimensions = [
        ("Quality",      metric.conversational_quality_maxim),
        ("Quantity",     metric.conversational_quantity_maxim),
        ("Relation",     metric.conversational_relation_maxim),
        ("Manner",       metric.conversational_manner_maxim),
        ("Memory",       metric.conversational_memory),
        ("Language",     metric.conversational_language),
        ("Sensibleness", metric.conversational_sensibleness),
    ]

    for name, score in dimensions:
        ci = f"  [{score.ci_low:.1f}, {score.ci_high:.1f}]" if score.ci_low is not None else ""
        print(f"  {name:<14} {score.mean:.1f}/10{ci}")