Skip to content

Metrics

DiscreteMetric dataclass

DiscreteMetric(
    name: str,
    allowed_values: List[str] = (
        lambda: ["pass", "fail"]
    )(),
    prompt: Optional[Union[str, "Prompt"]] = None,
)

Bases: SimpleLLMMetric, DiscreteValidator

Metric for categorical/discrete evaluations with predefined allowed values.

This class is used for metrics that output categorical values like "pass/fail", "good/bad/excellent", or custom discrete categories. Uses the instructor library for structured LLM outputs.

Attributes:

Name Type Description
allowed_values List[str]

List of allowed categorical values the metric can output. Default is ["pass", "fail"].

prompt Optional[Union[str, Prompt]]

The prompt template for the metric. Should contain placeholders for evaluation inputs that will be formatted at runtime.

Examples:

>>> from ragas.metrics import DiscreteMetric
>>> from ragas.llms import llm_factory
>>> from openai import OpenAI
>>>
>>> # Create an LLM instance
>>> client = OpenAI(api_key="your-api-key")
>>> llm = llm_factory("<model>", client=client)
>>>
>>> # Create a custom discrete metric
>>> metric = DiscreteMetric(
...     name="quality_check",
...     prompt="Check the quality of the response: {response}. Return 'excellent', 'good', or 'poor'.",
...     allowed_values=["excellent", "good", "poor"]
... )
>>>
>>> # Score with the metric
>>> result = metric.score(
...     llm=llm,
...     response="This is a great response!"
... )
>>> print(result.value)  # Output: "excellent" or similar

get_correlation

get_correlation(
    gold_labels: List[str], predictions: List[str]
) -> float

Calculate the correlation between gold labels and predictions. This is a placeholder method and should be implemented based on the specific metric.

Source code in src/ragas/metrics/discrete.py
def get_correlation(
    self, gold_labels: t.List[str], predictions: t.List[str]
) -> float:
    """
    Calculate the correlation between gold labels and predictions.
    This is a placeholder method and should be implemented based on the specific metric.
    """
    try:
        from sklearn.metrics import cohen_kappa_score
    except ImportError:
        raise ImportError(
            "scikit-learn is required for correlation calculation. "
            "Please install it with `pip install scikit-learn`."
        )
    return cohen_kappa_score(gold_labels, predictions)

load classmethod

load(
    path: str,
    embedding_model: Optional[EmbeddingModelType] = None,
) -> DiscreteMetric

Load a DiscreteMetric saved by save().

Only narrows SimpleLLMMetric.load: the response model is built from allowed_values, so none is taken. embedding_model is needed if the saved prompt is a DynamicFewShotPrompt. Raises ValueError if the file holds another metric type.

Source code in src/ragas/metrics/discrete.py
@classmethod
def load(
    cls, path: str, embedding_model: t.Optional["EmbeddingModelType"] = None
) -> "DiscreteMetric":
    """Load a DiscreteMetric saved by ``save()``.

    Only narrows ``SimpleLLMMetric.load``: the response model is built
    from ``allowed_values``, so none is taken. ``embedding_model`` is
    needed if the saved prompt is a DynamicFewShotPrompt. Raises
    ``ValueError`` if the file holds another metric type.
    """
    return super().load(path, embedding_model=embedding_model)

NumericMetric dataclass

NumericMetric(
    name: str,
    allowed_values: Union[Tuple[float, float], range] = (
        0.0,
        1.0,
    ),
    prompt: Optional[Union[str, "Prompt"]] = None,
)

Bases: SimpleLLMMetric, NumericValidator

Metric for continuous numeric evaluations within a specified range.

This class is used for metrics that output numeric scores within a defined range, such as 0.0 to 1.0 for similarity scores or 1-10 ratings. Uses the instructor library for structured LLM outputs.

Attributes:

Name Type Description
allowed_values Union[Tuple[float, float], range]

The valid range for metric outputs. Can be a tuple of (min, max) floats or a range object. Default is (0.0, 1.0).

llm Optional[InstructorBaseRagasLLM]

The language model instance for evaluation. Can be created using llm_factory().

prompt Optional[Union[str, Prompt]]

The prompt template for the metric. Should contain placeholders for evaluation inputs that will be formatted at runtime.

Examples:

>>> from ragas.metrics import NumericMetric
>>> from ragas.llms import llm_factory
>>> from openai import OpenAI
>>>
>>> # Create an LLM instance
>>> client = OpenAI(api_key="your-api-key")
>>> llm = llm_factory("<model>", client=client)
>>>
>>> # Create a custom numeric metric with 0-10 range
>>> metric = NumericMetric(
...     name="quality_score",
...     llm=llm,
...     prompt="Rate the quality of this response on a scale of 0-10: {response}",
...     allowed_values=(0.0, 10.0)
... )
>>>
>>> # Score with the metric
>>> result = metric.score(
...     llm=llm,
...     response="This is a great response!"
... )
>>> print(result.value)  # Output: a float between 0.0 and 10.0

get_correlation

get_correlation(
    gold_labels: List[str], predictions: List[str]
) -> float

Calculate the correlation between gold labels and predictions. This is a placeholder method and should be implemented based on the specific metric.

Source code in src/ragas/metrics/numeric.py
def get_correlation(
    self, gold_labels: t.List[str], predictions: t.List[str]
) -> float:
    """
    Calculate the correlation between gold labels and predictions.
    This is a placeholder method and should be implemented based on the specific metric.
    """
    try:
        from scipy.stats import pearsonr
    except ImportError:
        raise ImportError(
            "scipy is required for correlation calculation. "
            "Please install it with `pip install scipy`."
        )
    # Convert strings to floats for correlation calculation
    gold_floats = [float(x) for x in gold_labels]
    pred_floats = [float(x) for x in predictions]
    result = pearsonr(gold_floats, pred_floats)
    # pearsonr returns (correlation, p-value) tuple
    correlation = t.cast(float, result[0])
    return correlation

load classmethod

load(
    path: str,
    embedding_model: Optional[EmbeddingModelType] = None,
) -> NumericMetric

Load a NumericMetric saved by save().

Only narrows SimpleLLMMetric.load: the response model is built from allowed_values, so none is taken. embedding_model is needed if the saved prompt is a DynamicFewShotPrompt. Raises ValueError if the file holds another metric type.

Source code in src/ragas/metrics/numeric.py
@classmethod
def load(
    cls, path: str, embedding_model: t.Optional["EmbeddingModelType"] = None
) -> "NumericMetric":
    """Load a NumericMetric saved by ``save()``.

    Only narrows ``SimpleLLMMetric.load``: the response model is built
    from ``allowed_values``, so none is taken. ``embedding_model`` is
    needed if the saved prompt is a DynamicFewShotPrompt. Raises
    ``ValueError`` if the file holds another metric type.
    """
    return super().load(path, embedding_model=embedding_model)

RankingMetric dataclass

RankingMetric(
    name: str,
    allowed_values: int = 2,
    prompt: Optional[Union[str, "Prompt"]] = None,
)

Bases: SimpleLLMMetric, RankingValidator

Metric for evaluations that produce ranked lists of items.

This class is used for metrics that output ordered lists, such as ranking search results, prioritizing features, or ordering responses by relevance. Uses the instructor library for structured LLM outputs.

Attributes:

Name Type Description
allowed_values int

Expected number of items in the ranking list. Default is 2.

llm Optional[InstructorBaseRagasLLM]

The language model instance for evaluation. Can be created using llm_factory().

prompt Optional[Union[str, Prompt]]

The prompt template for the metric. Should contain placeholders for evaluation inputs that will be formatted at runtime.

Examples:

>>> from ragas.metrics import RankingMetric
>>> from ragas.llms import llm_factory
>>> from openai import OpenAI
>>>
>>> # Create an LLM instance
>>> client = OpenAI(api_key="your-api-key")
>>> llm = llm_factory("<model>", client=client)
>>>
>>> # Create a ranking metric that returns top 3 items
>>> metric = RankingMetric(
...     name="relevance_ranking",
...     llm=llm,
...     prompt="Rank these results by relevance: {results}",
...     allowed_values=3
... )
>>>
>>> # Score with the metric
>>> result = metric.score(
...     llm=llm,
...     results="result1, result2, result3"
... )
>>> print(result.value)  # Output: a list of 3 ranked items

get_correlation

get_correlation(
    gold_labels: List[str], predictions: List[str]
) -> float

Calculate the correlation between gold labels and predictions. This is a placeholder method and should be implemented based on the specific metric.

Source code in src/ragas/metrics/ranking.py
def get_correlation(
    self, gold_labels: t.List[str], predictions: t.List[str]
) -> float:
    """
    Calculate the correlation between gold labels and predictions.
    This is a placeholder method and should be implemented based on the specific metric.
    """
    try:
        from sklearn.metrics import cohen_kappa_score
    except ImportError:
        raise ImportError(
            "scikit-learn is required for correlation calculation. "
            "Please install it with `pip install scikit-learn`."
        )

    kappa_scores = []
    for gold_item, prediction in zip(gold_labels, predictions):
        kappa = cohen_kappa_score(gold_item, prediction, weights="quadratic")
        kappa_scores.append(kappa)

    return sum(kappa_scores) / len(kappa_scores) if kappa_scores else 0.0

load classmethod

load(
    path: str,
    embedding_model: Optional[EmbeddingModelType] = None,
) -> RankingMetric

Load a RankingMetric saved by save().

Only narrows SimpleLLMMetric.load: the response model is built from allowed_values, so none is taken. embedding_model is needed if the saved prompt is a DynamicFewShotPrompt. Raises ValueError if the file holds another metric type.

Source code in src/ragas/metrics/ranking.py
@classmethod
def load(
    cls, path: str, embedding_model: t.Optional["EmbeddingModelType"] = None
) -> "RankingMetric":
    """Load a RankingMetric saved by ``save()``.

    Only narrows ``SimpleLLMMetric.load``: the response model is built
    from ``allowed_values``, so none is taken. ``embedding_model`` is
    needed if the saved prompt is a DynamicFewShotPrompt. Raises
    ``ValueError`` if the file holds another metric type.
    """
    return super().load(path, embedding_model=embedding_model)

MetricResult

MetricResult(
    value: Any,
    reason: Optional[str] = None,
    traces: Optional[Dict[str, Any]] = None,
)

Class to hold the result of a metric evaluation.

This class behaves like its underlying result value but still provides access to additional metadata like reasoning.

Works with: - DiscreteMetrics (string results) - NumericMetrics (float/int results) - RankingMetrics (list results)

Source code in src/ragas/metrics/result.py
def __init__(
    self,
    value: t.Any,
    reason: t.Optional[str] = None,
    traces: t.Optional[t.Dict[str, t.Any]] = None,
):
    if traces is not None:
        invalid_keys = [
            key for key in traces.keys() if key not in {"input", "output"}
        ]
        if invalid_keys:
            raise ValueError(
                f"Invalid keys in traces: {invalid_keys}. Allowed keys are 'input' and 'output'."
            )
    self._value = value
    self.reason = reason
    self.traces = traces

value property

value

Get the raw result value.

to_dict

to_dict()

Convert the result to a dictionary.

Source code in src/ragas/metrics/result.py
def to_dict(self):
    """Convert the result to a dictionary."""
    return {"result": self._value, "reason": self.reason}

validate classmethod

validate(value: Any, info: ValidationInfo)

Provide compatibility with older Pydantic versions.

Source code in src/ragas/metrics/result.py
@classmethod
def validate(cls, value: t.Any, info: ValidationInfo):
    """Provide compatibility with older Pydantic versions."""
    if isinstance(value, MetricResult):
        return value
    return cls(value=value)

discrete_metric

discrete_metric(
    *,
    name: Optional[str] = None,
    allowed_values: Optional[List[str]] = None,
    **metric_params: Any,
) -> Callable[[Callable[..., Any]], DiscreteMetricProtocol]

Decorator for creating discrete/categorical metrics.

This decorator transforms a regular function into a DiscreteMetric instance that can be used for evaluation with predefined categorical outputs.

Parameters:

Name Type Description Default
name str

Name for the metric. If not provided, uses the function name.

None
allowed_values List[str]

List of allowed categorical values for the metric output. Default is ["pass", "fail"].

None
**metric_params Any

Additional parameters to pass to the metric initialization.

{}

Returns:

Type Description
Callable[[Callable[..., Any]], DiscreteMetricProtocol]

A decorator that transforms a function into a DiscreteMetric instance.

Examples:

>>> from ragas.metrics import discrete_metric
>>>
>>> @discrete_metric(name="sentiment", allowed_values=["positive", "neutral", "negative"])
>>> def sentiment_analysis(user_input: str, response: str) -> str:
...     '''Analyze sentiment of the response.'''
...     if "great" in response.lower() or "good" in response.lower():
...         return "positive"
...     elif "bad" in response.lower() or "poor" in response.lower():
...         return "negative"
...     return "neutral"
>>>
>>> result = sentiment_analysis(
...     user_input="How was your day?",
...     response="It was great!"
... )
>>> print(result.value)  # "positive"
Source code in src/ragas/metrics/discrete.py
def discrete_metric(
    *,
    name: t.Optional[str] = None,
    allowed_values: t.Optional[t.List[str]] = None,
    **metric_params: t.Any,
) -> t.Callable[[t.Callable[..., t.Any]], DiscreteMetricProtocol]:
    """
    Decorator for creating discrete/categorical metrics.

    This decorator transforms a regular function into a DiscreteMetric instance
    that can be used for evaluation with predefined categorical outputs.

    Parameters
    ----------
    name : str, optional
        Name for the metric. If not provided, uses the function name.
    allowed_values : List[str], optional
        List of allowed categorical values for the metric output.
        Default is ["pass", "fail"].
    **metric_params : Any
        Additional parameters to pass to the metric initialization.

    Returns
    -------
    Callable[[Callable[..., Any]], DiscreteMetricProtocol]
        A decorator that transforms a function into a DiscreteMetric instance.

    Examples
    --------
    >>> from ragas.metrics import discrete_metric
    >>>
    >>> @discrete_metric(name="sentiment", allowed_values=["positive", "neutral", "negative"])
    >>> def sentiment_analysis(user_input: str, response: str) -> str:
    ...     '''Analyze sentiment of the response.'''
    ...     if "great" in response.lower() or "good" in response.lower():
    ...         return "positive"
    ...     elif "bad" in response.lower() or "poor" in response.lower():
    ...         return "negative"
    ...     return "neutral"
    >>>
    >>> result = sentiment_analysis(
    ...     user_input="How was your day?",
    ...     response="It was great!"
    ... )
    >>> print(result.value)  # "positive"
    """
    if allowed_values is None:
        allowed_values = ["pass", "fail"]

    decorator_factory = create_metric_decorator()
    return decorator_factory(name=name, allowed_values=allowed_values, **metric_params)  # type: ignore[return-value]

numeric_metric

numeric_metric(
    *,
    name: Optional[str] = None,
    allowed_values: Optional[
        Union[Tuple[float, float], range]
    ] = None,
    **metric_params: Any,
) -> Callable[[Callable[..., Any]], NumericMetricProtocol]

Decorator for creating numeric/continuous metrics.

This decorator transforms a regular function into a NumericMetric instance that outputs continuous values within a specified range.

Parameters:

Name Type Description Default
name str

Name for the metric. If not provided, uses the function name.

None
allowed_values Union[Tuple[float, float], range]

The valid range for metric outputs as (min, max) tuple or range object. Default is (0.0, 1.0).

None
**metric_params Any

Additional parameters to pass to the metric initialization.

{}

Returns:

Type Description
Callable[[Callable[..., Any]], NumericMetricProtocol]

A decorator that transforms a function into a NumericMetric instance.

Examples:

>>> from ragas.metrics import numeric_metric
>>>
>>> @numeric_metric(name="relevance_score", allowed_values=(0.0, 1.0))
>>> def calculate_relevance(user_input: str, response: str) -> float:
...     '''Calculate relevance score between 0 and 1.'''
...     # Simple word overlap example
...     user_words = set(user_input.lower().split())
...     response_words = set(response.lower().split())
...     if not user_words:
...         return 0.0
...     overlap = len(user_words & response_words)
...     return overlap / len(user_words)
>>>
>>> result = calculate_relevance(
...     user_input="What is Python?",
...     response="Python is a programming language"
... )
>>> print(result.value)  # Numeric score between 0.0 and 1.0
Source code in src/ragas/metrics/numeric.py
def numeric_metric(
    *,
    name: t.Optional[str] = None,
    allowed_values: t.Optional[t.Union[t.Tuple[float, float], range]] = None,
    **metric_params: t.Any,
) -> t.Callable[[t.Callable[..., t.Any]], NumericMetricProtocol]:
    """
    Decorator for creating numeric/continuous metrics.

    This decorator transforms a regular function into a NumericMetric instance
    that outputs continuous values within a specified range.

    Parameters
    ----------
    name : str, optional
        Name for the metric. If not provided, uses the function name.
    allowed_values : Union[Tuple[float, float], range], optional
        The valid range for metric outputs as (min, max) tuple or range object.
        Default is (0.0, 1.0).
    **metric_params : Any
        Additional parameters to pass to the metric initialization.

    Returns
    -------
    Callable[[Callable[..., Any]], NumericMetricProtocol]
        A decorator that transforms a function into a NumericMetric instance.

    Examples
    --------
    >>> from ragas.metrics import numeric_metric
    >>>
    >>> @numeric_metric(name="relevance_score", allowed_values=(0.0, 1.0))
    >>> def calculate_relevance(user_input: str, response: str) -> float:
    ...     '''Calculate relevance score between 0 and 1.'''
    ...     # Simple word overlap example
    ...     user_words = set(user_input.lower().split())
    ...     response_words = set(response.lower().split())
    ...     if not user_words:
    ...         return 0.0
    ...     overlap = len(user_words & response_words)
    ...     return overlap / len(user_words)
    >>>
    >>> result = calculate_relevance(
    ...     user_input="What is Python?",
    ...     response="Python is a programming language"
    ... )
    >>> print(result.value)  # Numeric score between 0.0 and 1.0
    """
    if allowed_values is None:
        allowed_values = (0.0, 1.0)

    decorator_factory = create_metric_decorator()
    return decorator_factory(name=name, allowed_values=allowed_values, **metric_params)  # type: ignore[return-value]

ranking_metric

ranking_metric(
    *,
    name: Optional[str] = None,
    allowed_values: Optional[int] = None,
    **metric_params: Any,
) -> Callable[[Callable[..., Any]], RankingMetricProtocol]

Decorator for creating ranking/ordering metrics.

This decorator transforms a regular function into a RankingMetric instance that outputs ordered lists of items.

Parameters:

Name Type Description Default
name str

Name for the metric. If not provided, uses the function name.

None
allowed_values int

Expected number of items in the ranking list. Default is 2.

None
**metric_params Any

Additional parameters to pass to the metric initialization.

{}

Returns:

Type Description
Callable[[Callable[..., Any]], RankingMetricProtocol]

A decorator that transforms a function into a RankingMetric instance.

Examples:

>>> from ragas.metrics import ranking_metric
>>>
>>> @ranking_metric(name="priority_ranker", allowed_values=3)
>>> def rank_by_urgency(user_input: str, responses: list) -> list:
...     '''Rank responses by urgency keywords.'''
...     urgency_keywords = ["urgent", "asap", "critical"]
...     scored = []
...     for resp in responses:
...         score = sum(kw in resp.lower() for kw in urgency_keywords)
...         scored.append((score, resp))
...     # Sort by score descending and return top items
...     ranked = sorted(scored, key=lambda x: x[0], reverse=True)
...     return [item[1] for item in ranked[:3]]
>>>
>>> result = rank_by_urgency(
...     user_input="What should I do first?",
...     responses=["This is urgent", "Take your time", "Critical issue!"]
... )
>>> print(result.value)  # Ranked list of responses
Source code in src/ragas/metrics/ranking.py
def ranking_metric(
    *,
    name: t.Optional[str] = None,
    allowed_values: t.Optional[int] = None,
    **metric_params: t.Any,
) -> t.Callable[[t.Callable[..., t.Any]], RankingMetricProtocol]:
    """
    Decorator for creating ranking/ordering metrics.

    This decorator transforms a regular function into a RankingMetric instance
    that outputs ordered lists of items.

    Parameters
    ----------
    name : str, optional
        Name for the metric. If not provided, uses the function name.
    allowed_values : int, optional
        Expected number of items in the ranking list. Default is 2.
    **metric_params : Any
        Additional parameters to pass to the metric initialization.

    Returns
    -------
    Callable[[Callable[..., Any]], RankingMetricProtocol]
        A decorator that transforms a function into a RankingMetric instance.

    Examples
    --------
    >>> from ragas.metrics import ranking_metric
    >>>
    >>> @ranking_metric(name="priority_ranker", allowed_values=3)
    >>> def rank_by_urgency(user_input: str, responses: list) -> list:
    ...     '''Rank responses by urgency keywords.'''
    ...     urgency_keywords = ["urgent", "asap", "critical"]
    ...     scored = []
    ...     for resp in responses:
    ...         score = sum(kw in resp.lower() for kw in urgency_keywords)
    ...         scored.append((score, resp))
    ...     # Sort by score descending and return top items
    ...     ranked = sorted(scored, key=lambda x: x[0], reverse=True)
    ...     return [item[1] for item in ranked[:3]]
    >>>
    >>> result = rank_by_urgency(
    ...     user_input="What should I do first?",
    ...     responses=["This is urgent", "Take your time", "Critical issue!"]
    ... )
    >>> print(result.value)  # Ranked list of responses
    """
    if allowed_values is None:
        allowed_values = 2

    decorator_factory = create_metric_decorator()
    return decorator_factory(name=name, allowed_values=allowed_values, **metric_params)  # type: ignore[return-value]

Collections of metrics using modern component architecture.

BleuScore

BleuScore(
    name: str = "bleu_score",
    kwargs: Optional[Dict[str, Any]] = None,
    **base_kwargs,
)

Bases: BaseMetric

Calculate BLEU score between reference and response texts.

This implementation provides automatic validation and pure async design without requiring LLM or embedding components. Uses sacrebleu library.

Usage: >>> from ragas.metrics.collections import BleuScore >>> >>> metric = BleuScore() >>> >>> result = await metric.ascore( ... reference="The capital of France is Paris.", ... response="Paris is the capital of France." ... ) >>> print(f"Score: {result.value}") >>> >>> results = await metric.abatch_score([ ... {"reference": "Text 1", "response": "Response 1"}, ... {"reference": "Text 2", "response": "Response 2"}, ... ])

Attributes: name: The metric name kwargs: Additional arguments to pass to sacrebleu.corpus_bleu, e.g. {"smooth_method": "exp"} for sentence-level smoothing of short texts. Without tokenize, Chinese text uses "zh" and Japanese "char" automatically; other text uses sacreBLEU's default "13a" allowed_values: Score range (0.0 to 1.0)

Source code in src/ragas/metrics/collections/_bleu_score.py
def __init__(
    self,
    name: str = "bleu_score",
    kwargs: t.Optional[t.Dict[str, t.Any]] = None,
    **base_kwargs,
):
    """Initialize BleuScore metric."""
    super().__init__(name=name, **base_kwargs)
    self.kwargs = kwargs or {}

ascore async

ascore(reference: str, response: str) -> MetricResult

Calculate BLEU score asynchronously.

Args: reference: The reference/ground truth text response: The response text to evaluate

Returns: MetricResult with BLEU score (0.0-1.0)

Source code in src/ragas/metrics/collections/_bleu_score.py
async def ascore(
    self,
    reference: str,
    response: str,
) -> MetricResult:
    """
    Calculate BLEU score asynchronously.

    Args:
        reference: The reference/ground truth text
        response: The response text to evaluate

    Returns:
        MetricResult with BLEU score (0.0-1.0)
    """
    try:
        from sacrebleu import corpus_bleu
    except ImportError:
        raise ImportError(
            "sacrebleu is required for BLEU score calculation. "
            "Please install it using `pip install sacrebleu`"
        )

    assert isinstance(reference, str), "BleuScore expects a valid reference string"
    assert isinstance(response, str), "BleuScore expects a valid response string"

    # One sample is one segment: the whole response against the whole
    # reference. corpus_bleu takes a list of hypotheses and a list of
    # reference *streams*, each stream holding one reference per hypothesis.
    # Splitting into sentences here made every reference sentence its own
    # stream for only the first response sentence, so the rest of the
    # response was never scored (#50).
    kwargs = dict(self.kwargs)
    if "tokenize" not in kwargs:
        # sacreBLEU's default 13a tokenizer splits on spaces, so text in a
        # script written without them is one "word" (#51). Japanese mixes
        # kanji and kana: score it per character. Chinese: sacreBLEU's zh.
        if has_kana(reference, response):
            kwargs["tokenize"] = "char"
        elif has_han(reference, response):
            kwargs["tokenize"] = "zh"
    score = corpus_bleu([response], [[reference]], **kwargs).score / 100

    assert isinstance(score, float), "Expecting a float"
    return MetricResult(value=float(score))

IDBasedContextPrecision

IDBasedContextPrecision(
    name: str = "id_based_context_precision", **base_kwargs
)

Bases: BaseMetric

Proportion of retrieved context IDs that are relevant.

Relevance is exact membership in the reference IDs -- no text comparison is involved, so this is a cheap alternative to the LLM-judged context precision metrics when your retriever returns stable identifiers.

Usage: >>> from ragas.metrics.collections import IDBasedContextPrecision >>> >>> metric = IDBasedContextPrecision() >>> >>> result = await metric.ascore( ... retrieved_context_ids=["doc1", "doc2", "doc3"], ... reference_context_ids=["doc1", "doc2"], ... ) >>> print(result.value) # 0.666...

Attributes: name: The metric name allowed_values: Score range (0.0 to 1.0)

Source code in src/ragas/metrics/collections/_context_ids.py
def __init__(self, name: str = "id_based_context_precision", **base_kwargs):
    super().__init__(name=name, **base_kwargs)

ascore async

ascore(
    retrieved_context_ids: List[Union[str, int]],
    reference_context_ids: List[Union[str, int]],
) -> MetricResult

Args: retrieved_context_ids: IDs the retriever returned reference_context_ids: IDs known to be relevant

Returns: MetricResult with the proportion of retrieved IDs that are relevant, or NaN when nothing was retrieved.

Source code in src/ragas/metrics/collections/_context_ids.py
async def ascore(
    self,
    retrieved_context_ids: t.List[t.Union[str, int]],
    reference_context_ids: t.List[t.Union[str, int]],
) -> MetricResult:
    """
    Args:
        retrieved_context_ids: IDs the retriever returned
        reference_context_ids: IDs known to be relevant

    Returns:
        MetricResult with the proportion of retrieved IDs that are relevant,
        or NaN when nothing was retrieved.
    """
    retrieved = _as_id_set(retrieved_context_ids)
    reference = _as_id_set(reference_context_ids)

    if not retrieved:
        logger.warning(
            "No retrieved context IDs provided, cannot calculate precision."
        )
        return MetricResult(value=float(np.nan))

    hits = len(retrieved & reference)
    return MetricResult(value=hits / len(retrieved))

IDBasedContextRecall

IDBasedContextRecall(
    name: str = "id_based_context_recall", **base_kwargs
)

Bases: BaseMetric

Proportion of reference context IDs that were retrieved.

The mirror of :class:IDBasedContextPrecision: precision asks how much of what you retrieved was useful, recall asks how much of what was useful you retrieved.

Usage: >>> from ragas.metrics.collections import IDBasedContextRecall >>> >>> metric = IDBasedContextRecall() >>> >>> result = await metric.ascore( ... retrieved_context_ids=["doc1", "doc2"], ... reference_context_ids=["doc1", "doc2", "doc3"], ... ) >>> print(result.value) # 0.666...

Attributes: name: The metric name allowed_values: Score range (0.0 to 1.0)

Source code in src/ragas/metrics/collections/_context_ids.py
def __init__(self, name: str = "id_based_context_recall", **base_kwargs):
    super().__init__(name=name, **base_kwargs)

ascore async

ascore(
    retrieved_context_ids: List[Union[str, int]],
    reference_context_ids: List[Union[str, int]],
) -> MetricResult

Args: retrieved_context_ids: IDs the retriever returned reference_context_ids: IDs known to be relevant

Returns: MetricResult with the proportion of reference IDs that were retrieved, or NaN when there are no reference IDs.

Source code in src/ragas/metrics/collections/_context_ids.py
async def ascore(
    self,
    retrieved_context_ids: t.List[t.Union[str, int]],
    reference_context_ids: t.List[t.Union[str, int]],
) -> MetricResult:
    """
    Args:
        retrieved_context_ids: IDs the retriever returned
        reference_context_ids: IDs known to be relevant

    Returns:
        MetricResult with the proportion of reference IDs that were
        retrieved, or NaN when there are no reference IDs.
    """
    retrieved = _as_id_set(retrieved_context_ids)
    reference = _as_id_set(reference_context_ids)

    if not reference:
        logger.warning(
            "No reference context IDs provided, cannot calculate recall."
        )
        return MetricResult(value=float(np.nan))

    hits = len(reference & retrieved)
    return MetricResult(value=hits / len(reference))

NonLLMContextPrecisionWithReference

NonLLMContextPrecisionWithReference(
    name: str = "non_llm_context_precision_with_reference",
    distance_measure: DistanceMeasure = LEVENSHTEIN,
    threshold: float = 0.5,
    **base_kwargs,
)

Bases: BaseMetric

Context precision judged by string distance against reference contexts.

Each retrieved context is matched against its closest reference context; a match at or above threshold counts as relevant. The verdicts are then scored with average precision, so relevant contexts ranked earlier score higher.

Usage: >>> from ragas.metrics.collections import NonLLMContextPrecisionWithReference >>> >>> metric = NonLLMContextPrecisionWithReference() >>> >>> result = await metric.ascore( ... retrieved_contexts=["Paris is the capital of France."], ... reference_contexts=["Paris is the capital of France."], ... ) >>> print(result.value) # 1.0

Attributes: name: The metric name distance_measure: Which string distance to use (default Levenshtein) threshold: Similarity at or above which a context counts as relevant allowed_values: Score range (0.0 to 1.0)

Source code in src/ragas/metrics/collections/_non_llm_context.py
def __init__(
    self,
    name: str = "non_llm_context_precision_with_reference",
    distance_measure: DistanceMeasure = DistanceMeasure.LEVENSHTEIN,
    threshold: float = 0.5,
    **base_kwargs,
):
    super().__init__(name=name, **base_kwargs)
    self.threshold = threshold
    self.distance_measure = distance_measure
    self._similarity = NonLLMStringSimilarity(distance_measure=distance_measure)

ascore async

ascore(
    retrieved_contexts: List[str],
    reference_contexts: List[str],
) -> MetricResult

Args: retrieved_contexts: Contexts the retriever returned, in rank order reference_contexts: Contexts known to be relevant

Returns: MetricResult with the average precision of the retrieved ranking.

Source code in src/ragas/metrics/collections/_non_llm_context.py
async def ascore(
    self,
    retrieved_contexts: t.List[str],
    reference_contexts: t.List[str],
) -> MetricResult:
    """
    Args:
        retrieved_contexts: Contexts the retriever returned, in rank order
        reference_contexts: Contexts known to be relevant

    Returns:
        MetricResult with the average precision of the retrieved ranking.
    """
    scores = [
        await _best_match(self._similarity, rc, reference_contexts)
        for rc in retrieved_contexts
    ]
    # A context at or above the threshold counts (same rule as recall, #64).
    verdicts = [1 if score >= self.threshold else 0 for score in scores]
    return MetricResult(value=average_precision(verdicts))

NonLLMContextRecall

NonLLMContextRecall(
    name: str = "non_llm_context_recall",
    distance_measure: DistanceMeasure = LEVENSHTEIN,
    threshold: float = 0.5,
    **base_kwargs,
)

Bases: BaseMetric

Context recall judged by string distance against reference contexts.

Each reference context is matched against its closest retrieved context; a match at or above threshold counts as recalled. The score is the proportion of reference contexts recalled.

Usage: >>> from ragas.metrics.collections import NonLLMContextRecall >>> >>> metric = NonLLMContextRecall() >>> >>> result = await metric.ascore( ... retrieved_contexts=["Paris is the capital of France."], ... reference_contexts=["Paris is the capital of France."], ... ) >>> print(result.value) # 1.0

Attributes: name: The metric name distance_measure: Which string distance to use (default Levenshtein) threshold: Similarity at or above which a context counts as recalled allowed_values: Score range (0.0 to 1.0)

Source code in src/ragas/metrics/collections/_non_llm_context.py
def __init__(
    self,
    name: str = "non_llm_context_recall",
    distance_measure: DistanceMeasure = DistanceMeasure.LEVENSHTEIN,
    threshold: float = 0.5,
    **base_kwargs,
):
    super().__init__(name=name, **base_kwargs)
    self.threshold = threshold
    self.distance_measure = distance_measure
    self._similarity = NonLLMStringSimilarity(distance_measure=distance_measure)

ascore async

ascore(
    retrieved_contexts: List[str],
    reference_contexts: List[str],
) -> MetricResult

Args: retrieved_contexts: Contexts the retriever returned reference_contexts: Contexts known to be relevant

Returns: MetricResult with the proportion of reference contexts recalled, or NaN when there are no reference contexts.

Source code in src/ragas/metrics/collections/_non_llm_context.py
async def ascore(
    self,
    retrieved_contexts: t.List[str],
    reference_contexts: t.List[str],
) -> MetricResult:
    """
    Args:
        retrieved_contexts: Contexts the retriever returned
        reference_contexts: Contexts known to be relevant

    Returns:
        MetricResult with the proportion of reference contexts recalled,
        or NaN when there are no reference contexts.
    """
    scores = [
        await _best_match(self._similarity, rc, retrieved_contexts)
        for rc in reference_contexts
    ]
    # `>=`, matching NonLLMContextPrecisionWithReference. Legacy used `>`
    # here, so an exact match at threshold=1.0 counted for precision but not
    # for recall (#64).
    verdicts = [1 if score >= self.threshold else 0 for score in scores]
    if not verdicts:
        return MetricResult(value=float(np.nan))
    return MetricResult(value=sum(verdicts) / len(verdicts))

RougeScore

RougeScore(
    name: str = "rouge_score",
    rouge_type: Literal["rouge1", "rougeL"] = "rougeL",
    mode: Literal[
        "fmeasure", "precision", "recall"
    ] = "fmeasure",
    tokenizer: Union[
        Literal["auto", "default", "unicode"], Any
    ] = "auto",
    use_stemmer: bool = True,
    **kwargs,
)

Bases: BaseMetric

Calculate ROUGE score between reference and response texts.

This implementation provides automatic validation and pure async design without requiring LLM or embedding components.

Usage: >>> from ragas.metrics.collections import RougeScore >>> >>> # Create metric instance (no LLM/embeddings needed) >>> metric = RougeScore(rouge_type="rougeL", mode="fmeasure") >>> >>> # Single evaluation >>> result = await metric.ascore( ... reference="The capital of France is Paris.", ... response="Paris is the capital of France." ... ) >>> print(f"Score: {result.value}") >>> >>> # Batch evaluation >>> results = await metric.abatch_score([ ... {"reference": "Text 1", "response": "Response 1"}, ... {"reference": "Text 2", "response": "Response 2"}, ... ])

Attributes: name: The metric name rouge_type: Type of ROUGE metric ("rouge1" for unigrams, "rougeL" for LCS) mode: Scoring mode ("fmeasure", "precision", or "recall") allowed_values: Score range (0.0 to 1.0)

Note: This metric doesn't define llm or embeddings fields, so no validation is performed.

Args: tokenizer: "auto" (default) uses rouge_score's own tokenizer when both texts are ASCII, so English scores are unchanged, and "unicode" otherwise. "default" always uses rouge_score's, which drops every non-[a-z0-9] character, so non-Latin text scores 0. "unicode" keeps words in any script and splits CJK into characters. Any object with a tokenize(text) method (e.g. a language-specific segmenter) is used as-is. use_stemmer: Porter-stem English tokens (as rouge_score does). Never applied to non-ASCII tokens.

Source code in src/ragas/metrics/collections/_rouge_score.py
def __init__(
    self,
    name: str = "rouge_score",
    rouge_type: t.Literal["rouge1", "rougeL"] = "rougeL",
    mode: t.Literal["fmeasure", "precision", "recall"] = "fmeasure",
    tokenizer: t.Union[t.Literal["auto", "default", "unicode"], t.Any] = "auto",
    use_stemmer: bool = True,
    **kwargs,
):
    """Initialize RougeScore metric.

    Args:
        tokenizer: ``"auto"`` (default) uses rouge_score's own tokenizer when
            both texts are ASCII, so English scores are unchanged, and
            ``"unicode"`` otherwise. ``"default"`` always uses rouge_score's,
            which drops every non-``[a-z0-9]`` character, so non-Latin text
            scores 0. ``"unicode"`` keeps words in any script and splits CJK
            into characters. Any object with a ``tokenize(text)`` method
            (e.g. a language-specific segmenter) is used as-is.
        use_stemmer: Porter-stem English tokens (as rouge_score does).
            Never applied to non-ASCII tokens.
    """
    super().__init__(name=name, **kwargs)
    self.rouge_type = rouge_type
    self.mode = mode
    self.tokenizer = tokenizer
    self.use_stemmer = use_stemmer

ascore async

ascore(reference: str, response: str) -> MetricResult

Calculate ROUGE score asynchronously.

Args: reference: The reference/ground truth text response: The response text to evaluate

Returns: MetricResult with ROUGE score (0.0-1.0)

Source code in src/ragas/metrics/collections/_rouge_score.py
async def ascore(
    self,
    reference: str,
    response: str,
) -> MetricResult:
    """
    Calculate ROUGE score asynchronously.

    Args:
        reference: The reference/ground truth text
        response: The response text to evaluate

    Returns:
        MetricResult with ROUGE score (0.0-1.0)
    """
    # Import and check dependencies
    try:
        from rouge_score import rouge_scorer
    except ImportError:
        raise ImportError(
            "rouge_score is required for ROUGE score calculation. "
            "Please install it using `pip install rouge_score`"
        )

    # Calculate ROUGE score
    scorer = rouge_scorer.RougeScorer(
        [self.rouge_type],
        use_stemmer=self.use_stemmer,
        tokenizer=self._tokenizer_for(reference, response),
    )
    scores = scorer.score(reference, response)
    score_value = getattr(scores[self.rouge_type], self.mode)

    return MetricResult(value=float(score_value))

SemanticSimilarity

SemanticSimilarity(
    embeddings: BaseRagasEmbedding,
    name: str = "semantic_similarity",
    threshold: Optional[float] = None,
    **kwargs,
)

Bases: BaseMetric

Evaluate semantic similarity between reference and response using embeddings.

Scores the semantic similarity of ground truth with generated answer using cosine similarity of embeddings. Based on the SAS paper: https://arxiv.org/pdf/2108.06130.pdf

Usage: >>> from openai import AsyncOpenAI >>> from ragas.embeddings.base import embedding_factory >>> from ragas.metrics.collections import SemanticSimilarity >>> >>> # Setup embeddings >>> client = AsyncOpenAI() >>> embeddings = embedding_factory("openai", model="text-embedding-ada-002", client=client, interface="modern") >>> >>> # Create metric instance >>> metric = SemanticSimilarity(embeddings=embeddings) >>> >>> # Single evaluation >>> result = await metric.ascore( ... reference="Paris is the capital of France.", ... response="The capital of France is Paris." ... ) >>> print(f"Score: {result.value}") >>> >>> # Batch evaluation >>> results = await metric.abatch_score([ ... {"reference": "Text 1", "response": "Response 1"}, ... {"reference": "Text 2", "response": "Response 2"}, ... ])

Attributes: embeddings: Modern embeddings model with embed_text() method name: The metric name threshold: Optional threshold for binary classification allowed_values: Score range (0.0 to 1.0)

Source code in src/ragas/metrics/collections/_semantic_similarity.py
def __init__(
    self,
    embeddings: "BaseRagasEmbedding",
    name: str = "semantic_similarity",
    threshold: t.Optional[float] = None,
    **kwargs,
):
    """Initialize SemanticSimilarity metric with required embeddings."""
    self.threshold = threshold

    super().__init__(embeddings=embeddings, name=name, **kwargs)

ascore async

ascore(reference: str, response: str) -> MetricResult

Calculate semantic similarity score asynchronously.

Components are guaranteed to be validated and non-None by the base class.

Args: reference: The reference/ground truth text response: The response text to evaluate

Returns: MetricResult with similarity score (0.0-1.0)

Source code in src/ragas/metrics/collections/_semantic_similarity.py
async def ascore(self, reference: str, response: str) -> MetricResult:
    """
    Calculate semantic similarity score asynchronously.

    Components are guaranteed to be validated and non-None by the base class.

    Args:
        reference: The reference/ground truth text
        response: The response text to evaluate

    Returns:
        MetricResult with similarity score (0.0-1.0)
    """
    reference = reference or " "
    response = response or " "

    # Async API: the sync embed_text blocked the event loop inside ascore.
    embedding_1 = np.array(await self.embeddings.aembed_text(reference))
    embedding_2 = np.array(await self.embeddings.aembed_text(response))

    similarity = np.array([cosine(embedding_1, embedding_2)])
    # Cosine similarity spans [-1, 1]; the metric is defined on [0, 1], and
    # unrelated texts can come out slightly negative with some embedding
    # models (#107). Same treatment as AnswerRelevancy (#56).
    score = np.clip(similarity.flatten(), 0.0, 1.0)

    assert isinstance(score, np.ndarray), "Expects ndarray"
    if self.threshold:
        score = score >= self.threshold

    return MetricResult(value=float(score.item()))

ExactMatch

ExactMatch(name: str = 'exact_match', **base_kwargs)

Bases: BaseMetric

Check if reference and response are exactly identical.

This implementation provides automatic validation and pure async design without requiring LLM or embedding components.

Usage: >>> from ragas.metrics.collections import ExactMatch >>> >>> metric = ExactMatch() >>> >>> result = await metric.ascore( ... reference="Hello World", ... response="Hello World" ... ) >>> print(f"Score: {result.value}") # 1.0 >>> >>> results = await metric.abatch_score([ ... {"reference": "Text 1", "response": "Text 1"}, ... {"reference": "Text 2", "response": "Different"}, ... ])

Attributes: name: The metric name allowed_values: Score range (0.0 to 1.0)

Source code in src/ragas/metrics/collections/_string.py
def __init__(
    self,
    name: str = "exact_match",
    **base_kwargs,
):
    """Initialize ExactMatch metric."""
    super().__init__(name=name, **base_kwargs)

ascore async

ascore(reference: str, response: str) -> MetricResult

Check if reference and response match exactly.

Args: reference: The reference/ground truth text response: The response text to evaluate

Returns: MetricResult with 1.0 if exact match, 0.0 otherwise

Source code in src/ragas/metrics/collections/_string.py
async def ascore(
    self,
    reference: str,
    response: str,
) -> MetricResult:
    """
    Check if reference and response match exactly.

    Args:
        reference: The reference/ground truth text
        response: The response text to evaluate

    Returns:
        MetricResult with 1.0 if exact match, 0.0 otherwise
    """
    score = float(reference == response)
    return MetricResult(value=score)

NonLLMStringSimilarity

NonLLMStringSimilarity(
    name: str = "non_llm_string_similarity",
    distance_measure: DistanceMeasure = LEVENSHTEIN,
    **base_kwargs,
)

Bases: BaseMetric

Calculate string similarity between reference and response using various distance measures.

This implementation provides automatic validation and pure async design without requiring LLM or embedding components. Uses rapidfuzz library.

Usage: >>> from ragas.metrics.collections import NonLLMStringSimilarity, DistanceMeasure >>> >>> metric = NonLLMStringSimilarity(distance_measure=DistanceMeasure.LEVENSHTEIN) >>> >>> result = await metric.ascore( ... reference="The capital of France is Paris.", ... response="Paris is the capital of France." ... ) >>> print(f"Score: {result.value}") >>> >>> results = await metric.abatch_score([ ... {"reference": "Text 1", "response": "Response 1"}, ... {"reference": "Text 2", "response": "Response 2"}, ... ])

Attributes: name: The metric name distance_measure: The distance measure to use (default: LEVENSHTEIN) allowed_values: Score range (0.0 to 1.0)

Source code in src/ragas/metrics/collections/_string.py
def __init__(
    self,
    name: str = "non_llm_string_similarity",
    distance_measure: DistanceMeasure = DistanceMeasure.LEVENSHTEIN,
    **base_kwargs,
):
    """Initialize NonLLMStringSimilarity metric."""
    super().__init__(name=name, **base_kwargs)
    self.distance_measure = distance_measure
    rapidfuzz_distances(DistanceMeasure)  # rapidfuzz is installed

ascore async

ascore(reference: str, response: str) -> MetricResult

Calculate string similarity score asynchronously.

Args: reference: The reference/ground truth text response: The response text to evaluate

Returns: MetricResult with similarity score (0.0-1.0)

Source code in src/ragas/metrics/collections/_string.py
async def ascore(
    self,
    reference: str,
    response: str,
) -> MetricResult:
    """
    Calculate string similarity score asynchronously.

    Args:
        reference: The reference/ground truth text
        response: The response text to evaluate

    Returns:
        MetricResult with similarity score (0.0-1.0)
    """
    assert isinstance(reference, str), (
        "NonLLMStringSimilarity expects a valid reference string"
    )
    assert isinstance(response, str), (
        "NonLLMStringSimilarity expects a valid response string"
    )

    score = 1 - self.distance_measure_map[
        self.distance_measure
    ].normalized_distance(reference, response)

    assert isinstance(score, float), "Expecting a float"
    return MetricResult(value=float(score))

StringPresence

StringPresence(name: str = 'string_present', **base_kwargs)

Bases: BaseMetric

Check if reference string is present in the response.

This implementation provides automatic validation and pure async design without requiring LLM or embedding components.

Usage: >>> from ragas.metrics.collections import StringPresence >>> >>> metric = StringPresence() >>> >>> result = await metric.ascore( ... reference="Paris", ... response="The capital of France is Paris." ... ) >>> print(f"Score: {result.value}") # 1.0 >>> >>> results = await metric.abatch_score([ ... {"reference": "cat", "response": "The cat sat on the mat"}, ... {"reference": "dog", "response": "The cat sat on the mat"}, ... ])

Attributes: name: The metric name allowed_values: Score range (0.0 to 1.0)

Source code in src/ragas/metrics/collections/_string.py
def __init__(
    self,
    name: str = "string_present",
    **base_kwargs,
):
    """Initialize StringPresence metric."""
    super().__init__(name=name, **base_kwargs)

ascore async

ascore(reference: str, response: str) -> MetricResult

Check if reference is present in response.

Args: reference: The reference string to search for response: The response text to search in

Returns: MetricResult with 1.0 if reference is in response, 0.0 otherwise

Source code in src/ragas/metrics/collections/_string.py
async def ascore(
    self,
    reference: str,
    response: str,
) -> MetricResult:
    """
    Check if reference is present in response.

    Args:
        reference: The reference string to search for
        response: The response text to search in

    Returns:
        MetricResult with 1.0 if reference is in response, 0.0 otherwise
    """
    assert isinstance(reference, str), (
        "StringPresence expects a valid reference string"
    )
    assert isinstance(response, str), (
        "StringPresence expects a valid response string"
    )

    score = float(reference in response)
    return MetricResult(value=score)

AgentGoalAccuracyWithoutReference

AgentGoalAccuracyWithoutReference(
    llm: InstructorBaseRagasLLM,
    name: str = "agent_goal_accuracy",
    **kwargs,
)

Bases: _AgentGoalAccuracyBase

Measures if an agent achieved the user's inferred goal.

This metric evaluates whether the final state of an agentic workflow matches what the user intended, without requiring a reference. It uses an LLM to: 1. Infer the user's goal from the conversation 2. Infer the end state from the conversation 3. Compare if the end state matches the inferred goal

This is a binary metric: 1.0 if the goal was achieved, 0.0 otherwise.

Usage: >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import AgentGoalAccuracyWithoutReference >>> from ragas.messages import HumanMessage, AIMessage, ToolMessage >>> >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> metric = AgentGoalAccuracyWithoutReference(llm=llm) >>> >>> result = await metric.ascore( ... user_input=[ ... HumanMessage(content="Book a table at a Chinese restaurant"), ... AIMessage(content="I'll search for restaurants...", tool_calls=[...]), ... ToolMessage(content="Found Golden Dragon"), ... AIMessage(content="Table booked at Golden Dragon for 8pm!"), ... ], ... ) >>> print(f"Goal Achieved: {result.value}")

Attributes: llm: Modern instructor-based LLM for goal inference and comparison name: The metric name

Source code in src/ragas/metrics/collections/agent_goal_accuracy/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "agent_goal_accuracy",
    **kwargs,
):
    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    user_input: List[
        Union[HumanMessage, AIMessage, ToolMessage]
    ],
) -> MetricResult

Calculate agent goal accuracy without a reference.

Args: user_input: List of conversation messages representing the workflow

Returns: MetricResult with binary score (1.0 if goal achieved, 0.0 otherwise)

Source code in src/ragas/metrics/collections/agent_goal_accuracy/metric.py
async def ascore(
    self,
    user_input: List[Union[HumanMessage, AIMessage, ToolMessage]],
) -> MetricResult:
    """
    Calculate agent goal accuracy without a reference.

    Args:
        user_input: List of conversation messages representing the workflow

    Returns:
        MetricResult with binary score (1.0 if goal achieved, 0.0 otherwise)
    """
    return await self._score(user_input, None)

AgentGoalAccuracyWithReference

AgentGoalAccuracyWithReference(
    llm: InstructorBaseRagasLLM,
    name: str = "agent_goal_accuracy",
    **kwargs,
)

Bases: _AgentGoalAccuracyBase

Measures if an agent achieved the user's goal compared to a reference outcome.

This metric evaluates whether the final state of an agentic workflow matches the expected reference outcome. It uses an LLM to: 1. Infer the end state from the conversation 2. Compare the end state against the provided reference

This is a binary metric: 1.0 if the goal was achieved, 0.0 otherwise.

Usage: >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import AgentGoalAccuracyWithReference >>> from ragas.messages import HumanMessage, AIMessage, ToolMessage >>> >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> metric = AgentGoalAccuracyWithReference(llm=llm) >>> >>> result = await metric.ascore( ... user_input=[ ... HumanMessage(content="Book a table at a Chinese restaurant"), ... AIMessage(content="I'll search for restaurants...", tool_calls=[...]), ... ToolMessage(content="Found Golden Dragon"), ... AIMessage(content="Table booked at Golden Dragon for 8pm!"), ... ], ... reference="Table booked at a Chinese restaurant", ... ) >>> print(f"Goal Achieved: {result.value}")

Attributes: llm: Modern instructor-based LLM for goal inference and comparison name: The metric name

Source code in src/ragas/metrics/collections/agent_goal_accuracy/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "agent_goal_accuracy",
    **kwargs,
):
    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    user_input: List[
        Union[HumanMessage, AIMessage, ToolMessage]
    ],
    reference: str,
) -> MetricResult

Calculate agent goal accuracy against a reference outcome.

Args: user_input: List of conversation messages representing the workflow reference: The expected/desired outcome

Returns: MetricResult with binary score (1.0 if goal achieved, 0.0 otherwise)

Source code in src/ragas/metrics/collections/agent_goal_accuracy/metric.py
async def ascore(
    self,
    user_input: List[Union[HumanMessage, AIMessage, ToolMessage]],
    reference: str,
) -> MetricResult:
    """
    Calculate agent goal accuracy against a reference outcome.

    Args:
        user_input: List of conversation messages representing the workflow
        reference: The expected/desired outcome

    Returns:
        MetricResult with binary score (1.0 if goal achieved, 0.0 otherwise)
    """
    require(reference=reference)
    return await self._score(user_input, reference)

AnswerAccuracy

AnswerAccuracy(
    llm: InstructorBaseRagasLLM,
    name: str = "answer_accuracy",
    max_retries: int = 5,
    **kwargs,
)

Bases: DualJudgeMetric

Answer Accuracy metric using dual-judge evaluation.

Measures answer accuracy compared to ground truth using a dual-judge system. This metric averages two distinct judge prompts to ensure robust evaluation.

The metric uses NVIDIA's proven dual-judge approach: 1. Judge 1: Direct User Answer vs Reference Answer comparison 2. Judge 2: Swapped perspective for fairness 3. Average both judges for final score

Rating scale: 0 (no match), 2 (partial match), 4 (exact match) Final score: Average of both judges converted to 0.0-1.0 scale

Usage: >>> import instructor >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import AnswerAccuracy >>> >>> # Setup dependencies >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance >>> metric = AnswerAccuracy(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... user_input="When was Einstein born?", ... response="Albert Einstein was born in 1879.", ... reference="Albert Einstein was born in 1879." ... ) >>> print(f"Answer Accuracy: {result.value}")

Attributes: llm: Modern instructor-based LLM for dual-judge evaluation name: The metric name allowed_values: Score range (0.0 to 1.0, higher is better) max_retries: Attempts per judge; a retry after an off-scale rating is a fresh sample, not a resend of the same request

name: The metric name
max_retries: Attempts per judge (see ``DualJudgeMetric``)
Source code in src/ragas/metrics/collections/answer_accuracy/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "answer_accuracy",
    max_retries: int = 5,
    **kwargs,
):
    """
    Args:
        llm: Modern instructor-based LLM for dual-judge evaluation
        name: The metric name
        max_retries: Attempts per judge (see ``DualJudgeMetric``)
    """
    super().__init__(llm=llm, name=name, max_retries=max_retries, **kwargs)

ascore async

ascore(
    user_input: str, response: str, reference: str
) -> MetricResult

Calculate answer accuracy score using dual-judge evaluation.

Args: user_input: The original question response: The user's answer to evaluate reference: The ground truth reference answer

Returns: MetricResult with answer accuracy score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/answer_accuracy/metric.py
async def ascore(
    self, user_input: str, response: str, reference: str
) -> MetricResult:
    """
    Calculate answer accuracy score using dual-judge evaluation.

    Args:
        user_input: The original question
        response: The user's answer to evaluate
        reference: The ground truth reference answer

    Returns:
        MetricResult with answer accuracy score (0.0-1.0, higher is better)
    """
    require(user_input=user_input, response=response, reference=reference)

    # Judge 2 sees the answers swapped, for fairness.
    return await self._judge_both(
        AnswerAccuracyInput(
            query=user_input, user_answer=response, reference_answer=reference
        ),
        AnswerAccuracyInput(
            query=user_input, user_answer=reference, reference_answer=response
        ),
    )

AnswerCorrectness

AnswerCorrectness(
    llm: InstructorBaseRagasLLM,
    embeddings: Optional[BaseRagasEmbedding] = None,
    name: str = "answer_correctness",
    weights: List[float] = [0.75, 0.25],
    beta: float = 1.0,
    **kwargs,
)

Bases: BaseMetric

Answer Correctness metric using multi-step pipeline evaluation.

Measures answer correctness as a weighted combination of: - Factuality: F1 score from statement-level TP/FP/FN classification - Similarity: Semantic similarity between answer and reference

This implementation uses modern instructor LLMs with structured output and modern embeddings. Only supports modern components - legacy wrappers are rejected with clear error messages.

Usage: >>> from openai import AsyncOpenAI >>> from ragas.llms import llm_factory >>> from ragas.embeddings.base import embedding_factory >>> from ragas.metrics.collections import AnswerCorrectness >>> >>> # Setup dependencies >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> embeddings = embedding_factory("openai", model="text-embedding-ada-002", client=client, interface="modern") >>> >>> # Create metric instance >>> metric = AnswerCorrectness(llm=llm, embeddings=embeddings) >>> >>> # Single evaluation >>> result = await metric.ascore( ... user_input="What is the capital of France?", ... response="Paris is the capital of France and has many museums.", ... reference="Paris is the capital of France." ... ) >>> print(f"Correctness Score: {result.value}") >>> >>> # Custom weights (more factuality focus) >>> factual_metric = AnswerCorrectness( ... llm=llm, ... embeddings=embeddings, ... weights=[0.9, 0.1] ... )

Attributes: llm: Modern instructor-based LLM for statement generation and classification embeddings: Modern embeddings model for similarity calculation name: The metric name weights: [factuality_weight, similarity_weight] - must sum to > 0 beta: F-beta score parameter (β>1 favors recall, β<1 favors precision) allowed_values: Score range (0.0 to 1.0)

Args: llm: Modern instructor-based LLM for statement generation and classification embeddings: Modern embeddings model for similarity calculation. Optional if similarity weight is 0 (pure factuality evaluation). Required if similarity weight > 0. name: The metric name weights: [factuality_weight, similarity_weight]. Must sum to > 0. beta: F-beta score parameter. β>1 favors recall, β<1 favors precision.

Raises: ValueError: If weights are invalid or embeddings are missing when needed for similarity scoring.

Examples: Pure factuality (no embeddings needed): >>> metric = AnswerCorrectness(llm=llm, weights=[1.0, 0.0])

Factuality + Similarity (embeddings required):
    >>> metric = AnswerCorrectness(llm=llm, embeddings=embeddings, weights=[0.75, 0.25])
Source code in src/ragas/metrics/collections/answer_correctness/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    embeddings: t.Optional["BaseRagasEmbedding"] = None,
    name: str = "answer_correctness",
    weights: List[float] = [0.75, 0.25],
    beta: float = 1.0,
    **kwargs,
):
    """
    Initialize AnswerCorrectness metric with required components.

    Args:
        llm: Modern instructor-based LLM for statement generation and classification
        embeddings: Modern embeddings model for similarity calculation. Optional if similarity
                   weight is 0 (pure factuality evaluation). Required if similarity weight > 0.
        name: The metric name
        weights: [factuality_weight, similarity_weight]. Must sum to > 0.
        beta: F-beta score parameter. β>1 favors recall, β<1 favors precision.

    Raises:
        ValueError: If weights are invalid or embeddings are missing when needed for similarity scoring.

    Examples:
        Pure factuality (no embeddings needed):
            >>> metric = AnswerCorrectness(llm=llm, weights=[1.0, 0.0])

        Factuality + Similarity (embeddings required):
            >>> metric = AnswerCorrectness(llm=llm, embeddings=embeddings, weights=[0.75, 0.25])
    """
    self.weights = weights
    self.beta = beta
    self.statement_generator_prompt = StatementGeneratorPrompt()
    self.correctness_classifier_prompt = CorrectnessClassifierPrompt()

    # Validate weights
    if len(weights) != 2:
        raise ValueError(
            "Expects a list of two weights. First for factuality, second for semantic similarity"
        )
    if all([w == 0 for w in weights]):
        raise ValueError("At least one weight must be non-zero")
    if not all([w >= 0 for w in weights]):
        raise ValueError("Weights must be non-negative")

    # Validate embeddings availability when similarity weight > 0
    if weights[1] > 0 and embeddings is None:
        raise ValueError(
            "Embeddings are required for semantic similarity scoring. "
            "Either provide embeddings or set similarity weight to 0 (weights=[1.0, 0.0]) "
            "for pure factuality-only evaluation."
        )

    # Validate beta
    if not isinstance(beta, float):
        raise ValueError(
            "Beta must be a float. A beta > 1 gives more weight to recall, while beta < 1 favors precision."
        )

    super().__init__(llm=llm, embeddings=embeddings, name=name, **kwargs)

ascore async

ascore(
    user_input: str, response: str, reference: str
) -> MetricResult

Calculate answer correctness score using multi-step pipeline.

Components are guaranteed to be validated and non-None by the base class.

Args: user_input: The original question response: The answer to evaluate reference: The ground truth reference

Returns: MetricResult with correctness score (0.0-1.0)

Source code in src/ragas/metrics/collections/answer_correctness/metric.py
async def ascore(
    self, user_input: str, response: str, reference: str
) -> MetricResult:
    """
    Calculate answer correctness score using multi-step pipeline.

    Components are guaranteed to be validated and non-None by the base class.

    Args:
        user_input: The original question
        response: The answer to evaluate
        reference: The ground truth reference

    Returns:
        MetricResult with correctness score (0.0-1.0)
    """
    # Step 1: Generate statements from both response and reference
    response_statements = await self._generate_statements(user_input, response)
    reference_statements = await self._generate_statements(user_input, reference)

    # Step 2: Calculate factuality score via TP/FP/FN classification.
    # With no statements on either side there is nothing to classify: the
    # factuality, and so the sample's score, is undefined (NaN), matching
    # FactualCorrectness (#49). It used to "assume perfect match" (1.0),
    # giving evasive answers full factuality credit (#136).
    if not (response_statements and reference_statements):
        side = "response" if not response_statements else "reference"
        return MetricResult(
            value=float("nan"),
            reason=f"No statements could be extracted from the {side}, so "
            f"factuality (and the score) is undefined.",
        )
    classification = await self._classify_statements(
        user_input, response_statements, reference_statements
    )
    factuality_score = self._compute_f1_score(classification)

    # Step 3: Calculate semantic similarity score
    if self.weights[1] == 0:
        similarity_score = 0.0
    else:
        similarity_score = await self._calculate_similarity(response, reference)

    # Step 4: Combine scores with weighted average
    final_score = np.average(
        [factuality_score, similarity_score],
        weights=self.weights,
    )

    lines = [
        f"Factuality {factuality_score:.2f} (weight {self.weights[0]}), "
        f"semantic similarity {similarity_score:.2f} (weight {self.weights[1]})."
    ]
    for label, items in (
        ("In both response and reference (TP)", classification.TP),
        ("Only in the response (FP)", classification.FP),
        ("Missing from the response (FN)", classification.FN),
    ):
        if items:
            lines.append(f"{label}:")
            lines.extend(f"- {i.statement}: {i.reason}" for i in items)
    return MetricResult(value=float(final_score), reason="\n".join(lines))

AnswerRelevancy

AnswerRelevancy(
    llm: InstructorBaseRagasLLM,
    embeddings: BaseRagasEmbedding,
    name: str = "answer_relevancy",
    strictness: int = 3,
    **kwargs,
)

Bases: BaseMetric

Modern v2 implementation of answer relevancy evaluation.

Evaluates answer relevancy by generating multiple questions from the response and comparing them to the original question using cosine similarity. The metric detects evasive/noncommittal answers.

This implementation uses modern instructor LLMs with structured output and modern embeddings for semantic comparison. Only supports modern components - legacy wrappers are rejected with clear error messages.

Usage: >>> import openai >>> from ragas.llms.base import llm_factory >>> from ragas.embeddings.base import embedding_factory >>> from ragas.metrics.collections import AnswerRelevancy >>> >>> # Setup dependencies >>> client = openai.AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> embeddings = embedding_factory("openai", model="text-embedding-ada-002", client=client) >>> >>> # Create metric instance >>> metric = AnswerRelevancy(llm=llm, embeddings=embeddings, strictness=3) >>> >>> # Single evaluation >>> result = await metric.ascore( ... user_input="What is the capital of France?", ... response="Paris is the capital of France." ... ) >>> print(f"Answer Relevancy: {result.value}")

Attributes: llm: Modern instructor-based LLM for question generation embeddings: Modern embeddings model for semantic comparison name: The metric name strictness: Number of questions to generate (default: 3) allowed_values: Score range (0.0 to 1.0, higher is better)

Args: llm: Modern instructor-based LLM for question generation embeddings: Modern embeddings model for semantic comparison name: The metric name (default: "answer_relevancy") strictness: Number of questions to generate (default: 3) **kwargs: Additional arguments passed to BaseMetric

Source code in src/ragas/metrics/collections/answer_relevancy/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    embeddings: "BaseRagasEmbedding",
    name: str = "answer_relevancy",
    strictness: int = 3,
    **kwargs,
):
    """
    Initialize AnswerRelevancy metric with required components.

    Args:
        llm: Modern instructor-based LLM for question generation
        embeddings: Modern embeddings model for semantic comparison
        name: The metric name (default: "answer_relevancy")
        strictness: Number of questions to generate (default: 3)
        **kwargs: Additional arguments passed to BaseMetric
    """
    self.strictness = strictness
    self.prompt = AnswerRelevancePrompt()  # Initialize prompt class once

    super().__init__(llm=llm, embeddings=embeddings, name=name, **kwargs)

ascore async

ascore(user_input: str, response: str) -> MetricResult

Calculate answer relevancy score asynchronously.

Components are guaranteed to be validated and non-None by the base class.

Args: user_input: The original question response: The response to evaluate

Returns: MetricResult with relevancy score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/answer_relevancy/metric.py
async def ascore(self, user_input: str, response: str) -> MetricResult:
    """
    Calculate answer relevancy score asynchronously.

    Components are guaranteed to be validated and non-None by the base class.

    Args:
        user_input: The original question
        response: The response to evaluate

    Returns:
        MetricResult with relevancy score (0.0-1.0, higher is better)
    """
    require(user_input=user_input, response=response)

    # Generate multiple questions from response
    generated_questions = []
    noncommittal_flags = []

    input_data = AnswerRelevanceInput(response=response)
    prompt_string = self.prompt.to_string(input_data)
    for result in await sample_n(
        self.llm, prompt_string, AnswerRelevanceOutput, self.strictness
    ):
        if result.question:
            generated_questions.append(result.question)
            noncommittal_flags.append(result.noncommittal)

    if not generated_questions:
        return MetricResult(
            value=0.0,
            reason="No question could be generated from the response (it is "
            "likely evasive or empty), so it is scored as not relevant.",
        )

    # Check if all responses are noncommittal
    all_noncommittal = np.all(noncommittal_flags)

    # Embed the original question
    question_vec = np.asarray(
        await self.embeddings.aembed_text(user_input)
    ).reshape(1, -1)

    # Embed the generated questions
    gen_question_vec = np.asarray(
        await self.embeddings.aembed_texts(generated_questions)
    ).reshape(len(generated_questions), -1)

    cosine_sim = cosine_matrix(gen_question_vec, question_vec).reshape(-1)

    # Score is average cosine similarity, reduced to 0 if response is noncommittal.
    # Cosine similarity can be slightly negative for unrelated embeddings;
    # the metric is documented as 0-1, so clip (#56).
    score = cosine_sim.mean() * int(not all_noncommittal)
    score = float(np.clip(score, 0.0, 1.0))

    if all_noncommittal:
        summary = (
            "The response was judged noncommittal (evasive or vague), so the "
            "score is 0. Questions generated from it:"
        )
    else:
        summary = (
            f"Mean similarity {float(cosine_sim.mean()):.2f} between the user "
            f"input and questions generated from the response:"
        )
    reason = "\n".join(
        [summary]
        + [
            f"- {q!r} (similarity {float(sim):.2f}"
            + (", noncommittal)" if flag else ")")
            for q, sim, flag in zip(
                generated_questions, cosine_sim, noncommittal_flags
            )
        ]
    )
    return MetricResult(value=float(score), reason=reason)

AspectCritic

AspectCritic(
    llm: InstructorBaseRagasLLM,
    definition: str,
    name: str = "aspect_critic",
    strictness: int = 1,
    **kwargs,
)

Bases: CriteriaJudgeMetric

Judge a submission against a criterion you define, in binary terms.

You supply the criterion as free text; the LLM answers yes (1) or no (0). This is the most open-ended metric in ragas -- anything you can phrase as a yes/no question about a response can be scored with it.

Usage: >>> from openai import AsyncOpenAI >>> from ragas.llms import llm_factory >>> from ragas.metrics.collections import AspectCritic >>> >>> llm = llm_factory("", client=AsyncOpenAI()) >>> >>> metric = AspectCritic( ... llm=llm, ... name="maliciousness", ... definition="Is the submission intended to harm, deceive, or exploit users?", ... ) >>> result = await metric.ascore( ... user_input="How do I bake bread?", ... response="Mix flour, water, salt and yeast, then bake at 220C.", ... ) >>> print(result.value) # 0.0

Multi-turn conversations are scored by passing the rendered exchange as user_input; metrics_experiment does that rendering for you.

Attributes: llm: Modern instructor-based LLM for evaluation definition: The criterion to judge against strictness: How many independent judgements to take a majority vote over name: The metric name allowed_values: Score range (0.0 to 1.0)

definition: The criterion to judge against
name: The metric name
strictness: Number of independent judgements to majority-vote over.
    Forced odd to avoid ties. Note this differs from the legacy
    ``AspectCritic``, where the parameter was accepted but never
    took more than one sample, so the vote was a no-op.
Source code in src/ragas/metrics/collections/aspect_critic/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    definition: str,
    name: str = "aspect_critic",
    strictness: int = 1,
    **kwargs,
):
    """
    Args:
        llm: Modern instructor-based LLM
        definition: The criterion to judge against
        name: The metric name
        strictness: Number of independent judgements to majority-vote over.
            Forced odd to avoid ties. Note this differs from the legacy
            ``AspectCritic``, where the parameter was accepted but never
            took more than one sample, so the vote was a no-op.
    """
    super().__init__(
        llm=llm,
        definition=definition,
        name=name,
        strictness=strictness,
        allowed_values=(0.0, 1.0),
        **kwargs,
    )

BaseMetric

BaseMetric(
    name: str = "base_metric",
    allowed_values: Tuple[float, float] = (0.0, 1.0),
    *,
    llm: Any = _UNSET,
    embeddings: Any = _UNSET,
    **kwargs,
)

Bases: SimpleBaseMetric, NumericValidator

Base class for metrics collections with modern component validation.

This class inherits from SimpleBaseMetric and NumericValidator to provide: - All the base metric functionality (ascore, abatch_score, score, batch_score) - Numeric validation with configurable ranges - Modern LLM and embedding component validation (when defined by subclass) - Rejection of legacy wrappers with helpful error messages - Consistent error handling and type safety

Attributes: name: The metric name allowed_values: Score range for numeric validation (tuple of min, max)

Components: a metric that needs an LLM and/or embeddings lists them in required_components (one it can do without, such as AnswerCorrectness's embeddings, in optional_components) and passes them on with super().__init__(llm=..., embeddings=...). The base stores each one and validates it, so attributes can be set in any order around that call.

Prompts: every BasePrompt the metric holds as an attribute is listed by get_prompts() under its attribute name and replaced with set_prompts(); train() tunes them against human annotations.

Args: name: The metric name. allowed_values: Score range for numeric validation. llm: The metric's LLM, if it declares "llm" in required_components or optional_components. embeddings: The metric's embeddings, likewise.

Raises: TypeError: for any keyword argument no constructor in the class chain consumed, including a component the metric doesn't declare. They used to be dropped silently, so a typo such as AspectCritic(..., strictnes=3) built a metric with the default strictness. ValueError: for a required component that isn't a modern LLM / embedding, or an optional one that is neither that nor None.

Source code in src/ragas/metrics/collections/base.py
def __init__(
    self,
    name: str = "base_metric",
    allowed_values: t.Tuple[float, float] = (0.0, 1.0),
    *,
    llm: t.Any = _UNSET,
    embeddings: t.Any = _UNSET,
    **kwargs,
):
    """Initialize the base metric with validation.

    Args:
        name: The metric name.
        allowed_values: Score range for numeric validation.
        llm: The metric's LLM, if it declares ``"llm"`` in
            ``required_components`` or ``optional_components``.
        embeddings: The metric's embeddings, likewise.

    Raises:
        TypeError: for any keyword argument no constructor in the class
            chain consumed, including a component the metric doesn't
            declare. They used to be dropped silently, so a typo such as
            ``AspectCritic(..., strictnes=3)`` built a metric with the
            default strictness.
        ValueError: for a required component that isn't a modern LLM /
            embedding, or an optional one that is neither that nor None.
    """
    declared = (*self.required_components, *self.optional_components)
    for component, value in (("llm", llm), ("embeddings", embeddings)):
        if value is _UNSET:
            continue
        if component in declared:
            setattr(self, component, value)
        else:
            kwargs[component] = value
    if kwargs:
        unknown = ", ".join(repr(k) for k in sorted(kwargs))
        raise TypeError(
            f"{type(self).__name__}() got unexpected keyword argument(s): {unknown}"
        )
    super().__init__(name=name, allowed_values=allowed_values)

    validators = {
        "llm": self._validate_llm,
        "embeddings": self._validate_embeddings,
    }
    for component in self.required_components:
        validators[component]()
    for component in self.optional_components:
        if getattr(self, component, None) is not None:
            validators[component]()

score

score(**kwargs) -> MetricResult

Synchronous scoring method that wraps ascore().

This is a convenience method for backward compatibility and sync usage. For better performance, prefer using ascore() directly in async contexts.

Returns: MetricResult object

Source code in src/ragas/metrics/collections/base.py
def score(self, **kwargs) -> MetricResult:
    """
    Synchronous scoring method that wraps ascore().

    This is a convenience method for backward compatibility and sync usage.
    For better performance, prefer using ascore() directly in async contexts.

    Returns:
        MetricResult object
    """
    # On ragas' shared background loop: works inside Jupyter too, and an
    # async client stays bound to one loop across calls. asyncio.run()
    # made a new loop per call, so a second score() with AsyncOpenAI
    # failed with "bound to a different event loop".
    return run_sync(self.ascore(**kwargs))

batch_score

batch_score(
    inputs: List[Dict[str, Any]],
) -> List[MetricResult]

Synchronous batch scoring that wraps abatch_score().

This is a convenience method for backward compatibility and sync usage. For better performance, prefer using abatch_score() directly in async contexts.

Args: inputs: List of input dictionaries for scoring

Returns: List of MetricResult objects

Source code in src/ragas/metrics/collections/base.py
def batch_score(
    self,
    inputs: t.List[t.Dict[str, t.Any]],
) -> t.List[MetricResult]:
    """
    Synchronous batch scoring that wraps abatch_score().

    This is a convenience method for backward compatibility and sync usage.
    For better performance, prefer using abatch_score() directly in async contexts.

    Args:
        inputs: List of input dictionaries for scoring

    Returns:
        List of MetricResult objects
    """
    # On ragas' shared background loop: works inside Jupyter too, and an
    # async client stays bound to one loop across calls. asyncio.run()
    # made a new loop per call, so a second score() with AsyncOpenAI
    # failed with "bound to a different event loop".
    return run_sync(self.abatch_score(inputs))

get_prompts

get_prompts() -> Dict[str, BasePrompt]

The metric's prompts, keyed by attribute name.

Each value is a copy: change it and pass it to set_prompts to use it. Editing it in place changes neither this metric nor any other metric or class that holds the same prompt object.

Returns: {attribute name: prompt}, e.g. statement_generator_prompt and nli_statement_prompt for Faithfulness. Empty for a metric without prompts.

Source code in src/ragas/metrics/collections/base.py
def get_prompts(self) -> t.Dict[str, BasePrompt]:
    """The metric's prompts, keyed by attribute name.

    Each value is a copy: change it and pass it to ``set_prompts`` to use
    it. Editing it in place changes neither this metric nor any other
    metric or class that holds the same prompt object.

    Returns:
        ``{attribute name: prompt}``, e.g. ``statement_generator_prompt``
        and ``nli_statement_prompt`` for Faithfulness. Empty for a metric
        without prompts.
    """
    return {
        name: copy.deepcopy(prompt)
        for name, prompt in self._prompt_attributes().items()
    }

set_prompts

set_prompts(**prompts: BasePrompt) -> None

Replace prompts, by the names get_prompts() uses.

Each prompt is copied onto this metric, so later changes to the object passed in don't reach the metric, and a prompt defined on the class (shared by its instances) is shadowed, never modified.

Raises: ValueError: for a name the metric has no prompt under. TypeError: for a value that isn't a BasePrompt, or whose input or output model differs from the prompt it replaces (the metric renders that input and parses that output).

Source code in src/ragas/metrics/collections/base.py
def set_prompts(self, **prompts: BasePrompt) -> None:
    """Replace prompts, by the names ``get_prompts()`` uses.

    Each prompt is copied onto this metric, so later changes to the object
    passed in don't reach the metric, and a prompt defined on the class
    (shared by its instances) is shadowed, never modified.

    Raises:
        ValueError: for a name the metric has no prompt under.
        TypeError: for a value that isn't a ``BasePrompt``, or whose input
            or output model differs from the prompt it replaces (the
            metric renders that input and parses that output).
    """
    current = self._prompt_attributes()
    for name, prompt in prompts.items():
        if name not in current:
            available = ", ".join(sorted(current)) or "none"
            raise ValueError(
                f"{type(self).__name__} has no prompt named '{name}'. "
                f"Available prompts: {available}."
            )
        if not isinstance(prompt, BasePrompt):
            raise TypeError(
                f"Prompt '{name}' must be a ragas.prompt.metrics.BasePrompt, "
                f"got {type(prompt).__name__}."
            )
        old = current[name]
        for side in ("input_model", "output_model"):
            if getattr(prompt, side) is not getattr(old, side):
                raise TypeError(
                    f"Prompt '{name}' must keep {side} "
                    f"{getattr(old, side).__name__}, got "
                    f"{getattr(prompt, side).__name__}."
                )
    for name, prompt in prompts.items():
        setattr(self, name, copy.deepcopy(prompt))

train

train(
    path: Optional[str] = None,
    demonstration_config: Optional[
        DemonstrationConfig
    ] = None,
    instruction_config: Optional[InstructionConfig] = None,
    *,
    dataset: Optional[
        Union[MetricAnnotation, SingleMetricAnnotation]
    ] = None,
    run_config: Optional[RunConfig] = None,
    batch_size: Optional[int] = None,
    raise_exceptions: bool = True,
) -> None

Tune this metric's prompts to human annotations.

With instruction_config, its optimizer (GeneticOptimizer by default) rewrites the annotated prompts' instructions. With demonstration_config, each annotated prompt then picks the annotated examples most similar to its input as few-shot demonstrations. The results replace this metric's prompts through set_prompts; the prompt objects it held before are not modified.

Args: path: A JSON annotation file (MetricAnnotation format), whose prompt names are this metric's get_prompts() keys. demonstration_config: Few-shot demonstration settings. instruction_config: Instruction optimization settings. dataset: The annotations as an object, instead of path. run_config: Execution limits for the scoring runs. batch_size: Jobs per batch for the scoring runs. raise_exceptions: Raise on a failed job instead of scoring NaN.

See ragas.optimizers.train_metric.

Source code in src/ragas/metrics/collections/base.py
def train(
    self,
    path: t.Optional[str] = None,
    demonstration_config: t.Optional["DemonstrationConfig"] = None,
    instruction_config: t.Optional["InstructionConfig"] = None,
    *,
    dataset: t.Optional[
        t.Union["MetricAnnotation", "SingleMetricAnnotation"]
    ] = None,
    run_config: t.Optional["RunConfig"] = None,
    batch_size: t.Optional[int] = None,
    raise_exceptions: bool = True,
) -> None:
    """Tune this metric's prompts to human annotations.

    With ``instruction_config``, its optimizer (``GeneticOptimizer`` by
    default) rewrites the annotated prompts' instructions. With
    ``demonstration_config``, each annotated prompt then picks the
    annotated examples most similar to its input as few-shot
    demonstrations. The results replace this metric's prompts through
    ``set_prompts``; the prompt objects it held before are not modified.

    Args:
        path: A JSON annotation file (``MetricAnnotation`` format), whose
            prompt names are this metric's ``get_prompts()`` keys.
        demonstration_config: Few-shot demonstration settings.
        instruction_config: Instruction optimization settings.
        dataset: The annotations as an object, instead of ``path``.
        run_config: Execution limits for the scoring runs.
        batch_size: Jobs per batch for the scoring runs.
        raise_exceptions: Raise on a failed job instead of scoring NaN.

    See ``ragas.optimizers.train_metric``.
    """
    from ragas.optimizers.training import train_metric

    train_metric(
        self,
        path=path,
        dataset=dataset,
        demonstration_config=demonstration_config,
        instruction_config=instruction_config,
        run_config=run_config,
        batch_size=batch_size,
        raise_exceptions=raise_exceptions,
    )

CHRFScore

CHRFScore(
    name: str = "chrf_score",
    kwargs: Optional[Dict[str, Any]] = None,
    **base_kwargs,
)

Bases: BaseMetric

Calculate CHRF (Character F-score) between reference and response texts.

CHRF is a character n-gram F-score metric that correlates well with human judgments for machine translation quality. Unlike BLEU which operates on words, CHRF operates on character-level n-grams, making it more robust to morphological variations and better suited for morphologically rich languages.

This implementation uses the sacrebleu library for consistent and reproducible scoring.

Usage: >>> from ragas.metrics.collections import CHRFScore >>> >>> metric = CHRFScore() >>> >>> result = await metric.ascore( ... reference="The capital of France is Paris.", ... response="Paris is the capital of France." ... ) >>> print(f"Score: {result.value}") >>> >>> results = await metric.abatch_score([ ... {"reference": "Text 1", "response": "Response 1"}, ... {"reference": "Text 2", "response": "Response 2"}, ... ])

Attributes: name: The metric name (default: "chrf_score") kwargs: Additional arguments to pass to sacrebleu.corpus_chrf (e.g., char_order, word_order, beta, eps_smoothing) allowed_values: Score range (0.0 to 1.0)

Source code in src/ragas/metrics/collections/chrf_score/metric.py
def __init__(
    self,
    name: str = "chrf_score",
    kwargs: t.Optional[t.Dict[str, t.Any]] = None,
    **base_kwargs,
):
    """Initialize CHRFScore metric."""
    super().__init__(name=name, **base_kwargs)
    self.kwargs = kwargs or {}

ascore async

ascore(reference: str, response: str) -> MetricResult

Calculate CHRF score asynchronously.

Args: reference: The reference/ground truth text response: The response text to evaluate

Returns: MetricResult with CHRF score (0.0-1.0)

Source code in src/ragas/metrics/collections/chrf_score/metric.py
async def ascore(
    self,
    reference: str,
    response: str,
) -> MetricResult:
    """
    Calculate CHRF score asynchronously.

    Args:
        reference: The reference/ground truth text
        response: The response text to evaluate

    Returns:
        MetricResult with CHRF score (0.0-1.0)
    """
    try:
        from sacrebleu import corpus_chrf
    except ImportError:
        raise ImportError(
            "sacrebleu is required for CHRF score calculation. "
            "Please install it using `pip install sacrebleu`"
        )

    # A missing or unusable reference is a gap in the dataset, not a
    # model failure, so the score is undefined (NaN) rather than 0 (#61).
    if not isinstance(reference, str) or not isinstance(response, str):
        return MetricResult(
            value=float("nan"),
            reason="Invalid input: reference and response must be strings",
        )

    if not reference.strip():
        return MetricResult(
            value=float("nan"),
            reason="Empty input: reference is empty, score is undefined",
        )

    # An empty response against a real reference is a genuine miss.
    if not response.strip():
        return MetricResult(
            value=0.0,
            reason="Empty input: response is empty",
        )

    # corpus_chrf expects hypotheses as list of strings and references as list of list of strings
    references = [[reference]]
    hypotheses = [response]

    score = corpus_chrf(hypotheses, references, **self.kwargs).score / 100

    return MetricResult(value=float(score))

ContextEntityRecall

ContextEntityRecall(
    llm: InstructorBaseRagasLLM,
    name: str = "context_entity_recall",
    **kwargs,
)

Bases: BaseMetric

Modern v2 implementation of context entity recall evaluation.

Calculates recall based on entities present in ground truth and retrieved contexts. Let CN be the set of entities present in context, GN be the set of entities present in the ground truth. Context Entity recall = | CN ∩ GN | / | GN |

This implementation uses modern instructor LLMs with structured output. Only supports modern components - legacy wrappers are rejected with clear error messages.

Usage: >>> import openai >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import ContextEntityRecall >>> >>> # Setup dependencies >>> client = openai.AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance >>> metric = ContextEntityRecall(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... reference="Paris is the capital of France, established in 52 BC.", ... retrieved_contexts=["France's capital city is Paris.", "The city was founded in ancient times."] ... ) >>> print(f"Entity Recall: {result.value}")

Attributes: llm: Modern instructor-based LLM for entity extraction name: The metric name allowed_values: Score range (0.0 to 1.0, higher is better)

Args: llm: Modern instructor-based LLM for entity extraction name: The metric name (default: "context_entity_recall") **kwargs: Additional arguments passed to BaseMetric

Source code in src/ragas/metrics/collections/context_entity_recall/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "context_entity_recall",
    **kwargs,
):
    """
    Initialize ContextEntityRecall metric with required components.

    Args:
        llm: Modern instructor-based LLM for entity extraction
        name: The metric name (default: "context_entity_recall")
        **kwargs: Additional arguments passed to BaseMetric
    """
    self.prompt = ExtractEntitiesPrompt()  # Initialize prompt class once

    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    reference: str, retrieved_contexts: List[str]
) -> MetricResult

Calculate context entity recall score.

Components are guaranteed to be validated and non-None by the base class.

Args: reference: The ground truth reference text retrieved_contexts: List of retrieved context strings

Returns: MetricResult with entity recall score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/context_entity_recall/metric.py
async def ascore(
    self, reference: str, retrieved_contexts: List[str]
) -> MetricResult:
    """
    Calculate context entity recall score.

    Components are guaranteed to be validated and non-None by the base class.

    Args:
        reference: The ground truth reference text
        retrieved_contexts: List of retrieved context strings

    Returns:
        MetricResult with entity recall score (0.0-1.0, higher is better)
    """
    # Extract entities from reference (ground truth)
    reference_entities = await self._extract_entities(reference)

    # Extract entities from retrieved contexts (combined)
    combined_contexts = "\n".join(retrieved_contexts)
    context_entities = await self._extract_entities(combined_contexts)

    # Calculate recall score
    recall_score = self._compute_recall_score(reference_entities, context_entities)

    found = set(context_entities)
    reference_set = list(dict.fromkeys(reference_entities))
    recalled = sum(1 for e in reference_set if e in found)
    reason = itemized(
        f"{count_phrase(recalled, len(reference_set), 'entity')} in the "
        f"reference appear in the retrieved context."
        if reference_set
        else "No entities were found in the reference.",
        ((e, e in found, None) for e in reference_set),
        yes="found",
        no="missing",
    )
    return MetricResult(value=float(recall_score), reason=reason)

ContextPrecision

ContextPrecision(
    llm: InstructorBaseRagasLLM,
    name: str = "context_precision",
    **kwargs,
)

Bases: ContextPrecisionWithReference

Modern v2 wrapper for ContextPrecisionWithReference with shorter name.

This is a simple wrapper that provides the legacy "context_precision" name while using the modern V2 implementation underneath.

Usage: >>> import openai >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import ContextPrecision >>> >>> # Setup dependencies >>> client = openai.AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance (same as ContextPrecisionWithReference) >>> metric = ContextPrecision(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... user_input="What is the capital of France?", ... reference="Paris is the capital of France.", ... retrieved_contexts=["Paris is the capital and largest city of France."] ... )

Source code in src/ragas/metrics/collections/context_precision/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "context_precision",
    **kwargs,
):
    """Initialize ContextPrecision with the legacy default name."""
    super().__init__(llm, name=name, **kwargs)

ContextPrecisionWithoutReference

ContextPrecisionWithoutReference(
    llm: InstructorBaseRagasLLM,
    name: str = "context_precision_without_reference",
    **kwargs,
)

Bases: _ContextPrecisionBase

Modern v2 implementation of context precision without reference.

Evaluates whether retrieved contexts are useful for answering a question by comparing each context against the generated response. The metric calculates average precision based on the usefulness verdicts from an LLM.

This implementation uses modern instructor LLMs with structured output. Only supports modern components - legacy wrappers are rejected with clear error messages.

Usage: >>> import openai >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import ContextPrecisionWithoutReference >>> >>> # Setup dependencies >>> client = openai.AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance >>> metric = ContextPrecisionWithoutReference(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... user_input="What is the capital of France?", ... response="Paris is the capital of France.", ... retrieved_contexts=["Paris is the capital and largest city of France.", "Berlin is the capital of Germany."] ... ) >>> print(f"Context Precision: {result.value}")

Attributes: llm: Modern instructor-based LLM for context evaluation name: The metric name allowed_values: Score range (0.0 to 1.0, higher is better)

Args: llm: Modern instructor-based LLM for context evaluation name: The metric name

Source code in src/ragas/metrics/collections/context_precision/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "context_precision_without_reference",
    **kwargs,
):
    """
    Initialize ContextPrecisionWithoutReference metric with required components.

    Args:
        llm: Modern instructor-based LLM for context evaluation
        name: The metric name
    """
    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    user_input: str,
    response: str,
    retrieved_contexts: List[str],
) -> MetricResult

Calculate context precision score using response.

Args: user_input: The question being asked response: The response that was generated retrieved_contexts: The retrieved contexts to evaluate

Returns: MetricResult with context precision score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/context_precision/metric.py
async def ascore(
    self, user_input: str, response: str, retrieved_contexts: List[str]
) -> MetricResult:
    """
    Calculate context precision score using response.

    Args:
        user_input: The question being asked
        response: The response that was generated
        retrieved_contexts: The retrieved contexts to evaluate

    Returns:
        MetricResult with context precision score (0.0-1.0, higher is better)
    """
    require(
        user_input=user_input,
        response=response,
        retrieved_contexts=retrieved_contexts,
    )
    return await self._score(user_input, response, retrieved_contexts)

ContextPrecisionWithReference

ContextPrecisionWithReference(
    llm: InstructorBaseRagasLLM,
    name: str = "context_precision_with_reference",
    **kwargs,
)

Bases: _ContextPrecisionBase

Modern v2 implementation of context precision with reference.

Evaluates whether retrieved contexts are useful for answering a question by comparing each context against a reference answer. The metric calculates average precision based on the usefulness verdicts from an LLM.

This implementation uses modern instructor LLMs with structured output. Only supports modern components - legacy wrappers are rejected with clear error messages.

Usage: >>> import openai >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import ContextPrecisionWithReference >>> >>> # Setup dependencies >>> client = openai.AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance >>> metric = ContextPrecisionWithReference(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... user_input="What is the capital of France?", ... reference="Paris is the capital of France.", ... retrieved_contexts=["Paris is the capital and largest city of France.", "Berlin is the capital of Germany."] ... ) >>> print(f"Context Precision: {result.value}")

Attributes: llm: Modern instructor-based LLM for context evaluation name: The metric name allowed_values: Score range (0.0 to 1.0, higher is better)

Args: llm: Modern instructor-based LLM for context evaluation name: The metric name

Source code in src/ragas/metrics/collections/context_precision/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "context_precision_with_reference",
    **kwargs,
):
    """
    Initialize ContextPrecisionWithReference metric with required components.

    Args:
        llm: Modern instructor-based LLM for context evaluation
        name: The metric name
    """
    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    user_input: str,
    reference: str,
    retrieved_contexts: List[str],
) -> MetricResult

Calculate context precision score using reference.

Args: user_input: The question being asked reference: The reference answer to compare against retrieved_contexts: The retrieved contexts to evaluate

Returns: MetricResult with context precision score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/context_precision/metric.py
async def ascore(
    self, user_input: str, reference: str, retrieved_contexts: List[str]
) -> MetricResult:
    """
    Calculate context precision score using reference.

    Args:
        user_input: The question being asked
        reference: The reference answer to compare against
        retrieved_contexts: The retrieved contexts to evaluate

    Returns:
        MetricResult with context precision score (0.0-1.0, higher is better)
    """
    require(
        user_input=user_input,
        reference=reference,
        retrieved_contexts=retrieved_contexts,
    )
    return await self._score(user_input, reference, retrieved_contexts)

ContextUtilization

ContextUtilization(
    llm: InstructorBaseRagasLLM,
    name: str = "context_utilization",
    **kwargs,
)

Bases: ContextPrecisionWithoutReference

Modern v2 wrapper for ContextPrecisionWithoutReference with shorter name.

This is a simple wrapper that provides the legacy "context_utilization" name while using the modern V2 implementation underneath.

Usage: >>> import openai >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import ContextUtilization >>> >>> # Setup dependencies >>> client = openai.AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance (same as ContextPrecisionWithoutReference) >>> metric = ContextUtilization(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... user_input="What is the capital of France?", ... response="Paris is the capital of France.", ... retrieved_contexts=["Paris is the capital and largest city of France."] ... )

Source code in src/ragas/metrics/collections/context_precision/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "context_utilization",
    **kwargs,
):
    """Initialize ContextUtilization with the legacy default name."""
    super().__init__(llm, name=name, **kwargs)

ContextRecall

ContextRecall(
    llm: InstructorBaseRagasLLM,
    name: str = "context_recall",
    **kwargs,
)

Bases: BaseMetric

Modern v2 implementation of context recall evaluation.

Evaluates context recall by classifying if statements in a reference answer can be attributed to the retrieved context. Uses an LLM to verify attribution for each statement and calculates recall as the proportion of attributed statements.

This implementation uses modern instructor LLMs with structured output. Only supports modern components - legacy wrappers are rejected with clear error messages.

Usage: >>> import openai >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import ContextRecall >>> >>> # Setup dependencies >>> client = openai.AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance >>> metric = ContextRecall(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... user_input="What is the capital of France?", ... retrieved_contexts=["Paris is the capital of France."], ... reference="Paris is the capital and largest city of France." ... ) >>> print(f"Context Recall: {result.value}")

Attributes: llm: Modern instructor-based LLM for statement classification name: The metric name allowed_values: Score range (0.0 to 1.0, higher is better)

Args: llm: Modern instructor-based LLM for statement classification name: The metric name (default: "context_recall") **kwargs: Additional arguments passed to BaseMetric

Source code in src/ragas/metrics/collections/context_recall/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "context_recall",
    **kwargs,
):
    """
    Initialize ContextRecall metric with required components.

    Args:
        llm: Modern instructor-based LLM for statement classification
        name: The metric name (default: "context_recall")
        **kwargs: Additional arguments passed to BaseMetric
    """
    self.prompt = ContextRecallPrompt()  # Initialize prompt class once

    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    user_input: str,
    retrieved_contexts: List[str],
    reference: str,
) -> MetricResult

Calculate context recall score asynchronously.

Components are guaranteed to be validated and non-None by the base class.

Args: user_input: The original question retrieved_contexts: List of retrieved context strings reference: The reference answer to evaluate

Returns: MetricResult with recall score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/context_recall/metric.py
async def ascore(
    self,
    user_input: str,
    retrieved_contexts: List[str],
    reference: str,
) -> MetricResult:
    """
    Calculate context recall score asynchronously.

    Components are guaranteed to be validated and non-None by the base class.

    Args:
        user_input: The original question
        retrieved_contexts: List of retrieved context strings
        reference: The reference answer to evaluate

    Returns:
        MetricResult with recall score (0.0-1.0, higher is better)
    """
    require(
        user_input=user_input,
        reference=reference,
        retrieved_contexts=retrieved_contexts,
    )

    # Combine contexts into a single string
    context = "\n".join(retrieved_contexts) if retrieved_contexts else ""

    # The reference is split into sentences here and the judge classifies
    # exactly those, one verdict each. The judge used to do the splitting
    # itself, so a sentence it dropped silently left the denominator (#52).
    statements = split_sentences(reference)
    if not statements:
        return MetricResult(value=np.nan)
    input_data = ContextRecallInput(
        question=user_input,
        context=context,
        answer=reference,
        statements=statements,
    )
    prompt_string = self.prompt.to_string(input_data)

    async def ask() -> List[ContextRecallClassification]:
        result = await self.llm.agenerate(prompt_string, ContextRecallOutput)
        return result.classifications

    aligned = await judge_each(
        statements,
        ask,
        text_of=lambda c: c.statement,
        what="classifications",
        metric=self.name,
    )
    if aligned is None:
        return MetricResult(
            value=np.nan, reason=UNMATCHED_VERDICTS.format(what="sentences")
        )

    attributed = sum(1 for c in aligned if c.attributed)
    score = attributed / len(aligned)
    reason = itemized(
        f"{count_phrase(attributed, len(aligned), 'sentence')} of the reference "
        f"{count_verb(attributed, len(aligned))} attributable to the retrieved "
        f"context.",
        ((s, bool(c.attributed), c.reason) for s, c in zip(statements, aligned)),
        yes="attributed",
        no="not attributed",
    )
    return MetricResult(value=float(score), reason=reason)

ContextRelevance

ContextRelevance(
    llm: InstructorBaseRagasLLM,
    name: str = "context_relevance",
    max_retries: int = 5,
    **kwargs,
)

Bases: DualJudgeMetric

Context Relevance metric using dual-judge evaluation.

Evaluates whether the retrieved contexts are pertinent to the user input using a dual-judge system. This metric averages two distinct judge prompts to ensure robust evaluation.

The metric uses NVIDIA's proven dual-judge approach: 1. Judge 1: Direct context relevance evaluation 2. Judge 2: Alternative perspective for fairness 3. Average both judges for final score

Rating scale: 0 (not relevant), 1 (partially relevant), 2 (fully relevant) Final score: Average of both judges converted to 0.0-1.0 scale

Usage: >>> import instructor >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import ContextRelevance >>> >>> # Setup dependencies >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance >>> metric = ContextRelevance(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... user_input="When was Einstein born?", ... retrieved_contexts=["Albert Einstein was born March 14, 1879."] ... ) >>> print(f"Context Relevance: {result.value}")

Attributes: llm: Modern instructor-based LLM for dual-judge evaluation name: The metric name allowed_values: Score range (0.0 to 1.0, higher is better) max_retries: Attempts per judge; a retry after an off-scale rating is a fresh sample, not a resend of the same request

name: The metric name
max_retries: Attempts per judge (see ``DualJudgeMetric``)
Source code in src/ragas/metrics/collections/context_relevance/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "context_relevance",
    max_retries: int = 5,
    **kwargs,
):
    """
    Args:
        llm: Modern instructor-based LLM for dual-judge evaluation
        name: The metric name
        max_retries: Attempts per judge (see ``DualJudgeMetric``)
    """
    super().__init__(llm=llm, name=name, max_retries=max_retries, **kwargs)

ascore async

ascore(
    user_input: str, retrieved_contexts: List[str]
) -> MetricResult

Calculate context relevance score using dual-judge evaluation.

Args: user_input: The original question retrieved_contexts: The retrieved contexts to evaluate for relevance

Returns: MetricResult with context relevance score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/context_relevance/metric.py
async def ascore(
    self, user_input: str, retrieved_contexts: List[str]
) -> MetricResult:
    """
    Calculate context relevance score using dual-judge evaluation.

    Args:
        user_input: The original question
        retrieved_contexts: The retrieved contexts to evaluate for relevance

    Returns:
        MetricResult with context relevance score (0.0-1.0, higher is better)
    """
    require(user_input=user_input, retrieved_contexts=retrieved_contexts)

    context_str = "\n".join(retrieved_contexts)
    question, context = user_input.strip(), context_str.strip()
    # Empty input, or a context that only repeats (part of) the question.
    if not question or not context or context in question:
        return MetricResult(value=0.0)

    judge_input = ContextRelevanceInput(user_input=user_input, context=context_str)
    return await self._judge_both(judge_input, judge_input)

DataCompyScore

DataCompyScore(
    mode: Literal["rows", "columns"] = "rows",
    metric: Literal["precision", "recall", "f1"] = "f1",
    name: str = "data_compare_score",
    **kwargs,
)

Bases: BaseMetric

Compare CSV data using datacompy library to compute precision, recall, or F1 scores.

This metric compares two CSV strings (reference and response) and calculates matching statistics at either row or column level. Useful for evaluating SQL-to-text or data generation tasks where tabular output needs to be compared.

The metric supports three modes of comparison: - precision: Proportion of response rows/columns that match reference - recall: Proportion of reference rows/columns found in response - f1: Harmonic mean of precision and recall

Usage: >>> from ragas.metrics.collections import DataCompyScore >>> >>> metric = DataCompyScore(mode="rows", metric="f1") >>> >>> result = await metric.ascore( ... reference="id,name\n1,Alice\n2,Bob", ... response="id,name\n1,Alice\n2,Bob\n3,Charlie", ... ) >>> print(f"F1 Score: {result.value}")

Attributes: name: The metric name (default: "data_compare_score") mode: Comparison mode - "rows" or "columns" metric: Score type - "precision", "recall", or "f1"

Source code in src/ragas/metrics/collections/datacompy_score/metric.py
def __init__(
    self,
    mode: t.Literal["rows", "columns"] = "rows",
    metric: t.Literal["precision", "recall", "f1"] = "f1",
    name: str = "data_compare_score",
    **kwargs,
):
    super().__init__(name=name, **kwargs)

    # Check for required dependencies at init time
    try:
        import pandas as pd

        # Try new import path first (datacompy >= 0.14), fall back to legacy
        try:
            from datacompy.core import Compare
        except ImportError:
            from datacompy import Compare  # type: ignore[attr-defined]
    except ImportError as e:
        raise ImportError(
            f"{e.name} is required for DataCompyScore. "
            f"Please install it using `pip install {e.name}`"
        )

    self._pd = pd
    self._Compare = Compare

    if mode not in ["rows", "columns"]:
        raise ValueError("mode must be either 'rows' or 'columns'")
    if metric not in ["precision", "recall", "f1"]:
        raise ValueError("metric must be either 'precision', 'recall', or 'f1'")

    self.mode = mode
    self.metric = metric

ascore async

ascore(reference: str, response: str) -> MetricResult

Calculate data comparison score between reference and response CSV strings.

Args: reference: The reference CSV data as a string response: The response CSV data to evaluate

Returns: MetricResult with comparison score (0.0-1.0) or NaN if parsing fails

Source code in src/ragas/metrics/collections/datacompy_score/metric.py
async def ascore(
    self,
    reference: str,
    response: str,
) -> MetricResult:
    """
    Calculate data comparison score between reference and response CSV strings.

    Args:
        reference: The reference CSV data as a string
        response: The response CSV data to evaluate

    Returns:
        MetricResult with comparison score (0.0-1.0) or NaN if parsing fails
    """
    if not isinstance(reference, str):
        raise ValueError("reference must be a CSV string")
    if not isinstance(response, str):
        raise ValueError("response must be a CSV string")

    try:
        reference_df = self._pd.read_csv(StringIO(reference))
        response_df = self._pd.read_csv(StringIO(response))
    except Exception as e:
        logger.error(f"Error reading CSV: {e}")
        return MetricResult(value=float(np.nan), reason=f"CSV parsing error: {e}")

    compare = self._Compare(reference_df, response_df, on_index=True)

    if self.mode == "rows":
        matched = compare.count_matching_rows()
        expected, produced = reference_df.shape[0], response_df.shape[0]
    else:
        matched = len(
            [col for col in compare.column_stats if col["unequal_cnt"] == 0]
        )
        expected, produced = reference_df.shape[1], response_df.shape[1]
    # Matched rows/columns are the true positives; the rest of the response
    # is false positives and the rest of the reference false negatives.
    precision, recall = precision_recall(
        matched, produced - matched, expected - matched
    )

    if self.metric == "precision":
        score = precision
    elif self.metric == "recall":
        score = recall
    else:
        score = fbeta_from_precision_recall(precision, recall)

    return MetricResult(
        value=float(score),
        reason=f"Mode: {self.mode}, Precision: {precision:.4f}, Recall: {recall:.4f}",
    )

DomainSpecificRubrics

DomainSpecificRubrics(
    llm: InstructorBaseRagasLLM,
    rubrics: Optional[Dict[str, str]] = None,
    with_reference: bool = False,
    name: str = "domain_specific_rubrics",
    **kwargs,
)

Bases: BaseMetric

Evaluates responses using domain-specific rubrics with customizable scoring criteria.

This metric allows you to define custom rubrics (scoring criteria) to evaluate LLM responses. It supports both reference-free and reference-based evaluation, making it flexible for various evaluation scenarios.

The metric works by: 1. Taking the input, response, and optionally reference/contexts 2. Using an LLM to evaluate the response against the rubric criteria 3. Returning a score (1-5) with detailed feedback

Score interpretation (default rubrics): - Score 1: Response is entirely incorrect or irrelevant - Score 2: Response has partial accuracy with major errors - Score 3: Response is mostly accurate but lacks detail - Score 4: Response is accurate with minor omissions - Score 5: Response is completely accurate and thorough

Usage: >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import DomainSpecificRubrics >>> >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Reference-free evaluation >>> metric = DomainSpecificRubrics(llm=llm) >>> result = await metric.ascore( ... user_input="What is the capital of France?", ... response="The capital of France is Paris.", ... ) >>> print(f"Score: {result.value}, Feedback: {result.reason}") >>> >>> # Reference-based evaluation >>> metric_with_ref = DomainSpecificRubrics(llm=llm, with_reference=True) >>> result = await metric_with_ref.ascore( ... user_input="What is the capital of France?", ... response="The capital of France is Paris.", ... reference="Paris is the capital and largest city of France.", ... ) >>> >>> # Custom rubrics >>> custom_rubrics = { ... "score1_description": "Completely wrong", ... "score2_description": "Mostly wrong with some correct elements", ... "score3_description": "Partially correct", ... "score4_description": "Mostly correct with minor issues", ... "score5_description": "Fully correct and comprehensive", ... } >>> metric_custom = DomainSpecificRubrics(llm=llm, rubrics=custom_rubrics)

Attributes: llm: Modern instructor-based LLM for evaluation rubrics: Dictionary mapping score descriptions (e.g., "score1_description" to criteria text) with_reference: Whether to use reference-based evaluation (default: False) name: The metric name (default: "domain_specific_rubrics")

Source code in src/ragas/metrics/collections/domain_specific_rubrics/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    rubrics: t.Optional[t.Dict[str, str]] = None,
    with_reference: bool = False,
    name: str = "domain_specific_rubrics",
    **kwargs,
):
    self.with_reference = with_reference

    if rubrics is None:
        self.rubrics = (
            DEFAULT_WITH_REFERENCE_RUBRICS
            if with_reference
            else DEFAULT_REFERENCE_FREE_RUBRICS
        )
    else:
        self.rubrics = rubrics

    rubrics_text = format_rubrics(self.rubrics)
    self.scoring_prompt = RubricScorePrompt()
    self.scoring_prompt.instruction = (
        f"{self.scoring_prompt.instruction}\n\nScoring Rubrics:\n{rubrics_text}\n"
    )

    super().__init__(llm=llm, name=name, allowed_values=(1.0, 5.0), **kwargs)

ascore async

ascore(
    user_input: Optional[str] = None,
    response: Optional[str] = None,
    retrieved_contexts: Optional[List[str]] = None,
    reference_contexts: Optional[List[str]] = None,
    reference: Optional[str] = None,
) -> MetricResult

Score a response using the rubric criteria.

Args: user_input: The question or input provided to the system response: The response generated by the system retrieved_contexts: Contexts retrieved for generating the response reference_contexts: Reference contexts for evaluation reference: The reference/ground truth answer

Returns: MetricResult with score (1-5) and feedback as reason

Source code in src/ragas/metrics/collections/domain_specific_rubrics/metric.py
async def ascore(
    self,
    user_input: t.Optional[str] = None,
    response: t.Optional[str] = None,
    retrieved_contexts: t.Optional[t.List[str]] = None,
    reference_contexts: t.Optional[t.List[str]] = None,
    reference: t.Optional[str] = None,
) -> MetricResult:
    """
    Score a response using the rubric criteria.

    Args:
        user_input: The question or input provided to the system
        response: The response generated by the system
        retrieved_contexts: Contexts retrieved for generating the response
        reference_contexts: Reference contexts for evaluation
        reference: The reference/ground truth answer

    Returns:
        MetricResult with score (1-5) and feedback as reason
    """
    prompt_input = RubricScoreInput(
        user_input=user_input,
        response=response,
        retrieved_contexts=retrieved_contexts,
        reference_contexts=reference_contexts,
        reference=reference,
    )

    prompt_str = self.scoring_prompt.to_string(prompt_input)
    result: RubricScoreOutput = await self.llm.agenerate(
        prompt_str, RubricScoreOutput
    )

    return MetricResult(value=float(result.score), reason=result.feedback)

RubricsScoreWithoutReference

RubricsScoreWithoutReference(
    llm: InstructorBaseRagasLLM,
    rubrics: Optional[Dict[str, str]] = None,
    name: str = "rubrics_score_without_reference",
    **kwargs,
)

Bases: DomainSpecificRubrics

Convenience class for reference-free rubric-based evaluation.

This is equivalent to DomainSpecificRubrics(with_reference=False).

Source code in src/ragas/metrics/collections/domain_specific_rubrics/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    rubrics: t.Optional[t.Dict[str, str]] = None,
    name: str = "rubrics_score_without_reference",
    **kwargs,
):
    super().__init__(
        llm=llm, rubrics=rubrics, with_reference=False, name=name, **kwargs
    )

RubricsScoreWithReference

RubricsScoreWithReference(
    llm: InstructorBaseRagasLLM,
    rubrics: Optional[Dict[str, str]] = None,
    name: str = "rubrics_score_with_reference",
    **kwargs,
)

Bases: DomainSpecificRubrics

Convenience class for reference-based rubric-based evaluation.

This is equivalent to DomainSpecificRubrics(with_reference=True).

Source code in src/ragas/metrics/collections/domain_specific_rubrics/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    rubrics: t.Optional[t.Dict[str, str]] = None,
    name: str = "rubrics_score_with_reference",
    **kwargs,
):
    super().__init__(
        llm=llm, rubrics=rubrics, with_reference=True, name=name, **kwargs
    )

FactualCorrectness

FactualCorrectness(
    llm: InstructorBaseRagasLLM,
    mode: Literal["precision", "recall", "f1"] = "f1",
    beta: float = 1.0,
    atomicity: Literal["low", "high"] = "low",
    coverage: Literal["low", "high"] = "low",
    name: str = "factual_correctness",
    **kwargs,
)

Bases: BaseMetric

Modern v2 implementation of factual correctness evaluation.

Evaluates the factual correctness of responses by comparing claims made in the response against a reference text. Uses claim decomposition and natural language inference (NLI) to verify claims in both directions.

The metric supports three evaluation modes: - Precision: What fraction of response claims are supported by reference - Recall: What fraction of reference claims are covered by response - F1: Harmonic mean of precision and recall (with configurable beta)

The metric also supports configurable claim decomposition: - Atomicity: "low" (fewer, broader claims) vs "high" (more, atomic claims) - Coverage: "low" (partial coverage) vs "high" (comprehensive coverage)

Usage: >>> import instructor >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import FactualCorrectness >>> >>> # Setup dependencies >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance >>> metric = FactualCorrectness(llm=llm, mode="f1", beta=1.0) >>> >>> # Single evaluation >>> result = await metric.ascore( ... response="Einstein was born in Germany in 1879.", ... reference="Albert Einstein was born in Ulm, Germany on March 14, 1879." ... ) >>> print(f"Factual Correctness: {result.value}")

Attributes: llm: Modern instructor-based LLM for claim decomposition and NLI evaluation mode: Evaluation mode ("precision", "recall", or "f1") beta: Beta parameter for F1 score (>1 favors recall, <1 favors precision) atomicity: Claim decomposition atomicity ("low" or "high") coverage: Claim decomposition coverage ("low" or "high") name: The metric name allowed_values: Score range (0.0 to 1.0, higher is better)

Args: llm: Modern instructor-based LLM for claim decomposition and NLI evaluation mode: Evaluation mode ("precision", "recall", or "f1") beta: Beta parameter for F1 score (>1 favors recall, <1 favors precision) atomicity: Claim decomposition atomicity ("low" or "high") coverage: Claim decomposition coverage ("low" or "high") name: The metric name

Source code in src/ragas/metrics/collections/factual_correctness/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    mode: t.Literal["precision", "recall", "f1"] = "f1",
    beta: float = 1.0,
    atomicity: t.Literal["low", "high"] = "low",
    coverage: t.Literal["low", "high"] = "low",
    name: str = "factual_correctness",
    **kwargs,
):
    """
    Initialize FactualCorrectness metric with required components.

    Args:
        llm: Modern instructor-based LLM for claim decomposition and NLI evaluation
        mode: Evaluation mode ("precision", "recall", or "f1")
        beta: Beta parameter for F1 score (>1 favors recall, <1 favors precision)
        atomicity: Claim decomposition atomicity ("low" or "high")
        coverage: Claim decomposition coverage ("low" or "high")
        name: The metric name
    """
    self.mode = mode
    self.beta = beta
    self.atomicity = atomicity
    self.coverage = coverage
    self.prompt = ClaimDecompositionPrompt()
    self.nli_prompt = NLIStatementPrompt()

    # Validate beta parameter
    if not isinstance(beta, (int, float)):
        raise ValueError(
            "Beta must be a float. A beta > 1 gives more weight to recall, while beta < 1 favors precision."
        )

    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(response: str, reference: str) -> MetricResult

Calculate factual correctness score.

Args: response: The response to evaluate for factual correctness reference: The reference text to check claims against

Returns: MetricResult with factual correctness score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/factual_correctness/metric.py
async def ascore(self, response: str, reference: str) -> MetricResult:
    """
    Calculate factual correctness score.

    Args:
        response: The response to evaluate for factual correctness
        reference: The reference text to check claims against

    Returns:
        MetricResult with factual correctness score (0.0-1.0, higher is better)
    """
    require(response=response, reference=reference)

    # Precision: what fraction of the RESPONSE's claims the reference
    # supports. Recall: what fraction of the REFERENCE's claims the response
    # supports. Each direction is decomposed and verified independently, and
    # only the directions the mode needs are computed.
    precision = recall = float("nan")
    sections: List[str] = []
    if self.mode != "recall":
        verdicts = await self._judge_direction(response, reference)
        precision = _supported_fraction(verdicts)
        sections.append(
            _direction_reason("response", "reference", "Precision", verdicts)
        )
    if self.mode != "precision":
        verdicts = await self._judge_direction(reference, response)
        recall = _supported_fraction(verdicts)
        sections.append(
            _direction_reason("reference", "response", "Recall", verdicts)
        )

    if self.mode == "precision":
        score = precision
    elif self.mode == "recall":
        score = recall
    else:  # f-beta
        score = fbeta_from_precision_recall(precision, recall, self.beta)

    reason = "\n\n".join(sections)
    # NaN means "no claims to judge" (e.g. a one-word answer or an
    # abstention), which is undefined, not a failure.
    if np.isnan(score):
        return MetricResult(value=float("nan"), reason=reason)
    return MetricResult(value=float(np.round(score, 2)), reason=reason)

Faithfulness

Faithfulness(
    llm: InstructorBaseRagasLLM,
    name: str = "faithfulness",
    **kwargs,
)

Bases: BaseMetric

Faithfulness metric using multi-step pipeline evaluation.

Measures how factually consistent a response is with the retrieved context. A response is considered faithful if all its claims can be supported by the context.

The metric works by: 1. Breaking down the response into atomic statements 2. Checking each statement against the retrieved contexts using NLI 3. Computing faithfulness as the ratio of supported statements

This implementation uses modern instructor LLMs with structured output. Only supports modern components - legacy wrappers are rejected with clear error messages.

Usage: >>> import instructor >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import Faithfulness >>> >>> # Setup dependencies >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance >>> metric = Faithfulness(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... user_input="Where was Einstein born?", ... response="Einstein was born in Germany on 14th March 1879.", ... retrieved_contexts=["Albert Einstein was born in Germany..."] ... ) >>> print(f"Faithfulness Score: {result.value}")

Attributes: llm: Modern instructor-based LLM for statement generation and NLI evaluation name: The metric name allowed_values: Score range (0.0 to 1.0, higher is better)

Args: llm: Modern instructor-based LLM for statement generation and NLI evaluation name: The metric name

Source code in src/ragas/metrics/collections/faithfulness/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "faithfulness",
    **kwargs,
):
    """
    Initialize Faithfulness metric with required components.

    Args:
        llm: Modern instructor-based LLM for statement generation and NLI evaluation
        name: The metric name
    """
    self.statement_generator_prompt = StatementGeneratorPrompt()
    self.nli_statement_prompt = NLIStatementPrompt()

    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    user_input: str,
    response: str,
    retrieved_contexts: List[str],
) -> MetricResult

Calculate faithfulness score using multi-step pipeline.

Args: user_input: The original question response: The response to evaluate for faithfulness retrieved_contexts: The retrieved contexts to check against

Returns: MetricResult with faithfulness score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/faithfulness/metric.py
async def ascore(
    self, user_input: str, response: str, retrieved_contexts: List[str]
) -> MetricResult:
    """
    Calculate faithfulness score using multi-step pipeline.

    Args:
        user_input: The original question
        response: The response to evaluate for faithfulness
        retrieved_contexts: The retrieved contexts to check against

    Returns:
        MetricResult with faithfulness score (0.0-1.0, higher is better)
    """
    require(
        response=response,
        user_input=user_input,
        retrieved_contexts=retrieved_contexts,
    )

    # Step 1 (extract): break the response into atomic statements.
    statements = await self._create_statements(user_input, response)

    if not statements:
        # No statements generated - return NaN like legacy
        return MetricResult(
            value=float("nan"),
            reason="No statements could be extracted from the response, so "
            "faithfulness is undefined.",
        )

    # Step 2 (verify): check each statement against the joined contexts.
    return await self._verify_statements(statements, "\n".join(retrieved_contexts))

FaithfulnesswithHHEM

FaithfulnesswithHHEM(
    llm: InstructorBaseRagasLLM,
    name: str = "faithfulness_with_hhem",
    device: str = "cpu",
    batch_size: int = 10,
    classifier: Optional[Any] = None,
    revision: str = HHEM_REVISION,
    **kwargs,
)

Bases: Faithfulness

Faithfulness scored by Vectara's HHEM cross-encoder instead of the LLM.

Shares the first half of :class:~ragas.metrics.collections.Faithfulness: the LLM breaks the response into atomic statements. Each statement is then checked against the retrieved contexts by a local NLI model rather than by a second LLM call, which is cheaper and more reproducible at the cost of a model download.

Runs third-party code

The model is loaded with trust_remote_code=True, so importing it executes code published on the HuggingFace Hub. That is required by this model's architecture, and it is what the legacy metric always did, but it is worth knowing before adding this to a pipeline. The code is pinned to a reviewed commit (revision), so a new push to the model repo does not change what runs.

Checks only what the context states

HHEM is a strict entailment model: a statement is supported only if the context says it. The LLM-judged :class:Faithfulness also credits a statement the context implies through outside knowledge, e.g. that the "First AFL-NFL World Championship Game" was the first Super Bowl; HHEM scores that 0.

Requires transformers, which ragas does not install by default:

pip install transformers

Usage: >>> from openai import AsyncOpenAI >>> from ragas.llms import llm_factory >>> from ragas.metrics.collections import FaithfulnesswithHHEM >>> >>> llm = llm_factory("", client=AsyncOpenAI()) >>> >>> metric = FaithfulnesswithHHEM(llm=llm) >>> result = await metric.ascore( ... user_input="Where is the Eiffel Tower?", ... response="The Eiffel Tower is in Paris.", ... retrieved_contexts=["The Eiffel Tower is located in Paris, France."], ... ) >>> print(result.value)

Unlike the legacy metric, the model is loaded on first use rather than at construction, so building the metric neither downloads nor requires anything.

Attributes: llm: Modern instructor-based LLM, used for statement generation only device: Torch device to load the classifier onto batch_size: Statements scored per forward pass, to bound memory name: The metric name allowed_values: Score range (0.0 to 1.0)

name: The metric name
device: Torch device for the classifier
batch_size: Statements per forward pass
classifier: A pre-built model. Supplying one skips the download --
    useful for tests and for pinning a local copy.
revision: The Hub commit of the model (and of the remote code it
    runs) to load. Defaults to the reviewed ``HHEM_REVISION``.
Source code in src/ragas/metrics/collections/faithfulness_hhem/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "faithfulness_with_hhem",
    device: str = "cpu",
    batch_size: int = 10,
    classifier: t.Optional[t.Any] = None,
    revision: str = HHEM_REVISION,
    **kwargs,
):
    """
    Args:
        llm: Modern instructor-based LLM, used only to generate statements
        name: The metric name
        device: Torch device for the classifier
        batch_size: Statements per forward pass
        classifier: A pre-built model. Supplying one skips the download --
            useful for tests and for pinning a local copy.
        revision: The Hub commit of the model (and of the remote code it
            runs) to load. Defaults to the reviewed ``HHEM_REVISION``.
    """
    super().__init__(llm=llm, name=name, **kwargs)
    self.device = device
    self.batch_size = batch_size
    self._classifier = classifier
    self.revision = revision

classifier property

classifier: Any

The NLI model, downloaded on first access.

ImageSourcePolicy dataclass

ImageSourcePolicy(
    local_dir: Optional[Union[str, PathLike[str]]] = None,
    allow_private_urls: bool = False,
    max_bytes: int = MAX_DOWNLOAD_SIZE_BYTES,
    on_blocked: OnBlocked = "raise",
)

Where a multimodal metric may load context images from.

The default policy accepts data URIs and public http(s) URLs, reads no local files, and raises :class:~ragas.exceptions.ImageLoadError for an image reference it refuses or cannot load.

Args: local_dir: Directory local image paths are read from. Relative paths are resolved against it; absolute paths must lie inside it. Any path that resolves outside it, symlinks included, is refused. None (the default) reads no local files. allow_private_urls: Also fetch URLs whose host resolves to a private, loopback, link-local or otherwise non-public address. Only for data you trust, e.g. images on an internal server. max_bytes: Largest image, downloaded or read, in bytes. on_blocked: What to do with an image reference that is refused or fails to load. "raise" (the default) raises ImageLoadError; "warn" logs a warning and sends the reference to the LLM as text, so the score is computed without the image.

Note: The host is resolved for the check and again when connecting, so a hostname whose DNS answer changes in between (DNS rebinding) can still reach an internal address. For untrusted data, also restrict outbound network access where the evaluation runs.

InstanceSpecificRubrics

InstanceSpecificRubrics(
    llm: InstructorBaseRagasLLM,
    name: str = "instance_specific_rubrics",
    **kwargs,
)

Bases: BaseMetric

Evaluates responses using instance-specific rubrics where each sample has its own criteria.

Unlike DomainSpecificRubrics which uses the same rubric for all samples, this metric allows each evaluation instance to define its own scoring criteria. This is useful when: - Different questions require different evaluation criteria - You want to customize scoring based on the specific task or context - Evaluation criteria vary across your dataset

The metric works by: 1. Taking the input, response, and a rubrics dictionary for each sample 2. Using an LLM to evaluate the response against the provided rubric 3. Returning a score with detailed feedback

Usage: >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import InstanceSpecificRubrics >>> >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> metric = InstanceSpecificRubrics(llm=llm) >>> >>> # Each sample can have different rubrics >>> rubrics = { ... "score1_description": "The response is completely off-topic", ... "score2_description": "The response is partially relevant but misses key points", ... "score3_description": "The response addresses the topic but lacks depth", ... "score4_description": "The response is good with minor improvements needed", ... "score5_description": "The response is excellent and comprehensive", ... } >>> >>> result = await metric.ascore( ... user_input="Explain quantum computing", ... response="Quantum computing uses quantum bits...", ... rubrics=rubrics, ... ) >>> print(f"Score: {result.value}, Feedback: {result.reason}")

Attributes: llm: Modern instructor-based LLM for evaluation name: The metric name (default: "instance_specific_rubrics")

Source code in src/ragas/metrics/collections/instance_specific_rubrics/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "instance_specific_rubrics",
    **kwargs,
):
    self.scoring_prompt = InstanceRubricScorePrompt()

    super().__init__(llm=llm, name=name, allowed_values=(1.0, 5.0), **kwargs)

ascore async

ascore(
    rubrics: Dict[str, str],
    user_input: Optional[str] = None,
    response: Optional[str] = None,
    retrieved_contexts: Optional[List[str]] = None,
    reference_contexts: Optional[List[str]] = None,
    reference: Optional[str] = None,
) -> MetricResult

Score a response using instance-specific rubric criteria.

Args: rubrics: Dictionary mapping score descriptions (e.g., "score1_description") to criteria user_input: The question or input provided to the system response: The response generated by the system retrieved_contexts: Contexts retrieved for generating the response reference_contexts: Reference contexts for evaluation reference: The reference/ground truth answer

Returns: MetricResult with score and feedback as reason

Raises: ValueError: If rubrics is not provided

Source code in src/ragas/metrics/collections/instance_specific_rubrics/metric.py
async def ascore(
    self,
    rubrics: t.Dict[str, str],
    user_input: t.Optional[str] = None,
    response: t.Optional[str] = None,
    retrieved_contexts: t.Optional[t.List[str]] = None,
    reference_contexts: t.Optional[t.List[str]] = None,
    reference: t.Optional[str] = None,
) -> MetricResult:
    """
    Score a response using instance-specific rubric criteria.

    Args:
        rubrics: Dictionary mapping score descriptions (e.g., "score1_description") to criteria
        user_input: The question or input provided to the system
        response: The response generated by the system
        retrieved_contexts: Contexts retrieved for generating the response
        reference_contexts: Reference contexts for evaluation
        reference: The reference/ground truth answer

    Returns:
        MetricResult with score and feedback as reason

    Raises:
        ValueError: If rubrics is not provided
    """
    require(rubrics=rubrics)

    prompt_input = InstanceRubricScoreInput(
        user_input=user_input,
        response=response,
        retrieved_contexts=retrieved_contexts,
        reference_contexts=reference_contexts,
        reference=reference,
        rubrics=rubrics,
    )

    prompt_str = self.scoring_prompt.to_string(prompt_input)
    result: InstanceRubricScoreOutput = await self.llm.agenerate(
        prompt_str, InstanceRubricScoreOutput
    )

    return MetricResult(value=float(result.score), reason=result.feedback)

MultiModalFaithfulness

MultiModalFaithfulness(
    llm: InstructorBaseRagasLLM,
    name: str = "multi_modal_faithfulness",
    image_policy: Optional[ImageSourcePolicy] = None,
    **kwargs,
)

Bases: BaseMetric

MultiModalFaithfulness metric for evaluating response faithfulness against both visual and textual context.

Measures how factually consistent a response is with the retrieved context, which can include both text and images. A response is considered faithful if all its claims can be supported by the provided contexts.

The metric returns a binary score: - 1.0 if the response is faithful to the contexts - 0.0 if the response is not faithful

This implementation uses modern instructor LLMs with vision capabilities for multimodal evaluation.

Usage: >>> import instructor >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import MultiModalFaithfulness >>> >>> # Setup dependencies (use a vision-capable model) >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) # Vision-capable model >>> >>> # Create metric instance >>> metric = MultiModalFaithfulness(llm=llm) >>> >>> # Single evaluation with image context >>> result = await metric.ascore( ... response="The Tesla Model X is an electric SUV.", ... retrieved_contexts=["path/to/tesla_image.jpg", "Tesla makes electric vehicles."] ... ) >>> print(f"Faithfulness Score: {result.value}")

Attributes: llm: Modern instructor-based LLM with vision capabilities name: The metric name allowed_values: Score range (0.0 or 1.0)

Note: This metric requires an LLM that accepts image input to evaluate image contexts. Check the model's page on your provider's site.

Args: llm: Modern instructor-based LLM with vision capabilities name: The metric name image_policy: Where context images may be loaded from. The default fetches public http(s) URLs and data URIs only, and reads no local files; pass ImageSourcePolicy(local_dir=...) to read images from a directory. See ImageSourcePolicy.

Source code in src/ragas/metrics/collections/multi_modal_faithfulness/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "multi_modal_faithfulness",
    image_policy: t.Optional[ImageSourcePolicy] = None,
    **kwargs,
):
    """
    Initialize MultiModalFaithfulness metric with required components.

    Args:
        llm: Modern instructor-based LLM with vision capabilities
        name: The metric name
        image_policy: Where context images may be loaded from. The default
            fetches public http(s) URLs and data URIs only, and reads no
            local files; pass ``ImageSourcePolicy(local_dir=...)`` to read
            images from a directory. See ``ImageSourcePolicy``.
    """
    self.image_policy = image_policy or ImageSourcePolicy()
    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    response: str, retrieved_contexts: List[str]
) -> MetricResult

Calculate multimodal faithfulness score.

Args: response: The response to evaluate for faithfulness retrieved_contexts: List of retrieved contexts (text strings or image paths/URLs/base64 data)

Returns: MetricResult with faithfulness score (0.0 or 1.0)

Raises: ValueError: If response or retrieved_contexts is missing

Source code in src/ragas/metrics/collections/multi_modal_faithfulness/metric.py
async def ascore(
    self,
    response: str,
    retrieved_contexts: List[str],
) -> MetricResult:
    """
    Calculate multimodal faithfulness score.

    Args:
        response: The response to evaluate for faithfulness
        retrieved_contexts: List of retrieved contexts (text strings or
                          image paths/URLs/base64 data)

    Returns:
        MetricResult with faithfulness score (0.0 or 1.0)

    Raises:
        ValueError: If response or retrieved_contexts is missing
    """
    require(response=response, retrieved_contexts=retrieved_contexts)

    # Build multimodal message content
    message_content = build_multimodal_message_content(
        instruction=MULTIMODAL_FAITHFULNESS_INSTRUCTION,
        response=response,
        retrieved_contexts=retrieved_contexts,
        image_policy=self.image_policy,
    )

    # Call the LLM with multimodal content
    result = await self._evaluate_faithfulness(message_content)

    # Return score based on faithfulness verdict
    score = 1.0 if result.faithful else 0.0
    return MetricResult(value=score, reason=result.reason)

MultiModalRelevance

MultiModalRelevance(
    llm: InstructorBaseRagasLLM,
    name: str = "multi_modal_relevance",
    image_policy: Optional[ImageSourcePolicy] = None,
    **kwargs,
)

Bases: BaseMetric

MultiModalRelevance metric for evaluating response relevance against both visual and textual context.

Measures whether a response appropriately addresses the user's question and is in line with the retrieved context, which can include both text and images.

The metric returns a binary score: - 1.0 if the response is relevant to the question and contexts - 0.0 if the response is not relevant

This implementation uses modern instructor LLMs with vision capabilities for multimodal evaluation.

Usage: >>> import instructor >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import MultiModalRelevance >>> >>> # Setup dependencies (use a vision-capable model) >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) # Vision-capable model >>> >>> # Create metric instance >>> metric = MultiModalRelevance(llm=llm) >>> >>> # Single evaluation with image context >>> result = await metric.ascore( ... user_input="What type of vehicle is shown in the image?", ... response="The image shows a Tesla Model X, which is an electric SUV.", ... retrieved_contexts=["path/to/tesla_image.jpg", "Tesla makes electric vehicles."] ... ) >>> print(f"Relevance Score: {result.value}")

Attributes: llm: Modern instructor-based LLM with vision capabilities name: The metric name allowed_values: Score range (0.0 or 1.0)

Note: This metric requires an LLM that accepts image input to evaluate image contexts. Check the model's page on your provider's site.

Args: llm: Modern instructor-based LLM with vision capabilities name: The metric name image_policy: Where context images may be loaded from. The default fetches public http(s) URLs and data URIs only, and reads no local files; pass ImageSourcePolicy(local_dir=...) to read images from a directory. See ImageSourcePolicy.

Source code in src/ragas/metrics/collections/multi_modal_relevance/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "multi_modal_relevance",
    image_policy: t.Optional[ImageSourcePolicy] = None,
    **kwargs,
):
    """
    Initialize MultiModalRelevance metric with required components.

    Args:
        llm: Modern instructor-based LLM with vision capabilities
        name: The metric name
        image_policy: Where context images may be loaded from. The default
            fetches public http(s) URLs and data URIs only, and reads no
            local files; pass ``ImageSourcePolicy(local_dir=...)`` to read
            images from a directory. See ``ImageSourcePolicy``.
    """
    self.image_policy = image_policy or ImageSourcePolicy()
    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    user_input: str,
    response: str,
    retrieved_contexts: List[str],
) -> MetricResult

Calculate multimodal relevance score.

Args: user_input: The user's question or input response: The response to evaluate for relevance retrieved_contexts: List of retrieved contexts (text strings or image paths/URLs/base64 data)

Returns: MetricResult with relevance score (0.0 or 1.0)

Raises: ValueError: If user_input, response, or retrieved_contexts is missing

Source code in src/ragas/metrics/collections/multi_modal_relevance/metric.py
async def ascore(
    self,
    user_input: str,
    response: str,
    retrieved_contexts: List[str],
) -> MetricResult:
    """
    Calculate multimodal relevance score.

    Args:
        user_input: The user's question or input
        response: The response to evaluate for relevance
        retrieved_contexts: List of retrieved contexts (text strings or
                          image paths/URLs/base64 data)

    Returns:
        MetricResult with relevance score (0.0 or 1.0)

    Raises:
        ValueError: If user_input, response, or retrieved_contexts is missing
    """
    require(
        user_input=user_input,
        response=response,
        retrieved_contexts=retrieved_contexts,
    )

    # Build multimodal message content
    message_content = build_multimodal_relevance_message_content(
        instruction=MULTIMODAL_RELEVANCE_INSTRUCTION,
        user_input=user_input,
        response=response,
        retrieved_contexts=retrieved_contexts,
        image_policy=self.image_policy,
    )

    # Call the LLM with multimodal content
    result = await self._evaluate_relevance(message_content)

    # Return score based on relevance verdict
    score = 1.0 if result.relevant else 0.0
    return MetricResult(value=score, reason=result.reason)

NoiseSensitivity

NoiseSensitivity(
    llm: InstructorBaseRagasLLM,
    name: str = "noise_sensitivity",
    mode: Literal["relevant", "irrelevant"] = "relevant",
    **kwargs,
)

Bases: BaseMetric

Modern v2 implementation of noise sensitivity evaluation.

Measures how often a system makes errors by providing incorrect responses when utilizing either relevant or irrelevant retrieved documents.

The metric works by: 1. Decomposing reference and response into atomic statements 2. Using NLI to evaluate statement faithfulness against each retrieved context 3. Computing noise sensitivity based on incorrect claims from relevant/irrelevant contexts

This implementation uses modern instructor LLMs with structured output. Only supports modern components - legacy wrappers are rejected with clear error messages.

Usage: >>> import instructor >>> from openai import AsyncOpenAI >>> from ragas.llms.base import instructor_llm_factory >>> from ragas.metrics.collections import NoiseSensitivity >>> >>> # Setup dependencies >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance >>> metric = NoiseSensitivity(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... user_input="What is LIC known for?", ... response="LIC is the largest insurance company in India...", ... reference="LIC is known for managing investments...", ... retrieved_contexts=["LIC was established in 1956...", ...] ... ) >>> print(f"Noise Sensitivity: {result.value}") >>> >>> # Test irrelevant context sensitivity >>> irrelevant_metric = NoiseSensitivity(llm=llm, mode="irrelevant")

Attributes: llm: Modern instructor-based LLM for statement generation and NLI evaluation name: The metric name mode: Either "relevant" or "irrelevant" context sensitivity allowed_values: Score range (0.0 to 1.0, lower is better)

Args: llm: Modern instructor-based LLM for statement generation and NLI evaluation name: The metric name mode: Either "relevant" or "irrelevant" context sensitivity mode

Source code in src/ragas/metrics/collections/noise_sensitivity/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "noise_sensitivity",
    mode: Literal["relevant", "irrelevant"] = "relevant",
    **kwargs,
):
    """
    Initialize NoiseSensitivity metric with required components.

    Args:
        llm: Modern instructor-based LLM for statement generation and NLI evaluation
        name: The metric name
        mode: Either "relevant" or "irrelevant" context sensitivity mode
    """
    self.mode = mode
    self.statement_prompt = StatementGeneratorPrompt()
    self.faithfulness_prompt = NLIStatementPrompt()

    # Validate mode
    if mode not in {"relevant", "irrelevant"}:
        raise ValueError(
            f"Invalid argument passed for 'mode': {mode}. Must be 'relevant' or 'irrelevant'."
        )

    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    user_input: str,
    response: str,
    reference: str,
    retrieved_contexts: List[str],
) -> MetricResult

Calculate noise sensitivity score.

Args: user_input: The original question response: The answer to evaluate reference: The ground truth reference retrieved_contexts: The retrieved contexts used to generate the response

Returns: MetricResult with noise sensitivity score (0.0-1.0, lower is better)

Source code in src/ragas/metrics/collections/noise_sensitivity/metric.py
async def ascore(
    self,
    user_input: str,
    response: str,
    reference: str,
    retrieved_contexts: List[str],
) -> MetricResult:
    """
    Calculate noise sensitivity score.

    Args:
        user_input: The original question
        response: The answer to evaluate
        reference: The ground truth reference
        retrieved_contexts: The retrieved contexts used to generate the response

    Returns:
        MetricResult with noise sensitivity score (0.0-1.0, lower is better)
    """
    require(
        reference=reference,
        user_input=user_input,
        response=response,
        retrieved_contexts=retrieved_contexts,
    )

    # Step 1: Decompose reference and response into statements (independent)
    gt_statements, ans_statements = await asyncio.gather(
        self._decompose_answer_into_statements(reference, user_input),
        self._decompose_answer_into_statements(response, user_input),
    )

    # Step 2: Check both statement sets against every retrieved context, and
    # the answer statements against the reference. All 2N+1 checks are
    # independent, so they run concurrently instead of as 2N+1 sequential
    # round-trips under one per-job timeout (#58).
    n = len(retrieved_contexts)
    checks = await gather_bounded(
        [
            *(
                self._evaluate_statement_faithfulness(gt_statements, ctx)
                for ctx in retrieved_contexts
            ),
            *(
                self._evaluate_statement_faithfulness(ans_statements, ctx)
                for ctx in retrieved_contexts
            ),
            self._evaluate_statement_faithfulness(ans_statements, reference),
        ]
    )
    gt_verdictslist = checks[:n]
    ans_verdictslist = checks[n : 2 * n]
    gt_to_ans_verdicts = checks[2 * n]

    # A check whose verdicts couldn't be matched to the statements leaves a
    # hole in the matrices: the score is undefined (#52).
    if any(v is None for v in checks):
        return MetricResult(value=float("nan"))

    # Step 3: Build matrices for computation (exact legacy shape handling)
    answers = {}
    answers["retrieved2ground_truth"] = np.array(gt_verdictslist).T
    answers["retrieved2answer"] = np.array(ans_verdictslist).T
    # Wrap in another array to match legacy shape handling
    answers["ground_truth2answer"] = np.array([np.array(gt_to_ans_verdicts)])

    # Convert all to boolean arrays
    answers = {k: v.astype(bool) for k, v in answers.items()}

    # Step 4: Compute noise sensitivity score
    noisy = self._noisy_statements(answers)
    score = float(np.mean(noisy))
    reason = itemized(
        f"{count_phrase(int(noisy.sum()), len(noisy), 'response statement')} "
        f"{count_verb(int(noisy.sum()), len(noisy))} incorrect (not supported "
        f"by the reference) yet supported by "
        f"{'a relevant' if self.mode == 'relevant' else 'an irrelevant'} "
        f"retrieved context; lower is better.",
        ((stmt, not bool(flag), None) for stmt, flag in zip(ans_statements, noisy)),
        yes="ok",
        no="noise-induced error",
    )
    return MetricResult(value=score, reason=reason)

QuotedSpansAlignment

QuotedSpansAlignment(
    name: str = "quoted_spans_alignment",
    casefold: bool = True,
    min_span_words: int = 3,
    **base_kwargs,
)

Bases: BaseMetric

Measure citation alignment for quoted spans in model-generated answers.

This metric computes the fraction of quoted spans appearing verbatim in any of the provided source passages. If an answer quotes facts that cannot be found in the sources, the metric will reflect that drift.

The metric performs light normalization by collapsing whitespace and lower-casing strings. You can adjust the minimum length of a quoted span and choose to disable case folding if desired.

Usage: >>> from ragas.metrics.collections import QuotedSpansAlignment >>> >>> metric = QuotedSpansAlignment() >>> >>> result = await metric.ascore( ... response='The study found that "machine learning models improve accuracy".', ... retrieved_contexts=["Machine learning models improve accuracy by 15%."] ... ) >>> print(f"Score: {result.value}") >>> >>> results = await metric.abatch_score([ ... { ... "response": 'He said "the results are significant".', ... "retrieved_contexts": ["The results are significant according to the paper."] ... }, ... ])

Attributes: name: The metric name (default: "quoted_spans_alignment") casefold: Whether to normalize text by lower-casing before matching. min_span_words: Minimum number of words in a quoted span. allowed_values: Score range (0.0 to 1.0)

Args: name: The metric name. casefold: Whether to normalize text by lower-casing before matching. min_span_words: Minimum number of words in a quoted span. **base_kwargs: Additional arguments passed to BaseMetric.

Source code in src/ragas/metrics/collections/quoted_spans/metric.py
def __init__(
    self,
    name: str = "quoted_spans_alignment",
    casefold: bool = True,
    min_span_words: int = 3,
    **base_kwargs,
):
    """
    Initialize QuotedSpansAlignment metric.

    Args:
        name: The metric name.
        casefold: Whether to normalize text by lower-casing before matching.
        min_span_words: Minimum number of words in a quoted span.
        **base_kwargs: Additional arguments passed to BaseMetric.
    """
    super().__init__(name=name, **base_kwargs)
    self.casefold = casefold
    self.min_span_words = min_span_words

ascore async

ascore(
    response: str, retrieved_contexts: List[str]
) -> MetricResult

Calculate quoted spans alignment score asynchronously.

Args: response: The model response containing quoted spans. retrieved_contexts: List of source passages to check against.

Returns: MetricResult with alignment score (0.0-1.0) and metadata containing matched and total counts.

Source code in src/ragas/metrics/collections/quoted_spans/metric.py
async def ascore(
    self,
    response: str,
    retrieved_contexts: t.List[str],
) -> MetricResult:
    """
    Calculate quoted spans alignment score asynchronously.

    Args:
        response: The model response containing quoted spans.
        retrieved_contexts: List of source passages to check against.

    Returns:
        MetricResult with alignment score (0.0-1.0) and metadata containing
        matched and total counts.
    """
    if not isinstance(response, str):
        return MetricResult(
            value=0.0,
            reason="Invalid input: response must be a string",
        )

    if not isinstance(retrieved_contexts, list):
        return MetricResult(
            value=0.0,
            reason="Invalid input: retrieved_contexts must be a list of strings",
        )

    spans = extract_quoted_spans(response, min_len=self.min_span_words)

    if not spans:
        return MetricResult(
            value=1.0,
            reason="No quoted spans found in response",
        )

    matched, total = count_matched_spans(
        spans, retrieved_contexts, casefold=self.casefold
    )

    score = matched / total if total > 0 else 0.0

    reason = f"Matched {matched}/{total} quoted spans"
    return MetricResult(value=float(score), reason=reason)

ResponseGroundedness

ResponseGroundedness(
    llm: InstructorBaseRagasLLM,
    name: str = "response_groundedness",
    max_retries: int = 5,
    **kwargs,
)

Bases: DualJudgeMetric

Response Groundedness metric using dual-judge evaluation.

Evaluates how well grounded a response is in the retrieved contexts using a dual-judge system. This metric averages two distinct judge prompts to ensure robust evaluation.

The metric uses NVIDIA's proven dual-judge approach: 1. Judge 1: Direct groundedness evaluation with structured instructions 2. Judge 2: Alternative perspective for fairness 3. Average both judges for final score

Rating scale: 0 (not grounded), 1 (partially grounded), 2 (fully grounded) Final score: Average of both judges converted to 0.0-1.0 scale

Usage: >>> import instructor >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import ResponseGroundedness >>> >>> # Setup dependencies >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance >>> metric = ResponseGroundedness(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... response="Einstein was born in Germany in 1879.", ... retrieved_contexts=["Albert Einstein was born in Ulm, Germany on March 14, 1879."] ... ) >>> print(f"Response Groundedness: {result.value}")

Attributes: llm: Modern instructor-based LLM for dual-judge evaluation name: The metric name allowed_values: Score range (0.0 to 1.0, higher is better) max_retries: Attempts per judge; a retry after an off-scale rating is a fresh sample, not a resend of the same request

name: The metric name
max_retries: Attempts per judge (see ``DualJudgeMetric``)
Source code in src/ragas/metrics/collections/response_groundedness/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "response_groundedness",
    max_retries: int = 5,
    **kwargs,
):
    """
    Args:
        llm: Modern instructor-based LLM for dual-judge evaluation
        name: The metric name
        max_retries: Attempts per judge (see ``DualJudgeMetric``)
    """
    super().__init__(llm=llm, name=name, max_retries=max_retries, **kwargs)

ascore async

ascore(
    response: str, retrieved_contexts: List[str]
) -> MetricResult

Calculate response groundedness score using dual-judge evaluation.

Args: response: The response to evaluate for groundedness retrieved_contexts: The retrieved contexts to check groundedness against

Returns: MetricResult with response groundedness score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/response_groundedness/metric.py
async def ascore(
    self, response: str, retrieved_contexts: List[str]
) -> MetricResult:
    """
    Calculate response groundedness score using dual-judge evaluation.

    Args:
        response: The response to evaluate for groundedness
        retrieved_contexts: The retrieved contexts to check groundedness against

    Returns:
        MetricResult with response groundedness score (0.0-1.0, higher is better)
    """
    require(response=response, retrieved_contexts=retrieved_contexts)

    context_str = "\n".join(retrieved_contexts)
    if not response.strip() or not context_str.strip():
        return MetricResult(value=0.0)

    judge_input = ResponseGroundednessInput(response=response, context=context_str)
    return await self._judge_both(judge_input, judge_input)

SimpleCriteriaScore

SimpleCriteriaScore(
    llm: InstructorBaseRagasLLM,
    definition: str,
    name: str = "simple_criteria_score",
    strictness: int = 1,
    allowed_values: Tuple[float, float] = (0.0, 5.0),
    **kwargs,
)

Bases: CriteriaJudgeMetric

Score a submission against a criterion you define, numerically.

The sibling of :class:~ragas.metrics.collections.AspectCritic: same free-text criterion, but the LLM returns a score rather than a yes/no verdict. State the range you want inside the definition -- the metric does not impose one.

Usage: >>> from openai import AsyncOpenAI >>> from ragas.llms import llm_factory >>> from ragas.metrics.collections import SimpleCriteriaScore >>> >>> llm = llm_factory("", client=AsyncOpenAI()) >>> >>> metric = SimpleCriteriaScore( ... llm=llm, ... name="course_grained_score", ... definition="Score 0 to 5 by similarity to the reference.", ... allowed_values=(0.0, 5.0), ... ) >>> result = await metric.ascore( ... user_input="Where is the Eiffel Tower?", ... response="The Eiffel Tower is in Paris.", ... reference="The Eiffel Tower is located in Paris.", ... ) >>> print(result.value) # 5.0

Attributes: llm: Modern instructor-based LLM for evaluation definition: The criterion to score against strictness: How many independent scores to take a majority vote over name: The metric name allowed_values: Score range; widen it to match your definition

definition: The criterion to score against
name: The metric name
strictness: Number of independent scores to majority-vote over.
    Forced odd to avoid ties. Note this differs from the legacy
    ``SimpleCriteriaScore``, where the parameter was accepted but
    never took more than one sample, so the vote was a no-op.
allowed_values: The score range your definition asks for. Defaults
    to 0-5; the legacy metric imposed no range at all.
Source code in src/ragas/metrics/collections/simple_criteria/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    definition: str,
    name: str = "simple_criteria_score",
    strictness: int = 1,
    allowed_values: t.Tuple[float, float] = (0.0, 5.0),
    **kwargs,
):
    """
    Args:
        llm: Modern instructor-based LLM
        definition: The criterion to score against
        name: The metric name
        strictness: Number of independent scores to majority-vote over.
            Forced odd to avoid ties. Note this differs from the legacy
            ``SimpleCriteriaScore``, where the parameter was accepted but
            never took more than one sample, so the vote was a no-op.
        allowed_values: The score range your definition asks for. Defaults
            to 0-5; the legacy metric imposed no range at all.
    """
    super().__init__(
        llm=llm,
        definition=definition,
        name=name,
        strictness=strictness,
        allowed_values=allowed_values,
        **kwargs,
    )

SQLSemanticEquivalence

SQLSemanticEquivalence(
    llm: InstructorBaseRagasLLM,
    name: str = "sql_semantic_equivalence",
    **kwargs,
)

Bases: BaseMetric

Evaluates semantic equivalence between a generated SQL query and a reference query.

This metric uses an LLM to analyze whether two SQL queries would produce the same results when executed against the same database, regardless of syntactic differences. The metric considers the database schema context to make accurate equivalence judgments.

The metric returns: - 1.0 if the queries are semantically equivalent - 0.0 if the queries are not equivalent

Usage: >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import SQLSemanticEquivalence >>> >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> metric = SQLSemanticEquivalence(llm=llm) >>> >>> result = await metric.ascore( ... response="SELECT id, name FROM users WHERE active = true;", ... reference="SELECT id, name FROM users WHERE active = 1;", ... reference_contexts=[ ... "Table users: id (INT), name (VARCHAR), active (BOOLEAN)" ... ], ... ) >>> print(f"Equivalent: {result.value == 1.0}")

Attributes: llm: Modern instructor-based LLM for SQL analysis name: The metric name (default: "sql_semantic_equivalence")

Source code in src/ragas/metrics/collections/sql_semantic_equivalence/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "sql_semantic_equivalence",
    **kwargs,
):
    self.equivalence_prompt = SQLEquivalencePrompt()
    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    response: str,
    reference: str,
    reference_contexts: Optional[List[str]] = None,
) -> MetricResult

Calculate SQL semantic equivalence score.

Args: response: The generated SQL query to evaluate reference: The reference SQL query to compare against reference_contexts: List of database schema descriptions providing context for the comparison. These are joined with newlines.

Returns: MetricResult with equivalence score (1.0 if equivalent, 0.0 if not)

Source code in src/ragas/metrics/collections/sql_semantic_equivalence/metric.py
async def ascore(
    self,
    response: str,
    reference: str,
    reference_contexts: Optional[List[str]] = None,
) -> MetricResult:
    """
    Calculate SQL semantic equivalence score.

    Args:
        response: The generated SQL query to evaluate
        reference: The reference SQL query to compare against
        reference_contexts: List of database schema descriptions providing
            context for the comparison. These are joined with newlines.

    Returns:
        MetricResult with equivalence score (1.0 if equivalent, 0.0 if not)
    """
    if not isinstance(response, str) or not response.strip():
        raise ValueError("response must be a non-empty SQL query string")
    if not isinstance(reference, str) or not reference.strip():
        raise ValueError("reference must be a non-empty SQL query string")

    database_schema = ""
    if reference_contexts:
        database_schema = "\n".join(reference_contexts)

    input_data = SQLEquivalenceInput(
        reference=reference,
        response=response,
        database_schema=database_schema,
    )

    prompt_str = self.equivalence_prompt.to_string(input_data)
    result = await self.llm.agenerate(prompt_str, SQLEquivalenceOutput)

    score = 1.0 if result.equivalent else 0.0

    return MetricResult(
        value=score,
        reason=f"Response: {result.response_explanation}\nReference: {result.reference_explanation}",
    )

SummaryScore

SummaryScore(
    llm: InstructorBaseRagasLLM,
    name: str = "summary_score",
    length_penalty: bool = True,
    coeff: float = 0.5,
    **kwargs,
)

Bases: BaseMetric

Summary Score metric using multi-step pipeline evaluation.

Measures how well a summary captures important information from contexts by: 1. Extracting keyphrases from the original contexts 2. Generating yes/no questions from those keyphrases 3. Checking if the summary can answer those questions 4. Optionally penalizing overly long summaries for conciseness

This implementation uses modern instructor LLMs with structured output. Only supports modern components - legacy wrappers are rejected with clear error messages.

Usage: >>> import instructor >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import SummaryScore >>> >>> # Setup dependencies >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> # Create metric instance >>> metric = SummaryScore(llm=llm) >>> >>> # Single evaluation >>> result = await metric.ascore( ... reference_contexts=["Apple Inc. is a technology company..."], ... response="Apple is a tech company founded by Steve Jobs." ... ) >>> print(f"Summary Score: {result.value}") >>> >>> # Custom configuration (more conciseness focus) >>> concise_metric = SummaryScore( ... llm=llm, ... length_penalty=True, ... coeff=0.8 # More weight on conciseness ... )

Attributes: llm: Modern instructor-based LLM for keyphrase, question, and answer generation name: The metric name length_penalty: Whether to apply conciseness penalty for long summaries coeff: Weight for conciseness score (0.0=only QA, 1.0=only conciseness) allowed_values: Score range (0.0 to 1.0)

Args: llm: Modern instructor-based LLM for keyphrase, question, and answer generation name: The metric name length_penalty: Whether to apply conciseness penalty for long summaries coeff: Weight for conciseness score (0.0=only QA, 1.0=only conciseness)

Source code in src/ragas/metrics/collections/summary_score/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    name: str = "summary_score",
    length_penalty: bool = True,
    coeff: float = 0.5,
    **kwargs,
):
    """
    Initialize SummaryScore metric with required components.

    Args:
        llm: Modern instructor-based LLM for keyphrase, question, and answer generation
        name: The metric name
        length_penalty: Whether to apply conciseness penalty for long summaries
        coeff: Weight for conciseness score (0.0=only QA, 1.0=only conciseness)
    """
    self.length_penalty = length_penalty
    self.coeff = coeff
    self.extract_keyphrases_prompt = ExtractKeyphrasesPrompt()
    self.generate_questions_prompt = GenerateQuestionsPrompt()
    self.generate_answers_prompt = GenerateAnswersPrompt()

    # Validate coefficient
    if not (0.0 <= coeff <= 1.0):
        raise ValueError(f"Coefficient must be between 0.0 and 1.0, got {coeff}")

    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    reference_contexts: List[str], response: str
) -> MetricResult

Calculate summary score using multi-step pipeline.

Args: reference_contexts: The original contexts that were summarized response: The summary to evaluate

Returns: MetricResult with summary score (0.0-1.0)

Raises: ValueError: If reference_contexts is empty or response is empty/whitespace only

Source code in src/ragas/metrics/collections/summary_score/metric.py
async def ascore(
    self, reference_contexts: List[str], response: str
) -> MetricResult:
    """
    Calculate summary score using multi-step pipeline.

    Args:
        reference_contexts: The original contexts that were summarized
        response: The summary to evaluate

    Returns:
        MetricResult with summary score (0.0-1.0)

    Raises:
        ValueError: If reference_contexts is empty or response is empty/whitespace only
    """
    # Input validation
    if not reference_contexts or not any(ctx.strip() for ctx in reference_contexts):
        raise ValueError(
            "reference_contexts cannot be empty or contain only whitespace"
        )

    if not response or not response.strip():
        raise ValueError("response cannot be empty or whitespace only")

    # Step 1: Combine contexts and extract keyphrases
    text = "\n".join(reference_contexts)
    keyphrases = await self._extract_keyphrases(text)

    if not keyphrases:
        # Match legacy behavior: log error and continue with empty list
        logging.error("No keyphrases generated, unable to calculate the score.")
        keyphrases = []

    # Step 2: Generate questions from keyphrases
    questions = await self._generate_questions(text, keyphrases)

    if not questions:
        # Nothing to check the summary against: the score is undefined (#60).
        logging.warning(
            "SummaryScore: no questions generated from the reference contexts; "
            "returning NaN."
        )
        return MetricResult(
            value=float("nan"),
            reason="No questions could be generated from the source text, so "
            "the score is undefined.",
        )

    # Step 3: Check if summary can answer the questions
    answers = await self._generate_answers(response, questions)

    # Step 4: Calculate QA score
    qa_score = self._compute_qa_score(answers, len(questions))

    # Step 5: Calculate final score (with optional conciseness penalty)
    if self.length_penalty:
        conciseness_score = self._compute_conciseness_score(text, response)
        final_score = qa_score * (1 - self.coeff) + conciseness_score * self.coeff
        weighting = (
            f"QA score {qa_score:.2f} weighted {1 - self.coeff:.2f}, "
            f"conciseness {conciseness_score:.2f} weighted {self.coeff:.2f}."
        )
    else:
        final_score = qa_score
        weighting = f"QA score {qa_score:.2f} (no length penalty)."

    if len(answers) == len(questions):
        answered = sum(a.strip() == "1" for a in answers)
        reason = itemized(
            f"The summary answers "
            f"{count_phrase(answered, len(questions), 'question')} generated "
            f"from the source text. {weighting}",
            ((q, a.strip() == "1", None) for q, a in zip(questions, answers)),
            yes="answered",
            no="not answered",
        )
    else:
        reason = (
            f"The judge returned {len(answers)} answers for {len(questions)} "
            f"questions, so the QA score is undefined."
        )
    return MetricResult(value=float(final_score), reason=reason)

ToolCallAccuracy

ToolCallAccuracy(
    strict_order: bool = True,
    name: str = "tool_call_accuracy",
    **kwargs,
)

Bases: BaseMetric

Modern implementation of Tool Call Accuracy metric.

Measures how accurately an LLM agent makes tool calls compared to reference tool calls. This is a rule-based metric that evaluates: 1. Sequence alignment: Whether predicted and reference tool calls match in the required order 2. Argument accuracy: How well tool call arguments match between predicted and reference

The metric supports two evaluation modes: - Strict order (default): Tool calls must match exactly in sequence - Flexible order: Tool calls can be in any order (parallel evaluation)

Score calculation: - The shorter call list must be contained in the longer one: as an in-order subsequence (strict) or as a multiset of tool names (flexible). If not, some call is wrong and the score is 0. - Otherwise: score = sum of argument accuracy over matched calls, divided by the longer list's length. Equal lengths give the average argument accuracy; each missing or extra call lowers the score proportionally.

Usage: >>> from ragas.metrics.collections import ToolCallAccuracy >>> from ragas.messages import HumanMessage, AIMessage, ToolCall >>> >>> metric = ToolCallAccuracy(strict_order=True) >>> >>> result = await metric.ascore( ... user_input=[ ... HumanMessage(content="What's the weather in Paris?"), ... AIMessage( ... content="Let me check", ... tool_calls=[ToolCall(name="get_weather", args={"location": "Paris"})] ... ) ... ], ... reference_tool_calls=[ ... ToolCall(name="get_weather", args={"location": "Paris"}) ... ] ... ) >>> print(f"Tool Call Accuracy: {result.value}")

Attributes: strict_order: If True (default), tool calls must match exactly in sequence. If False, tool calls can be in any order. name: The metric name allowed_values: Score range (0.0 to 1.0, higher is better)

Args: strict_order: If True, tool calls must match exactly in sequence. If False, tool calls can be in any order (default: True) name: The metric name (default: "tool_call_accuracy") **kwargs: Additional arguments passed to BaseMetric

Source code in src/ragas/metrics/collections/tool_call_accuracy/metric.py
def __init__(
    self,
    strict_order: bool = True,
    name: str = "tool_call_accuracy",
    **kwargs,
):
    """
    Initialize ToolCallAccuracy metric.

    Args:
        strict_order: If True, tool calls must match exactly in sequence.
                     If False, tool calls can be in any order (default: True)
        name: The metric name (default: "tool_call_accuracy")
        **kwargs: Additional arguments passed to BaseMetric
    """
    self.strict_order = strict_order
    super().__init__(name=name, **kwargs)

ascore async

ascore(
    user_input: List[
        Union[HumanMessage, AIMessage, ToolMessage]
    ],
    reference_tool_calls: List[ToolCall],
) -> MetricResult

Calculate tool call accuracy score asynchronously.

Args: user_input: List of conversation messages (HumanMessage, AIMessage, ToolMessage) reference_tool_calls: List of expected tool calls

Returns: MetricResult with accuracy score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/tool_call_accuracy/metric.py
async def ascore(
    self,
    user_input: List[t.Union["HumanMessage", "AIMessage", "ToolMessage"]],
    reference_tool_calls: List[ToolCall],
) -> MetricResult:
    """
    Calculate tool call accuracy score asynchronously.

    Args:
        user_input: List of conversation messages (HumanMessage, AIMessage, ToolMessage)
        reference_tool_calls: List of expected tool calls

    Returns:
        MetricResult with accuracy score (0.0-1.0, higher is better)
    """
    # Input validation
    if not isinstance(user_input, list):
        raise ValueError("user_input must be a list of messages")
    if not isinstance(reference_tool_calls, list):
        raise ValueError("reference_tool_calls must be a list")

    # Extract predicted tool calls from AI messages
    pred_tool_calls = []
    for item in user_input:
        if isinstance(item, AIMessage) and item.tool_calls is not None:
            pred_tool_calls.extend(item.tool_calls)

    # Handle edge cases
    if not pred_tool_calls and not reference_tool_calls:
        return MetricResult(value=1.0)
    elif not pred_tool_calls:
        warnings.warn("No tool calls found in the user input")
        return MetricResult(value=0.0)
    elif not reference_tool_calls:
        warnings.warn("Reference tool calls are empty but predictions exist")
        return MetricResult(value=0.0)

    # Pair up calls. The shorter list must be contained in the longer one:
    # as an in-order subsequence (strict_order) or as a multiset of names
    # (flexible). A call that isn't in the other list at all is a wrong
    # call and scores 0, as before. Missing or extra calls are a coverage
    # gap: the matched calls' argument accuracy is divided by the longer
    # list's length, so each missing or extra call costs the same (#62).
    # Equal-length inputs score exactly as before.
    if len(pred_tool_calls) != len(reference_tool_calls):
        warnings.warn(
            f"Length mismatch: predicted tool calls ({len(pred_tool_calls)}) "
            f"vs reference tool calls ({len(reference_tool_calls)}). "
            f"Unmatched calls count against the score."
        )

    pairs = self._match_calls(pred_tool_calls, reference_tool_calls)
    if pairs is None:
        return MetricResult(value=0.0)

    total = sum(exact_match_args(pred.args, ref.args) for pred, ref in pairs)
    final_score = total / max(len(pred_tool_calls), len(reference_tool_calls))
    return MetricResult(value=float(final_score))

ToolCallF1

ToolCallF1(name: str = 'tool_call_f1', **kwargs)

Bases: BaseMetric

Modern implementation of Tool Call F1 metric.

Measures the F1 score between predicted and reference tool calls. This metric treats tool calls as a set, comparing the exact match of tool names and their arguments using set-based precision and recall.

The F1 score is calculated as: - Precision = TP / (TP + FP) where TP = true positives, FP = false positives - Recall = TP / (TP + FN) where FN = false negatives - F1 = 2 * (Precision * Recall) / (Precision + Recall)

A tool call is considered a match only if both the tool name and all arguments match exactly between predicted and reference.

Usage: >>> from ragas.metrics.collections import ToolCallF1 >>> from ragas.messages import HumanMessage, AIMessage, ToolCall >>> >>> metric = ToolCallF1() >>> >>> result = await metric.ascore( ... user_input=[ ... HumanMessage(content="What's the weather in Paris?"), ... AIMessage( ... content="Let me check", ... tool_calls=[ ... ToolCall(name="get_weather", args={"location": "Paris"}), ... ToolCall(name="get_uv_index", args={"location": "Paris"}) ... ] ... ) ... ], ... reference_tool_calls=[ ... ToolCall(name="get_weather", args={"location": "Paris"}) ... ] ... ) >>> print(f"Tool Call F1: {result.value}") # 0.67 (1 TP, 1 FP, 0 FN)

Attributes: name: The metric name allowed_values: Score range (0.0 to 1.0, higher is better)

Args: name: The metric name (default: "tool_call_f1") **kwargs: Additional arguments passed to BaseMetric

Source code in src/ragas/metrics/collections/tool_call_f1/metric.py
def __init__(self, name: str = "tool_call_f1", **kwargs):
    """
    Initialize ToolCallF1 metric.

    Args:
        name: The metric name (default: "tool_call_f1")
        **kwargs: Additional arguments passed to BaseMetric
    """
    super().__init__(name=name, **kwargs)

ascore async

ascore(
    user_input: List[
        Union[HumanMessage, AIMessage, ToolMessage]
    ],
    reference_tool_calls: List[ToolCall],
) -> MetricResult

Calculate tool call F1 score asynchronously.

Args: user_input: List of conversation messages (HumanMessage, AIMessage, ToolMessage) reference_tool_calls: List of expected tool calls

Returns: MetricResult with F1 score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/tool_call_f1/metric.py
async def ascore(
    self,
    user_input: t.List[t.Union["HumanMessage", "AIMessage", "ToolMessage"]],
    reference_tool_calls: t.List["ToolCall"],
) -> MetricResult:
    """
    Calculate tool call F1 score asynchronously.

    Args:
        user_input: List of conversation messages (HumanMessage, AIMessage, ToolMessage)
        reference_tool_calls: List of expected tool calls

    Returns:
        MetricResult with F1 score (0.0-1.0, higher is better)
    """
    # Input validation
    if not isinstance(user_input, list):
        raise ValueError("user_input must be a list of messages")
    if not isinstance(reference_tool_calls, list):
        raise ValueError("reference_tool_calls must be a list")

    # Convert reference tool calls to set
    expected: t.Set[t.Tuple[str, t.FrozenSet]] = set()
    for call in reference_tool_calls:
        expected.add(tool_call_to_hashable(call))

    # Extract and convert predicted tool calls to set
    actual: t.Set[t.Tuple[str, t.FrozenSet]] = set()
    for msg in user_input:
        if isinstance(msg, AIMessage) and msg.tool_calls is not None:
            for call in msg.tool_calls:
                actual.add(tool_call_to_hashable(call))

    # Calculate set-based metrics
    true_positives = len(actual & expected)
    false_positives = len(actual - expected)
    false_negatives = len(expected - actual)

    # No expected and no actual calls is a perfect 1.0 (metrics.utils policy).
    f1_score = fbeta_score(true_positives, false_positives, false_negatives)

    return MetricResult(value=round(f1_score, 4))

TopicAdherence

TopicAdherence(
    llm: InstructorBaseRagasLLM,
    mode: Literal["precision", "recall", "f1"] = "f1",
    name: str = "topic_adherence",
    **kwargs,
)

Bases: BaseMetric

Measures how well an AI system adheres to predefined topics during conversations.

AI systems deployed in real-world applications are expected to stay within domains of interest. This metric evaluates the ability of the AI to only answer queries related to predefined topics and refuse queries outside those topics.

The metric works by: 1. Extracting topics discussed in the conversation 2. Checking which topics the AI answered vs refused 3. Classifying if each topic falls within the reference topics 4. Computing precision, recall, or F1 based on these classifications

Score interpretation: - Precision: Ratio of answered topics that are within reference topics - Recall: Ratio of reference-aligned topics that were answered (not refused) - F1: Harmonic mean of precision and recall

Usage: >>> from openai import AsyncOpenAI >>> from ragas.llms.base import llm_factory >>> from ragas.metrics.collections import TopicAdherence >>> from ragas.messages import HumanMessage, AIMessage >>> >>> client = AsyncOpenAI() >>> llm = llm_factory("", client=client) >>> >>> metric = TopicAdherence(llm=llm, mode="precision") >>> >>> result = await metric.ascore( ... user_input=[ ... HumanMessage(content="Tell me about quantum physics"), ... AIMessage(content="Quantum physics is a branch of physics..."), ... ], ... reference_topics=["Physics", "Science"], ... ) >>> print(f"Topic Adherence: {result.value}")

Attributes: llm: Modern instructor-based LLM for topic extraction and classification mode: Evaluation mode - "precision", "recall", or "f1" (default: "f1") name: The metric name

Source code in src/ragas/metrics/collections/topic_adherence/metric.py
def __init__(
    self,
    llm: "InstructorBaseRagasLLM",
    mode: Literal["precision", "recall", "f1"] = "f1",
    name: str = "topic_adherence",
    **kwargs,
):
    self.mode = mode
    self.topic_extraction_prompt = TopicExtractionPrompt()
    self.topic_refused_prompt = TopicRefusedPrompt()
    self.topic_classification_prompt = TopicClassificationPrompt()

    super().__init__(llm=llm, name=name, **kwargs)

ascore async

ascore(
    user_input: List[
        Union[HumanMessage, AIMessage, ToolMessage]
    ],
    reference_topics: List[str],
) -> MetricResult

Calculate topic adherence score.

Args: user_input: List of conversation messages reference_topics: List of allowed topics the AI should adhere to

Returns: MetricResult with topic adherence score (0.0-1.0, higher is better)

Source code in src/ragas/metrics/collections/topic_adherence/metric.py
async def ascore(
    self,
    user_input: List[Union[HumanMessage, AIMessage, ToolMessage]],
    reference_topics: List[str],
) -> MetricResult:
    """
    Calculate topic adherence score.

    Args:
        user_input: List of conversation messages
        reference_topics: List of allowed topics the AI should adhere to

    Returns:
        MetricResult with topic adherence score (0.0-1.0, higher is better)
    """
    if not isinstance(user_input, list):
        raise ValueError("user_input must be a list of messages")
    if not isinstance(reference_topics, list) or not reference_topics:
        raise ValueError("reference_topics must be a non-empty list of topics")

    # Format conversation as pretty string
    conversation = format_conversation(user_input)

    # Step 1: Extract topics from the conversation
    topics = await self._extract_topics(conversation)
    if not topics:
        return MetricResult(value=float("nan"))

    # Step 2: Check which topics the AI answered vs refused
    topic_answered = await self._check_topics_answered(conversation, topics)

    # Step 3: Classify topics against reference topics
    topic_classifications = await self._classify_topics(reference_topics, topics)
    if topic_classifications is None:
        return MetricResult(value=float("nan"))

    # Step 4: Compute score based on mode
    score = self._compute_score(topic_answered, topic_classifications)

    lines = [
        f"{len(topics)} topics discussed; {self.mode} of answering in-scope "
        f"topics (reference topics: {', '.join(reference_topics)})."
    ]
    for topic, answered, in_scope in zip(
        topics, topic_answered, topic_classifications
    ):
        lines.append(
            f"- {topic}: {'answered' if answered else 'refused'}, "
            f"{'in scope' if in_scope else 'out of scope'}"
        )
    return MetricResult(value=float(score), reason="\n".join(lines))