Skip to content

Integrations

ragas.integrations.llama_index

aevaluate_query_engine async

aevaluate_query_engine(
    query_engine: Any,
    dataset: DataTable,
    metrics: Sequence["SimpleBaseMetric"],
    *,
    llm: Optional["InstructorBaseRagasLLM"] = None,
    embeddings: Optional["BaseRagasEmbedding"] = None,
    run_config: Optional[RunConfig] = None,
    batch_size: Optional[int] = None,
    usage: Optional["TokenUsageCollector"] = None,
    raise_exceptions: bool = False,
    column_map: Optional[Mapping[str, str]] = None,
    show_progress: bool = True,
    name: Optional[str] = None,
    backend: Optional[Union["BaseBackend", str]] = None,
    backend_kwargs: Optional[Dict[str, Any]] = None,
) -> "Experiment"

Run a LlamaIndex query engine over dataset and score the answers.

Each row's user_input goes to query_engine.aquery(); the answer and its source nodes become the row's response and retrieved_contexts. The filled-in rows are then scored with :func:ragas.metrics_experiment, and its Experiment is returned: use ragas.summarize() / ragas.check() on it.

Args: query_engine: A LlamaIndex query engine (anything with aquery). dataset: A Dataset of single-turn rows (SingleTurnSample objects or dicts). It isn't modified. metrics: ragas.metrics.collections metrics (or any SimpleBaseMetric). llm, embeddings: For metrics that take them per call; there is no default. LlamaIndex LLMs aren't wrapped (#215): reach the same provider with llm_factory(). run_config: Timeouts and retries for the queries and the metrics. batch_size: How many queries run at once (default: all). usage: A TokenUsageCollector for the metrics' token usage. column_map: {sample_field: dataset_column}, for a dataset whose columns are named differently, e.g. {"user_input": "question"}. raise_exceptions: Stop at the first failed query or metric. Otherwise a failed query leaves the row's response empty (None) and the row scores NaN for every metric, with query engine failed: ... as each reason, so summarize() counts it as missing and check() fails on it unless allow_missing permits it. A failed metric scores NaN on its own. Both are logged. show_progress: Show the query progress bar. name, backend, backend_kwargs: The experiment's name, and where to save it (default: in memory).

Raises: NotImplementedError: multi-turn rows.

Source code in src/ragas/integrations/llama_index.py
async def aevaluate_query_engine(
    query_engine: t.Any,
    dataset: DataTable,
    metrics: t.Sequence["SimpleBaseMetric"],
    *,
    llm: t.Optional["InstructorBaseRagasLLM"] = None,
    embeddings: t.Optional["BaseRagasEmbedding"] = None,
    run_config: t.Optional[RunConfig] = None,
    batch_size: t.Optional[int] = None,
    usage: t.Optional["TokenUsageCollector"] = None,
    raise_exceptions: bool = False,
    column_map: t.Optional[t.Mapping[str, str]] = None,
    show_progress: bool = True,
    name: t.Optional[str] = None,
    backend: t.Optional[t.Union["BaseBackend", str]] = None,
    backend_kwargs: t.Optional[t.Dict[str, t.Any]] = None,
) -> "Experiment":
    """Run a LlamaIndex query engine over ``dataset`` and score the answers.

    Each row's ``user_input`` goes to ``query_engine.aquery()``; the answer
    and its source nodes become the row's ``response`` and
    ``retrieved_contexts``. The filled-in rows are then scored with
    :func:`ragas.metrics_experiment`, and its ``Experiment`` is returned:
    use ``ragas.summarize()`` / ``ragas.check()`` on it.

    Args:
        query_engine: A LlamaIndex query engine (anything with ``aquery``).
        dataset: A ``Dataset`` of single-turn rows (``SingleTurnSample``
            objects or dicts). It isn't modified.
        metrics: ``ragas.metrics.collections`` metrics (or any
            ``SimpleBaseMetric``).
        llm, embeddings: For metrics that take them per call; there is no
            default. LlamaIndex LLMs aren't wrapped (#215): reach the same
            provider with ``llm_factory()``.
        run_config: Timeouts and retries for the queries and the metrics.
        batch_size: How many queries run at once (default: all).
        usage: A ``TokenUsageCollector`` for the metrics' token usage.
        column_map: ``{sample_field: dataset_column}``, for a dataset whose
            columns are named differently, e.g. ``{"user_input": "question"}``.
        raise_exceptions: Stop at the first failed query or metric. Otherwise
            a failed query leaves the row's response empty (None) and the row
            scores NaN for every metric, with ``query engine failed: ...`` as
            each reason, so ``summarize()`` counts it as missing and
            ``check()`` fails on it unless ``allow_missing`` permits it. A
            failed metric scores NaN on its own. Both are logged.
        show_progress: Show the query progress bar.
        name, backend, backend_kwargs: The experiment's name, and where to
            save it (default: in memory).

    Raises:
        NotImplementedError: multi-turn rows.
    """
    if not isinstance(dataset, DataTable):
        raise TypeError(
            "dataset must be a ragas Dataset of single-turn rows, e.g. "
            "Dataset('questions', 'inmemory', data_model=SingleTurnSample, "
            f"data=rows); got {type(dataset).__name__}."
        )
    samples = []
    for i, row in enumerate(dataset):
        if isinstance(row, SingleTurnSample):
            samples.append(row.model_copy(deep=True))
            continue
        fields = row.model_dump() if isinstance(row, BaseModel) else dict(row)
        for sample_field, column in (column_map or {}).items():
            if column in fields:
                fields[sample_field] = fields.pop(column)
        if isinstance(fields.get("user_input"), list):
            raise NotImplementedError(
                "Multi-turn evaluation is not implemented yet. Please do raise "
                "an issue on GitHub if you need this feature and we will "
                "prioritize it"
            )
        try:
            samples.append(SingleTurnSample(**fields))
        except Exception as e:
            raise ValueError(f"Row {i} is not a single-turn sample: {e}") from e

    exec = Executor(
        desc="Running Query Engine",
        keep_progress_bar=True,
        show_progress=show_progress,
        raise_exceptions=raise_exceptions,
        run_config=run_config,
        batch_size=batch_size,
    )

    async def query(q: t.Optional[str]) -> t.Any:
        try:
            return await query_engine.aquery(q)
        except Exception as e:
            if raise_exceptions:
                raise
            return e

    queries = [sample.user_input for sample in samples]
    for i, q in enumerate(queries):
        exec.submit(query, q, name=f"query-{i}")

    failures: t.Dict[int, str] = {}
    for i, (sample, r) in enumerate(zip(samples, await exec.aresults())):
        if isinstance(r, Exception) or (isinstance(r, float) and math.isnan(r)):
            reason = (
                f"query engine failed: {type(r).__name__}: {r}"
                if isinstance(r, Exception)
                else "query engine failed"
            )
            logger.warning(
                "Query %d (%r) failed; the row scores missing for every metric. %s",
                i,
                queries[i],
                reason,
            )
            sample.response = None
            sample.retrieved_contexts = None
            failures[id(sample)] = reason
            continue
        response: LlamaIndexResponse = t.cast("LlamaIndexResponse", r)
        sample.response = response.response if response.response is not None else ""
        sample.retrieved_contexts = [n.get_text() for n in response.source_nodes]

    answered = Dataset(
        f"{dataset.name}_answers",
        "inmemory",
        data_model=SingleTurnSample,
        data=samples,
    )
    run = _QueryEngineScoring(
        metrics,
        failures=failures,
        llm=llm,
        embeddings=embeddings,
        run_config=run_config,
        usage=usage,
        backend=backend or "inmemory",
        backend_kwargs=backend_kwargs,
        raise_exceptions=raise_exceptions,
    )
    return await run.arun(answered, name=name)

evaluate_query_engine

evaluate_query_engine(
    query_engine: Any,
    dataset: DataTable,
    metrics: Sequence["SimpleBaseMetric"],
    **kwargs: Any,
) -> "Experiment"

Sync :func:aevaluate_query_engine; takes the same arguments.

Source code in src/ragas/integrations/llama_index.py
def evaluate_query_engine(
    query_engine: t.Any,
    dataset: DataTable,
    metrics: t.Sequence["SimpleBaseMetric"],
    **kwargs: t.Any,
) -> "Experiment":
    """Sync :func:`aevaluate_query_engine`; takes the same arguments."""
    return run_sync(aevaluate_query_engine(query_engine, dataset, metrics, **kwargs))

convert_to_ragas_messages

convert_to_ragas_messages(
    events: List[Event],
) -> List[Message]

Convert a sequence of LlamIndex agent events into Ragas message objects.

This function processes a list of Event objects (e.g., AgentInput, AgentOutput, and ToolCallResult) and converts them into a list of Message objects (HumanMessage, AIMessage, and ToolMessage) that can be used for evaluation with the Ragas framework.

Parameters:

Name Type Description Default
events List[Event]

A list of agent events that represent a conversation trace. These can include user inputs (AgentInput), model outputs (AgentOutput), and tool responses (ToolCallResult).

required

Returns:

Type Description
List[Message]

A list of Ragas Message objects corresponding to the structured conversation. Tool calls are de-duplicated using their tool ID to avoid repeated entries.

Source code in src/ragas/integrations/llama_index.py
def convert_to_ragas_messages(events: t.List[Event]) -> t.List[Message]:
    """
    Convert a sequence of LlamIndex agent events into Ragas message objects.

    This function processes a list of `Event` objects (e.g., `AgentInput`, `AgentOutput`,
    and `ToolCallResult`) and converts them into a list of `Message` objects (`HumanMessage`,
    `AIMessage`, and `ToolMessage`) that can be used for evaluation with the Ragas framework.

    Parameters
    ----------
    events : List[Event]
        A list of agent events that represent a conversation trace. These can include
        user inputs (`AgentInput`), model outputs (`AgentOutput`), and tool responses
        (`ToolCallResult`).

    Returns
    -------
    List[Message]
        A list of Ragas `Message` objects corresponding to the structured conversation.
        Tool calls are de-duplicated using their tool ID to avoid repeated entries.
    """
    try:
        from llama_index.core.agent.workflow import (
            AgentInput,
            AgentOutput,
            ToolCallResult,
        )
        from llama_index.core.base.llms.types import MessageRole, TextBlock
    except ImportError:
        raise ImportError(
            "Please install the llama_index package to use this function."
        )
    ragas_messages = []
    tool_call_ids = set()

    for event in events:
        if isinstance(event, AgentInput):
            last_chat_message = event.input[-1]

            content = ""
            if last_chat_message.blocks:
                content = "\n".join(
                    str(block.text)
                    for block in last_chat_message.blocks
                    if isinstance(block, TextBlock)
                )

            if last_chat_message.role == MessageRole.USER:
                if ragas_messages and isinstance(ragas_messages[-1], ToolMessage):
                    continue
                ragas_messages.append(HumanMessage(content=content))

        elif isinstance(event, AgentOutput):
            content = "\n".join(
                str(block.text)
                for block in event.response.blocks
                if isinstance(block, TextBlock)
            )
            ragas_tool_calls = None

            if hasattr(event, "tool_calls"):
                raw_tool_calls = event.tool_calls
                ragas_tool_calls = []
                for tc in raw_tool_calls:
                    if tc.tool_id not in tool_call_ids:
                        tool_call_ids.add(tc.tool_id)
                        ragas_tool_calls.append(
                            ToolCall(
                                name=tc.tool_name,
                                args=tc.tool_kwargs,
                            )
                        )
            ragas_messages.append(
                AIMessage(
                    content=content,
                    tool_calls=ragas_tool_calls if ragas_tool_calls else None,
                )
            )
        elif isinstance(event, ToolCallResult):
            if event.return_direct:
                ragas_messages.append(AIMessage(content=event.tool_output.content))
            else:
                ragas_messages.append(ToolMessage(content=event.tool_output.content))

    return ragas_messages

ragas.integrations.helicone

ragas.integrations.amazon_bedrock

get_last_orchestration_value

get_last_orchestration_value(
    traces: List[Dict[str, Any]], key: str
)

Iterates through the traces to find the last occurrence of a specified key within the orchestrationTrace.

Returns: (index, value): Tuple where index is the last index at which the key was found, and value is the corresponding value, or (None, None) if not found.

Source code in src/ragas/integrations/amazon_bedrock.py
def get_last_orchestration_value(traces: t.List[t.Dict[str, t.Any]], key: str):
    """
    Iterates through the traces to find the last occurrence of a specified key
    within the orchestrationTrace.

    Returns:
        (index, value): Tuple where index is the last index at which the key was found, and value is the corresponding value, or (None, None) if not found.
    """
    last_index = -1
    last_value = None
    for i, trace in enumerate(traces):
        orchestration = trace.get("trace", {}).get("orchestrationTrace", {})
        if key in orchestration:
            last_index = i
            last_value = orchestration[key]
    return last_index, last_value

extract_messages_from_model_invocation

extract_messages_from_model_invocation(model_inv)

Extracts messages from the 'text' field of the modelInvocationInput. Ensures that each message's content is cast to a string.

Returns: List of messages as HumanMessage or AIMessage objects.

Source code in src/ragas/integrations/amazon_bedrock.py
def extract_messages_from_model_invocation(model_inv):
    """
    Extracts messages from the 'text' field of the modelInvocationInput.
    Ensures that each message's content is cast to a string.

    Returns:
        List of messages as HumanMessage or AIMessage objects.
    """
    messages = []
    text_json = json.loads(model_inv.get("text", "{}"))
    for msg in text_json.get("messages", []):
        content_str = str(msg.get("content", ""))
        role = msg.get("role")
        if role == "user":
            messages.append(HumanMessage(content=content_str))
        elif role == "assistant":
            messages.append(AIMessage(content=content_str))
    return messages[:-1]

convert_to_ragas_messages

convert_to_ragas_messages(traces: List)

Converts a list of trace dictionaries into a list of messages. It extracts messages from the last modelInvocationInput and appends the finalResponse from the observation (if it occurs after the model invocation).

Returns: List of HumanMessage and AIMessage objects.

Source code in src/ragas/integrations/amazon_bedrock.py
def convert_to_ragas_messages(traces: t.List):
    """
    Converts a list of trace dictionaries into a list of messages.
    It extracts messages from the last modelInvocationInput and appends
    the finalResponse from the observation (if it occurs after the model invocation).

    Returns:
        List of HumanMessage and AIMessage objects.
    """
    result = []

    # Get the last modelInvocationInput from the traces.
    last_model_inv_index, last_model_inv = get_last_orchestration_value(
        traces, "modelInvocationInput"
    )
    if last_model_inv is not None:
        result.extend(extract_messages_from_model_invocation(last_model_inv))

    # Get the last observation from the traces.
    last_obs_index, last_observation = get_last_orchestration_value(
        traces, "observation"
    )
    if last_observation is not None and last_obs_index > last_model_inv_index:
        final_text = str(last_observation.get("finalResponse", {}).get("text", ""))
        result.append(AIMessage(content=final_text))

    return result

extract_kb_trace

extract_kb_trace(traces)

Extracts groups of traces that follow the specific order: 1. An element with 'trace' -> 'orchestrationTrace' containing an 'invocationInput' with invocationType == "KNOWLEDGE_BASE" 2. Followed (later in the list or within the same trace) by an element with an 'observation' that contains 'knowledgeBaseLookupOutput' 3. Followed by an element with an 'observation' that contains 'finalResponse'

Returns a list of dictionaries each with keys: 'user_input', 'retrieved_contexts', and 'response'

This version supports multiple knowledge base invocation groups.

Source code in src/ragas/integrations/amazon_bedrock.py
def extract_kb_trace(traces):
    """
    Extracts groups of traces that follow the specific order:
      1. An element with 'trace' -> 'orchestrationTrace' containing an 'invocationInput'
         with invocationType == "KNOWLEDGE_BASE"
      2. Followed (later in the list or within the same trace) by an element with an 'observation'
         that contains 'knowledgeBaseLookupOutput'
      3. Followed by an element with an 'observation' that contains 'finalResponse'

    Returns a list of dictionaries each with keys:
      'user_input', 'retrieved_contexts', and 'response'

    This version supports multiple knowledge base invocation groups.
    """
    results = []
    groups_in_progress = []  # list to keep track of groups in progress

    for trace in traces:
        orchestration = trace.get("trace", {}).get("orchestrationTrace", {})

        # 1. Look for a KB invocation input.
        inv_input = orchestration.get("invocationInput")
        if inv_input and inv_input.get("invocationType") == "KNOWLEDGE_BASE":
            kb_input = inv_input.get("knowledgeBaseLookupInput", {})
            # Start a new group with the user's input text.
            groups_in_progress.append({"user_input": kb_input.get("text")})

        # 2. Process observations.
        obs = orchestration.get("observation", {})
        if obs:
            # If the observation contains a KB output, assign it to the earliest group
            # that does not yet have a 'retrieved_contexts' key.
            if "knowledgeBaseLookupOutput" in obs:
                for group in groups_in_progress:
                    if "user_input" in group and "retrieved_contexts" not in group:
                        kb_output = obs["knowledgeBaseLookupOutput"]
                        group["retrieved_contexts"] = [
                            retrieved.get("content", {}).get("text")
                            for retrieved in kb_output.get("retrievedReferences", [])
                        ]
                        break

            # 3. When we see a final response, assign it to all groups that have already
            # received their KB output but still lack a response.
            if "finalResponse" in obs:
                final_text = obs["finalResponse"].get("text")
                completed_groups = []
                for group in groups_in_progress:
                    if (
                        "user_input" in group
                        and "retrieved_contexts" in group
                        and "response" not in group
                    ):
                        group["response"] = final_text
                        completed_groups.append(group)
                # Remove completed groups from the in-progress list and add to the final results.
                groups_in_progress = [
                    g for g in groups_in_progress if g not in completed_groups
                ]
                results.extend(completed_groups)

    return results

ragas.integrations.swarm

convert_to_ragas_messages

convert_to_ragas_messages(
    messages: List[Dict[str, Any]],
) -> List[Union[HumanMessage, AIMessage, ToolMessage]]

Convert Swarm messages to Ragas message format.

Parameters:

Name Type Description Default
messages List[Union[Response, Dict]]

List of messages to convert, where each message can be either: - Response: A Swarm Response object containing messages - Dict: A dictionary containing a user message

required

Returns:

Type Description
List[Union[HumanMessage, AIMessage, ToolMessage]]

List of converted Ragas format messages where: - HumanMessage: For user messages - AIMessage: For assistant messages with optional tool calls - ToolMessage: For tool response messages

The messages are OpenAI chat-format dicts, so any OpenAI-format history
works, not only Swarm's. ``system`` and ``developer`` messages are
skipped. Content may be a string, ``None`` or a list of content parts
(text parts are kept).

Raises:

Type Description
KeyError

If a message is missing the required 'role' key

ValueError

If a message has a role other than user, assistant, tool, system or developer

Source code in src/ragas/integrations/swarm.py
def convert_to_ragas_messages(
    messages: List[Dict[str, Any]],
) -> List[Union[HumanMessage, AIMessage, ToolMessage]]:
    """
    Convert Swarm messages to Ragas message format.

    Parameters
    ----------
    messages : List[Union[Response, Dict]]
        List of messages to convert, where each message can be either:
        - Response: A Swarm Response object containing messages
        - Dict: A dictionary containing a user message

    Returns
    -------
    List[Union[HumanMessage, AIMessage, ToolMessage]]
        List of converted Ragas format messages where:
        - HumanMessage: For user messages
        - AIMessage: For assistant messages with optional tool calls
        - ToolMessage: For tool response messages

    The messages are OpenAI chat-format dicts, so any OpenAI-format history
    works, not only Swarm's. ``system`` and ``developer`` messages are
    skipped. Content may be a string, ``None`` or a list of content parts
    (text parts are kept).

    Raises
    ------
    KeyError
        If a message is missing the required 'role' key
    ValueError
        If a message has a role other than user, assistant, tool, system or
        developer
    """

    def convert_tool_calls(tool_calls_data: List[Dict[str, Any]]) -> List[ToolCall]:
        """Convert tool calls data to Ragas ToolCall objects"""
        return [
            ToolCall(
                name=tool_call["function"]["name"],
                # arguments that aren't a JSON object are kept as raw_args
                # instead of failing the conversion
                args=parse_tool_arguments(tool_call["function"].get("arguments")),
            )
            for tool_call in tool_calls_data
        ]

    def handle_assistant_message(message: Dict[str, Any]) -> AIMessage:
        """Convert assistant message to Ragas AIMessage"""
        # a plain reply has no tool_calls key, or None (#98)
        tool_calls = convert_tool_calls(message.get("tool_calls") or [])
        return AIMessage(
            content=text_from_content(message.get("content")),
            tool_calls=tool_calls,
        )

    def handle_tool_message(message: Dict[str, Any]) -> ToolMessage:
        """Convert tool message to Ragas ToolMessage"""
        return ToolMessage(content=text_from_content(message.get("content")))

    def handle_user_message(message: Dict[str, Any]) -> HumanMessage:
        """Convert user message to Ragas HumanMessage"""
        return HumanMessage(content=text_from_content(message.get("content")))

    converted_messages = []

    for message in messages:
        role = message.get("role")
        if role is None:
            raise KeyError("'role' key not present in message")

        if role == "assistant":
            converted_messages.append(handle_assistant_message(message))
        elif role == "tool":
            converted_messages.append(handle_tool_message(message))
        elif role == "user":
            converted_messages.append(handle_user_message(message))
        elif role in ("system", "developer"):
            # part of a full OpenAI-format history, but ragas multi-turn
            # samples carry no system messages
            logger.debug("Skipping %s message", role)
        else:
            raise ValueError(
                f"Role must be one of ['assistant', 'user', 'tool'], but found '{role}'"
            )

    return converted_messages