Skip to content

Schemas

Samples, messages, datasets and experiments. To score a Dataset of samples, see Scoring a dataset.

BaseSample

Bases: BaseModel

Base class for evaluation samples.

to_dict

to_dict() -> Dict

Get the dictionary representation of the sample without attributes that are None.

Source code in src/ragas/dataset_schema.py
def to_dict(self) -> t.Dict:
    """
    Get the dictionary representation of the sample without attributes that are None.
    """
    return self.model_dump(exclude_none=True)

get_features

get_features() -> List[str]

Get the features of the sample that are not None.

Source code in src/ragas/dataset_schema.py
def get_features(self) -> t.List[str]:
    """
    Get the features of the sample that are not None.
    """
    return list(self.to_dict().keys())

to_string

to_string() -> str

Get the string representation of the sample.

Source code in src/ragas/dataset_schema.py
def to_string(self) -> str:
    """
    Get the string representation of the sample.
    """
    sample_dict = self.to_dict()
    return "".join(f"\n{key}:\n\t{val}\n" for key, val in sample_dict.items())

SingleTurnSample

Bases: BaseSample

Represents evaluation samples for single-turn interactions.

Attributes:

Name Type Description
user_input Optional[str]

The input query from the user.

retrieved_contexts Optional[List[str]]

List of contexts retrieved for the query.

reference_contexts Optional[List[str]]

List of reference contexts for the query.

retrieved_context_ids Optional[List[Union[str, int]]]

List of IDs for retrieved contexts.

reference_context_ids Optional[List[Union[str, int]]]

List of IDs for reference contexts.

response Optional[str]

The generated response for the query.

multi_responses Optional[List[str]]

List of multiple responses generated for the query.

reference Optional[str]

The reference answer for the query.

rubric Optional[Dict[str, str]]

Evaluation rubric for the sample.

persona_name Optional[str]

Name of the persona used in query generation.

query_style Optional[str]

Style of the generated query (e.g., formal, casual).

query_length Optional[str]

Length category of the query (e.g., short, medium, long).

reference_contexts_metadata Optional[List[Dict[str, Any]]]

Metadata of the source document for each entry in reference_contexts, in the same order, e.g. {"source": "a.pdf"}. Set by testset generation. Values are stored JSON-safe: dates become ISO strings, and other non-JSON values become strings.

MultiTurnSample

Bases: BaseSample

Represents evaluation samples for multi-turn interactions.

Attributes:

Name Type Description
user_input List[Union[HumanMessage, AIMessage, ToolMessage]]

A list of messages representing the conversation turns.

reference (Optional[str], optional)

The reference answer or expected outcome for the conversation.

reference_tool_calls (Optional[List[ToolCall]], optional)

A list of expected tool calls for the conversation.

rubrics (Optional[Dict[str, str]], optional)

Evaluation rubrics for the conversation.

reference_topics (Optional[List[str]], optional)

A list of reference topics for the conversation.

validate_user_input classmethod

validate_user_input(
    messages: List[
        Union[HumanMessage, AIMessage, ToolMessage]
    ],
) -> List[Union[HumanMessage, AIMessage, ToolMessage]]

Validates the user input messages.

Source code in src/ragas/dataset_schema.py
@field_validator("user_input")
@classmethod
def validate_user_input(
    cls,
    messages: t.List[t.Union[HumanMessage, AIMessage, ToolMessage]],
) -> t.List[t.Union[HumanMessage, AIMessage, ToolMessage]]:
    """Validates the user input messages."""
    if not all(
        isinstance(m, (HumanMessage, AIMessage, ToolMessage)) for m in messages
    ):
        raise ValueError(
            "All inputs must be instances of HumanMessage, AIMessage, or ToolMessage."
        )

    has_seen_ai_message = False

    for i, m in enumerate(messages):
        if isinstance(m, AIMessage):
            has_seen_ai_message = True

        elif isinstance(m, ToolMessage):
            # Rule 1: ToolMessage must be preceded by an AIMessage somewhere in the conversation
            if not has_seen_ai_message:
                raise ValueError(
                    "ToolMessage must be preceded by an AIMessage somewhere in the conversation."
                )

            # Rule 2: ToolMessage must follow an AIMessage or another ToolMessage
            if i > 0:
                prev_message = messages[i - 1]

                if isinstance(prev_message, AIMessage):
                    # Rule 3: If following AIMessage, that message must have tool_calls
                    if not prev_message.tool_calls:
                        raise ValueError(
                            "ToolMessage must follow an AIMessage where tools were called."
                        )
                elif not isinstance(prev_message, ToolMessage):
                    # Not following AIMessage or ToolMessage
                    raise ValueError(
                        "ToolMessage must follow an AIMessage or another ToolMessage."
                    )

    return messages

to_messages

to_messages()

Converts the user input messages to a list of dictionaries.

Source code in src/ragas/dataset_schema.py
def to_messages(self):
    """Converts the user input messages to a list of dictionaries."""
    return [m.model_dump() for m in self.user_input]

pretty_repr

pretty_repr()

Returns a pretty string representation of the conversation.

Source code in src/ragas/dataset_schema.py
def pretty_repr(self):
    """Returns a pretty string representation of the conversation."""
    lines = []
    for m in self.user_input:
        lines.append(m.pretty_repr())

    return "\n".join(lines)

sample_from_dict

sample_from_dict(
    row: Mapping[str, Any],
) -> SingleTurnSampleOrMultiTurnSample

Build the sample a row dict describes (see :func:sample_type_for).

Source code in src/ragas/dataset_schema.py
def sample_from_dict(row: t.Mapping[str, t.Any]) -> SingleTurnSampleOrMultiTurnSample:
    """Build the sample a row dict describes (see :func:`sample_type_for`)."""
    return sample_type_for(row)(**row)

sample_type_for

sample_type_for(
    row: Mapping[str, Any],
) -> Type[SingleTurnSampleOrMultiTurnSample]

The sample class a row dict is: multi-turn if user_input is a list of messages.

Source code in src/ragas/dataset_schema.py
def sample_type_for(
    row: t.Mapping[str, t.Any],
) -> t.Type[SingleTurnSampleOrMultiTurnSample]:
    """The sample class a row dict is: multi-turn if ``user_input`` is a list of messages."""
    if isinstance(row.get("user_input"), list):
        return MultiTurnSample
    return SingleTurnSample

Message

Bases: BaseModel

Represents a generic message.

Attributes:

Name Type Description
content str

The content of the message.

metadata (Optional[Dict[str, Any]], optional)

Additional metadata associated with the message.

ToolCall

Bases: BaseModel

Represents a tool call with a name and arguments.

Attributes:

Name Type Description
name str

The name of the tool being called.

args Dict[str, Any]

A dictionary of arguments for the tool call, where keys are argument names and values can be strings, integers, or floats.

HumanMessage

Bases: Message

Represents a message from a human user.

Attributes:

Name Type Description
type Literal[human]

The type of the message, always set to "human".

Methods:

Name Description
pretty_repr

Returns a formatted string representation of the human message.

pretty_repr

pretty_repr()

Returns a formatted string representation of the human message.

Source code in src/ragas/messages.py
def pretty_repr(self):
    """Returns a formatted string representation of the human message."""
    return f"Human: {self.content}"

ToolMessage

Bases: Message

Represents a message from a tool.

Attributes:

Name Type Description
type Literal[tool]

The type of the message, always set to "tool".

Methods:

Name Description
pretty_repr

Returns a formatted string representation of the tool message.

pretty_repr

pretty_repr()

Returns a formatted string representation of the tool message.

Source code in src/ragas/messages.py
def pretty_repr(self):
    """Returns a formatted string representation of the tool message."""
    return f"ToolOutput: {self.content}"

AIMessage

Bases: Message

Represents a message from an AI.

Attributes:

Name Type Description
type Literal[ai]

The type of the message, always set to "ai".

tool_calls Optional[List[ToolCall]]

A list of tool calls made by the AI, if any.

metadata Optional[Dict[str, Any]]

Additional metadata associated with the AI message.

Methods:

Name Description
dict

Returns a dictionary representation of the AI message.

pretty_repr

Returns a formatted string representation of the AI message.

to_dict

to_dict(**kwargs)

Returns a dictionary representation of the AI message.

Source code in src/ragas/messages.py
def to_dict(self, **kwargs):
    """
    Returns a dictionary representation of the AI message.
    """
    content = (
        self.content
        if self.tool_calls is None
        else {
            "text": self.content,
            "tool_calls": [tc.dict() for tc in self.tool_calls],
        }
    )
    return {"content": content, "type": self.type}

pretty_repr

pretty_repr()

Returns a formatted string representation of the AI message.

Source code in src/ragas/messages.py
def pretty_repr(self):
    """
    Returns a formatted string representation of the AI message.
    """
    lines = []
    if self.content != "":
        lines.append(f"AI: {self.content}")
    if self.tool_calls is not None:
        lines.append("Tools:")
        for tc in self.tool_calls:
            lines.append(f"  {tc.name}: {tc.args}")

    return "\n".join(lines)

ragas.dataset.Dataset

Dataset(
    name: str,
    backend: BaseBackend,
    data_model: Type[T],
    data: Optional[List[T]] = None,
)
Dataset(
    name: str,
    backend: BaseBackend,
    data_model: None = None,
    data: Optional[List[Dict[str, Any]]] = None,
)
Dataset(
    name: str,
    backend: str,
    data_model: Type[T],
    data: Optional[List[T]] = None,
    **kwargs,
)
Dataset(
    name: str,
    backend: str,
    data_model: None = None,
    data: Optional[List[Dict[str, Any]]] = None,
    **kwargs,
)
Dataset(
    name: str,
    backend: Union[BaseBackend, str],
    data_model: Optional[Type[T]] = None,
    data: Optional[List[Any]] = None,
    **kwargs,
)

Bases: DataTable[T]

Dataset class for managing dataset entries.

Inherits all functionality from DataTable. This class represents datasets specifically (as opposed to experiments).

Source code in src/ragas/dataset.py
def __init__(
    self,
    name: str,
    backend: t.Union[BaseBackend, str],
    data_model: t.Optional[t.Type[T]] = None,
    data: t.Optional[t.List[t.Any]] = None,
    **kwargs,
):
    """Initialize a Dataset with a backend.

    Args:
        name: The name of the dataset
        backend: Either a BaseBackend instance or backend name string (e.g., "local/csv")
        data_model: Optional Pydantic model class for entries
        data: Optional initial data list
        **kwargs: Additional arguments passed to backend constructor (when using string backend)

    Examples:
        # Using string backend name
        dataset = Dataset("my_data", "local/csv", root_dir="./data")

        # Using backend instance (existing behavior)
        backend = LocalCSVBackend(root_dir="./data")
        dataset = Dataset("my_data", backend)
    """
    # Store basic properties
    self.name = name
    self.data_model = data_model
    # Resolve backend if string
    self.backend = self._resolve_backend(backend, **kwargs)
    self._data: t.List[t.Union[t.Dict, T]] = data or []

ragas.experiment.Experiment

Experiment(
    name: str,
    backend: BaseBackend,
    data_model: Type[T],
    data: Optional[List[T]] = None,
)
Experiment(
    name: str,
    backend: BaseBackend,
    data_model: None = None,
    data: Optional[List[Dict[str, Any]]] = None,
)
Experiment(
    name: str,
    backend: str,
    data_model: Type[T],
    data: Optional[List[T]] = None,
    **kwargs,
)
Experiment(
    name: str,
    backend: str,
    data_model: None = None,
    data: Optional[List[Dict[str, Any]]] = None,
    **kwargs,
)
Experiment(
    name: str,
    backend: Union[BaseBackend, str],
    data_model: Optional[Type[T]] = None,
    data: Optional[List[Any]] = None,
    **kwargs,
)

Bases: DataTable

Source code in src/ragas/dataset.py
def __init__(
    self,
    name: str,
    backend: t.Union[BaseBackend, str],
    data_model: t.Optional[t.Type[T]] = None,
    data: t.Optional[t.List[t.Any]] = None,
    **kwargs,
):
    """Initialize a Dataset with a backend.

    Args:
        name: The name of the dataset
        backend: Either a BaseBackend instance or backend name string (e.g., "local/csv")
        data_model: Optional Pydantic model class for entries
        data: Optional initial data list
        **kwargs: Additional arguments passed to backend constructor (when using string backend)

    Examples:
        # Using string backend name
        dataset = Dataset("my_data", "local/csv", root_dir="./data")

        # Using backend instance (existing behavior)
        backend = LocalCSVBackend(root_dir="./data")
        dataset = Dataset("my_data", backend)
    """
    # Store basic properties
    self.name = name
    self.data_model = data_model
    # Resolve backend if string
    self.backend = self._resolve_backend(backend, **kwargs)
    self._data: t.List[t.Union[t.Dict, T]] = data or []