Skip to content

Schemas

TestsetSample

Bases: BaseSample

Represents a sample in a test set.

Attributes:

Name Type Description
eval_sample Union[SingleTurnSample, MultiTurnSample]

The evaluation sample, which can be either a single-turn or multi-turn sample.

synthesizer_name str

The name of the synthesizer used to generate this sample.

TestsetPacket

Bases: BaseModel

A packet of testset samples to be uploaded to the server.

Testset dataclass

Testset(
    samples: List[TestsetSample],
    run_id: str = (lambda: str(uuid4()))(),
    cost_cb: Optional[TokenUsageCollector] = None,
    personas: Optional[List["Persona"]] = None,
)

Represents a test set containing multiple test samples.

Attributes:

Name Type Description
samples List[TestsetSample]

A list of TestsetSample objects representing the samples in the test set.

personas Optional[List[Persona]]

The personas the samples were generated from, set by TestsetGenerator.generate(). Pass them as TestsetGenerator(persona_list=...) to reuse them in another run. Kept in memory only; not written by to_list() or the file exports.

to_dataset

to_dataset(
    name: str,
    backend: Union["BaseBackend", str] = "inmemory",
    **backend_kwargs: Any,
) -> "Dataset"

The generated samples as a Dataset, to score or save.

Rows are copies of each sample's eval_sample, and data_model is their type (SingleTurnSample or MultiTurnSample). The synthesizer names are left out; to_list() keeps them. Nothing is written until you call save()::

dataset = testset.to_dataset("testset", "local/jsonl", root_dir="data")
dataset.save()
exp = metrics_experiment([Faithfulness(llm=llm)]).run(dataset)

Prefer local/jsonl to local/csv for a testset you'll score. CSV loads a missing text value (a sample with no reference) as "", which metrics_experiment scores as an empty reference instead of reporting the column missing.

Args: name: The dataset's name. backend: A backend name (default "inmemory") or instance. **backend_kwargs: Constructor arguments for a backend given by name, e.g. root_dir.

Raises: ValueError: the testset mixes single- and multi-turn samples.

Source code in src/ragas/testset/synthesizers/testset_schema.py
def to_dataset(
    self,
    name: str,
    backend: t.Union["BaseBackend", str] = "inmemory",
    **backend_kwargs: t.Any,
) -> "Dataset":
    """The generated samples as a ``Dataset``, to score or save.

    Rows are copies of each sample's ``eval_sample``, and ``data_model``
    is their type (``SingleTurnSample`` or ``MultiTurnSample``). The
    synthesizer names are left out; ``to_list()`` keeps them. Nothing is
    written until you call ``save()``::

        dataset = testset.to_dataset("testset", "local/jsonl", root_dir="data")
        dataset.save()
        exp = metrics_experiment([Faithfulness(llm=llm)]).run(dataset)

    Prefer ``local/jsonl`` to ``local/csv`` for a testset you'll score.
    CSV loads a missing text value (a sample with no ``reference``) as
    ``""``, which ``metrics_experiment`` scores as an empty reference
    instead of reporting the column missing.

    Args:
        name: The dataset's name.
        backend: A backend name (default ``"inmemory"``) or instance.
        **backend_kwargs: Constructor arguments for a backend given by
            name, e.g. ``root_dir``.

    Raises:
        ValueError: the testset mixes single- and multi-turn samples.
    """
    from ragas.dataset import Dataset

    rows = [sample.eval_sample.model_copy(deep=True) for sample in self.samples]
    types = {type(row) for row in rows}
    if len(types) > 1:
        raise ValueError(
            "The testset mixes SingleTurnSample and MultiTurnSample samples; "
            "a Dataset holds one sample type."
        )
    data_model = types.pop() if types else SingleTurnSample
    return Dataset(
        name, backend, data_model=data_model, data=rows, **backend_kwargs
    )

to_list

to_list() -> List[Dict]

Converts the Testset to a list of dictionaries.

Source code in src/ragas/testset/synthesizers/testset_schema.py
def to_list(self) -> t.List[t.Dict]:
    """
    Converts the Testset to a list of dictionaries.
    """
    list_dict = []
    for sample in self.samples:
        sample_dict = sample.eval_sample.model_dump(exclude_none=True)
        sample_dict["synthesizer_name"] = sample.synthesizer_name
        list_dict.append(sample_dict)
    return list_dict

from_list classmethod

from_list(data: List[Dict]) -> Testset

Converts a list of dictionaries to a Testset.

Source code in src/ragas/testset/synthesizers/testset_schema.py
@classmethod
def from_list(cls, data: t.List[t.Dict]) -> Testset:
    """
    Converts a list of dictionaries to a Testset.
    """
    # first create the samples
    samples = []
    for sample in data:
        synthesizer_name = sample["synthesizer_name"]
        # remove the synthesizer name from the sample
        sample.pop("synthesizer_name")
        # the remaining sample is the eval_sample
        eval_sample = sample

        # if user_input is a list it is MultiTurnSample
        if "user_input" in eval_sample and not isinstance(
            eval_sample.get("user_input"), list
        ):
            eval_sample = SingleTurnSample(**eval_sample)
        else:
            eval_sample = MultiTurnSample(**eval_sample)

        samples.append(
            TestsetSample(
                eval_sample=eval_sample, synthesizer_name=synthesizer_name
            )
        )
    # then create the testset
    return Testset(samples=samples)

to_pandas

to_pandas() -> 'PandasDataframe'

The samples (as to_list()) as a pandas DataFrame.

Source code in src/ragas/testset/synthesizers/testset_schema.py
def to_pandas(self) -> "PandasDataframe":
    """The samples (as ``to_list()``) as a pandas DataFrame."""
    try:
        import pandas as pd
    except ImportError:
        raise ImportError(
            "pandas is not installed. Please install it to use this function."
        )
    return pd.DataFrame(self.to_list())

to_csv

to_csv(path: Union[str, Path]) -> None

Write the samples (as to_list()) to a CSV file.

Source code in src/ragas/testset/synthesizers/testset_schema.py
def to_csv(self, path: t.Union[str, Path]) -> None:
    """Write the samples (as ``to_list()``) to a CSV file."""
    import csv

    from ragas.backends.flat import encode_flat_rows

    data = self.to_list()
    if not data:
        return
    # Every sample's fields make the header, and lists/dicts are JSON
    # text rather than Python repr (#100).
    fieldnames, rows = encode_flat_rows(data)
    with open(path, "w", newline="", encoding="utf-8") as csvfile:
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()
        writer.writerows(rows)

to_jsonl

to_jsonl(path: Union[str, Path]) -> None

Write the samples (as to_list()) to a JSONL file.

Source code in src/ragas/testset/synthesizers/testset_schema.py
def to_jsonl(self, path: t.Union[str, Path]) -> None:
    """Write the samples (as ``to_list()``) to a JSONL file."""
    with open(path, "w", encoding="utf-8") as jsonlfile:
        for sample in self.to_list():
            jsonlfile.write(json.dumps(sample, ensure_ascii=False) + "\n")

from_jsonl classmethod

from_jsonl(path: Union[str, Path]) -> Testset

Read a testset written by to_jsonl().

Source code in src/ragas/testset/synthesizers/testset_schema.py
@classmethod
def from_jsonl(cls, path: t.Union[str, Path]) -> Testset:
    """Read a testset written by ``to_jsonl()``."""
    with open(path, "r", encoding="utf-8") as jsonlfile:
        data = [json.loads(line) for line in jsonlfile if line.strip()]
    return cls.from_list(data)

total_tokens

total_tokens() -> Union[List[TokenUsage], TokenUsage]

Compute the total tokens used in the evaluation.

Source code in src/ragas/testset/synthesizers/testset_schema.py
def total_tokens(self) -> t.Union[t.List[TokenUsage], TokenUsage]:
    """
    Compute the total tokens used in the evaluation.
    """
    if self.cost_cb is None:
        raise ValueError(
            "The Testset was not configured for computing cost. Please provide a token_usage_parser function to TestsetGenerator to compute cost."
        )
    return self.cost_cb.total_tokens()

total_cost

total_cost(
    cost_per_input_token: Optional[float] = None,
    cost_per_output_token: Optional[float] = None,
) -> float

Compute the total cost of the evaluation.

Source code in src/ragas/testset/synthesizers/testset_schema.py
def total_cost(
    self,
    cost_per_input_token: t.Optional[float] = None,
    cost_per_output_token: t.Optional[float] = None,
) -> float:
    """
    Compute the total cost of the evaluation.
    """
    if self.cost_cb is None:
        raise ValueError(
            "The Testset was not configured for computing cost. Please provide a token_usage_parser function to TestsetGenerator to compute cost."
        )
    return self.cost_cb.total_cost(
        cost_per_input_token=cost_per_input_token,
        cost_per_output_token=cost_per_output_token,
    )

from_annotated classmethod

from_annotated(path: str) -> Testset

Loads a testset from an annotated JSON file.

Source code in src/ragas/testset/synthesizers/testset_schema.py
@classmethod
def from_annotated(cls, path: str) -> Testset:
    """
    Loads a testset from an annotated JSON file.
    """
    with open(path, "r", encoding="utf-8") as f:
        annotated_testset = json.load(f)

    samples = []
    for sample in annotated_testset:
        if sample["approval_status"] == "approved":
            samples.append(TestsetSample(**sample))
    return cls(samples=samples)

QueryLength

Bases: str, Enum

Enumeration of query lengths. Available options are: LONG, MEDIUM, SHORT

QueryStyle

Bases: str, Enum

Enumeration of query styles. Available options are: MISSPELLED, PERFECT_GRAMMAR, POOR_GRAMMAR, WEB_SEARCH_LIKE

BaseScenario

Bases: BaseModel

Base class for representing a scenario for generating test samples.

Attributes:

Name Type Description
nodes List[Node]

List of nodes involved in the scenario.

style QueryStyle

The style of the query.

length QueryLength

The length of the query.

persona Persona

A persona associated with the scenario.

SingleHopSpecificQuerySynthesizer dataclass

SingleHopSpecificQuerySynthesizer(
    name: str = "single_hop_specific_query_synthesizer",
    llm: InstructorBaseRagasLLM = _default_llm_factory(),
    llm_context: Optional[str] = None,
    generate_query_reference_prompt: BasePrompt = QueryAnswerGenerationPrompt(),
    theme_persona_matching_prompt: BasePrompt = ThemesPersonasMatchingPrompt(),
    property_name: str = "entities",
)

Bases: SingleHopQuerySynthesizer

MultiHopSpecificQuerySynthesizer dataclass

MultiHopSpecificQuerySynthesizer(
    name: str = "multi_hop_specific_query_synthesizer",
    llm: InstructorBaseRagasLLM = _default_llm_factory(),
    llm_context: Optional[str] = None,
    generate_query_reference_prompt: BasePrompt = QueryAnswerGenerationPrompt(),
    property_name: str = "entities",
    relation_type: str = "entities_overlap",
    relation_overlap_property: str = "overlapped_items",
    theme_persona_matching_prompt: BasePrompt = ThemesPersonasMatchingPrompt(),
)

Bases: MultiHopQuerySynthesizer

Synthesize multi-hop queries based on a chunk cluster defined by entity overlap.

get_node_clusters

get_node_clusters(
    knowledge_graph: KnowledgeGraph,
) -> List[Tuple]

Identify clusters of nodes based on the specified relationship condition.

Source code in src/ragas/testset/synthesizers/multi_hop/specific.py
def get_node_clusters(self, knowledge_graph: KnowledgeGraph) -> t.List[t.Tuple]:
    """Identify clusters of nodes based on the specified relationship condition."""
    node_clusters = find_two_nodes_single_rel(
        knowledge_graph,
        relationship_condition=lambda rel: rel.type == self.relation_type,
    )
    logger.info("found %d clusters", len(node_clusters))
    return node_clusters

ragas.testset.persona

Persona

Bases: BaseModel

A user archetype the generated queries are written from the point of view of.

Attributes:

Name Type Description
name str

A short label for the persona, e.g. "New Joinee".

role_description str

What this persona knows and what they are trying to find out. This is what steers query generation, so it carries most of the signal.

PersonaList

Bases: BaseModel

get

get(key: str) -> Optional[Persona]

Return the persona named key, or None if there is none.

Names match ignoring case and surrounding or repeated whitespace, because they come back from an LLM, which often changes either (#84). An exact match wins over a normalised one.

Source code in src/ragas/testset/persona.py
def get(self, key: str) -> t.Optional[Persona]:
    """Return the persona named ``key``, or None if there is none.

    Names match ignoring case and surrounding or repeated whitespace,
    because they come back from an LLM, which often changes either (#84).
    An exact match wins over a normalised one.
    """
    for persona in self.personas:
        if persona.name == key:
            return persona
    wanted = _normalise_persona_name(key)
    for persona in self.personas:
        if _normalise_persona_name(persona.name) == wanted:
            return persona
    return None

generate_personas_from_kg

generate_personas_from_kg(
    kg: KnowledgeGraph,
    llm: InstructorBaseRagasLLM,
    persona_generation_prompt: PersonaGenerationPrompt = PersonaGenerationPrompt(),
    num_personas: int = 3,
    filter_fn: Callable[[Node], bool] = default_filter,
    callbacks: Callbacks = [],
) -> List[Persona]

Generate personas from a knowledge graph based on cluster of similar document summaries.

parameters: kg: KnowledgeGraph The knowledge graph to generate personas from. llm: InstructorBaseRagasLLM The LLM to use for generating the persona, such as the output of llm_factory(). persona_generation_prompt: PersonaGenerationPrompt The prompt to use for generating the persona. num_personas: int The maximum number of personas to generate. filter_fn: Callable[[Node], bool] A function to filter nodes in the knowledge graph. callbacks: Callbacks The callbacks to use for the generation process.

returns: t.List[Persona] The list of generated personas.

Source code in src/ragas/testset/persona.py
def generate_personas_from_kg(
    kg: KnowledgeGraph,
    llm: InstructorBaseRagasLLM,
    persona_generation_prompt: PersonaGenerationPrompt = PersonaGenerationPrompt(),
    num_personas: int = 3,
    filter_fn: t.Callable[[Node], bool] = default_filter,
    callbacks: Callbacks = [],
) -> t.List[Persona]:
    """
    Generate personas from a knowledge graph based on cluster of similar document summaries.

    parameters:
        kg: KnowledgeGraph
            The knowledge graph to generate personas from.
        llm: InstructorBaseRagasLLM
            The LLM to use for generating the persona, such as the output of
            ``llm_factory()``.
        persona_generation_prompt: PersonaGenerationPrompt
            The prompt to use for generating the persona.
        num_personas: int
            The maximum number of personas to generate.
        filter_fn: Callable[[Node], bool]
            A function to filter nodes in the knowledge graph.
        callbacks: Callbacks
            The callbacks to use for the generation process.


    returns:
        t.List[Persona]
            The list of generated personas.
    """

    llm = require_llm(llm, "generate_personas_from_kg()")
    nodes = [node for node in kg.nodes if filter_fn(node)]
    if len(nodes) == 0:
        raise ValueError(
            "No nodes that satisfied the given filer. Try changing the filter."
        )

    # Keep each summary paired with its own node's embedding, so the similarity
    # matrix and the summaries list share indices.
    summaries: t.List[str] = []
    embeddings: t.List[t.Any] = []
    for node in nodes:
        summary = node.properties.get("summary")
        if not isinstance(summary, str):
            continue
        embedding = node.properties.get("summary_embedding")
        if embedding is None:
            raise ValueError(f"Node {node.id} has a summary but no summary_embedding")
        summaries.append(summary)
        embeddings.append(embedding)
    num_personas = min(num_personas, len(summaries))

    groups = []
    if summaries:
        embedding_matrix = np.asarray(embeddings, dtype=float)
        cosine_similarities = cosine_matrix(embedding_matrix, embedding_matrix)
        visited = set()
        threshold = 0.75

        for i in range(len(summaries)):
            if i in visited:
                continue
            group = [i]
            visited.add(i)
            for j in range(i + 1, len(summaries)):
                if cosine_similarities[i, j] > threshold:
                    group.append(j)
                    visited.add(j)
            groups.append(group)

    top_summaries = []
    for group in groups:
        representative_summary = max([summaries[i] for i in group], key=len)
        top_summaries.append(representative_summary)

    if len(top_summaries) <= num_personas:
        top_summaries.extend(
            np.random.choice(top_summaries, num_personas - len(top_summaries))
        )

    # use run_async_batch to generate personas in parallel
    kwargs_list = [
        {
            "llm": llm,
            "data": StringIO(text=summary),
            "callbacks": callbacks,
            "temperature": 1.0,
        }
        for summary in top_summaries[:num_personas]
    ]
    persona_list = run_async_batch(
        desc="Generating personas",
        func=persona_generation_prompt.generate,
        kwargs_list=kwargs_list,
    )

    return persona_list