Prompt API Reference
The prompt system in Ragas provides a flexible and type-safe way to define prompts for LLM-based metrics and other components. This page documents the core prompt classes and their usage.
Overview
Ragas uses a modular prompt architecture based on the BasePrompt class. Prompts can be:
- Input/Output Models: Pydantic BaseModel classes that define the structure of prompt inputs and outputs
- Prompt Classes: Inherit from
BasePrompt to define instructions, examples, and prompt generation logic
Core Classes
InputModel = t.TypeVar('InputModel', bound=BaseModel)
OutputModel
module-attribute
OutputModel = t.TypeVar('OutputModel', bound=BaseModel)
BasePrompt
BasePrompt(
name: Optional[str] = None,
language: Optional[str] = None,
original_hash: Optional[int] = None,
)
Bases: ABC, Generic[InputModel, OutputModel]
A structured prompt with type-safe input and output models.
Subclasses set these as class attributes:
input_model / output_model: Pydantic model classes.
instruction: the task description.
examples: (input, output) pairs for few-shot learning.
states_output_language: when True and the prompt is adapted to a
language other than English, the rendered prompt tells the model to
write its output in that language. Opt-in: only prompts whose output is
final, user-facing text set it; prompts whose output must copy values
from the input would break if those were translated (#106).
Source code in src/ragas/prompt/metrics/base_prompt.py
| def __init__(
self,
name: t.Optional[str] = None,
language: t.Optional[str] = None,
original_hash: t.Optional[int] = None,
):
self.name = name or camel_to_snake(self.__class__.__name__)
if language is not None:
self.language = language
self.original_hash = original_hash
|
to_string
to_string(data: Optional[InputModel] = None) -> str
The complete prompt string for data.
Source code in src/ragas/prompt/metrics/base_prompt.py
| def to_string(self, data: t.Optional[InputModel] = None) -> str:
"""The complete prompt string for ``data``."""
language_line = (
output_language_directive(self.language)
if self.states_output_language
else ""
)
input_line = (
f"input: {data.model_dump_json(indent=4, exclude_none=True)}\n"
if data is not None
else "Input: (None)\n"
)
# A blank line before the examples; without examples, none (both
# renderers that this replaced did so, #214).
examples = self._generate_examples()
examples_block = f"\n{examples}\n" if examples else "\n"
return (
f"{self._generate_instruction()}\n"
f"{language_line}"
f"{self._generate_output_signature()}\n"
f"{examples_block}"
"-----------------------------\n\n"
"Now perform the same with the following input\n"
f"{input_line}"
"Output: "
)
|
Hook: transform the input before rendering.
Source code in src/ragas/prompt/metrics/base_prompt.py
| def process_input(self, input: InputModel) -> InputModel:
"""Hook: transform the input before rendering."""
return input
|
process_output
Hook: validate or transform each output.
Source code in src/ragas/prompt/metrics/base_prompt.py
| def process_output(self, output: OutputModel, input: InputModel) -> OutputModel:
"""Hook: validate or transform each output."""
return output
|
generate
async
generate(
llm: "InstructorBaseRagasLLM",
data: InputModel,
temperature: Optional[float] = None,
callbacks: Optional[Callbacks] = None,
) -> OutputModel
Generate one output for data.
Source code in src/ragas/prompt/metrics/base_prompt.py
| async def generate(
self,
llm: "InstructorBaseRagasLLM",
data: InputModel,
temperature: t.Optional[float] = None,
callbacks: t.Optional[Callbacks] = None,
) -> OutputModel:
"""Generate one output for ``data``."""
outputs = await self.generate_multiple(
llm=llm,
data=data,
n=1,
temperature=temperature,
callbacks=callbacks,
)
return outputs[0]
|
generate_multiple
async
generate_multiple(
llm: "InstructorBaseRagasLLM",
data: InputModel,
n: int = 1,
temperature: Optional[float] = None,
callbacks: Optional[Callbacks] = None,
**options: Any,
) -> List[OutputModel]
Generate n outputs for data.
The prompt is traced as its own run, nested under the run in progress
(a metric, an experiment row) when no callbacks are given (#196).
Output-parsing failures raise RagasOutputParserException.
options are passed to _generate_outputs; the base class takes
none.
Source code in src/ragas/prompt/metrics/base_prompt.py
| async def generate_multiple(
self,
llm: "InstructorBaseRagasLLM",
data: InputModel,
n: int = 1,
temperature: t.Optional[float] = None,
callbacks: t.Optional[Callbacks] = None,
**options: t.Any,
) -> t.List[OutputModel]:
"""Generate ``n`` outputs for ``data``.
The prompt is traced as its own run, nested under the run in progress
(a metric, an experiment row) when no ``callbacks`` are given (#196).
Output-parsing failures raise ``RagasOutputParserException``.
``options`` are passed to ``_generate_outputs``; the base class takes
none.
"""
callbacks = callbacks or current_group() or []
processed_data = self.process_input(data)
prompt_rm, prompt_cb = new_group(
name=self.name,
inputs={"data": processed_data},
callbacks=callbacks,
metadata={"type": ChainType.RAGAS_PROMPT},
)
prompt_text = self.to_string(processed_data)
# Every failure ends the prompt's trace span (#94).
try:
# The prompt's run is current while the LLM is called, so the
# LLM's call span nests under it.
with use_group(prompt_cb):
output_models = await self._generate_outputs(
llm,
data,
prompt_text,
n,
temperature,
prompt_cb,
**options,
)
except Exception as e:
error = as_output_parser_exception(e)
prompt_rm.on_chain_error(error=error)
if isinstance(error, RagasOutputParserException):
logger.error("Prompt %s failed to parse output: %s", self.name, e)
if error is not e:
raise error from e
raise
prompt_rm.on_chain_end({"output": output_models})
return output_models
|
adapt
async
adapt(
target_language: str,
llm: "InstructorBaseRagasLLM",
adapt_instruction: bool = False,
) -> "BasePrompt[InputModel, OutputModel]"
A copy of this prompt adapted to target_language.
The examples' strings are translated (keys and structure are kept),
and with adapt_instruction the instruction too.
Source code in src/ragas/prompt/metrics/base_prompt.py
| async def adapt(
self,
target_language: str,
llm: "InstructorBaseRagasLLM",
adapt_instruction: bool = False,
) -> "BasePrompt[InputModel, OutputModel]":
"""A copy of this prompt adapted to ``target_language``.
The examples' strings are translated (keys and structure are kept),
and with ``adapt_instruction`` the instruction too.
"""
new_prompt = copy.deepcopy(self)
new_prompt.language = target_language
strings = get_all_strings(self.examples)
if strings:
translated = await _translate_strings(strings, target_language, llm)
new_prompt.examples = update_strings(
obj=self.examples, old_strings=strings, new_strings=translated
)
if adapt_instruction:
[new_prompt.instruction] = await _translate_strings(
[self.instruction], target_language, llm
)
new_prompt.original_hash = hash(new_prompt)
return new_prompt
|
save
Save the instruction, examples and language as JSON.
Source code in src/ragas/prompt/metrics/base_prompt.py
| def save(self, file_path: str):
"""Save the instruction, examples and language as JSON."""
from ragas._version import __version__
data = {
"ragas_version": __version__,
"original_hash": (
hash(self) if self.original_hash is None else self.original_hash
),
"language": self.language,
"instruction": self.instruction,
"examples": [
{"input": example[0].model_dump(), "output": example[1].model_dump()}
for example in self.examples
],
}
if os.path.exists(file_path):
raise FileExistsError(f"The file '{file_path}' already exists.")
with open(file_path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
logger.info("Prompt saved to %s", file_path)
|
load
classmethod
Load a prompt saved with save.
Source code in src/ragas/prompt/metrics/base_prompt.py
| @classmethod
def load(cls, file_path: str):
"""Load a prompt saved with ``save``."""
from ragas._version import __version__
with open(file_path, "r", encoding="utf-8") as f:
data = json.load(f)
ragas_version = data.get("ragas_version")
if ragas_version != __version__:
logger.warning(
"Prompt was saved with Ragas v%s, but you are loading it with "
"Ragas v%s. There might be incompatibilities.",
ragas_version,
__version__,
)
original_hash = data.get("original_hash")
prompt = cls()
prompt.instruction = data["instruction"]
prompt.examples = [
(
prompt.input_model(**example["input"]),
prompt.output_model(**example["output"]),
)
for example in data["examples"]
]
prompt.language = data.get("language", prompt.language)
if original_hash is not None and hash(prompt) != original_hash:
logger.warning("Loaded prompt hash does not match the saved hash.")
return prompt
|
PromptMixin
Mixin class for classes that have prompts.
eg: BaseSynthesizer
get_prompts
Returns a dictionary of prompts for the class.
Source code in src/ragas/prompt/mixin.py
| def get_prompts(self) -> t.Dict[str, BasePrompt]:
"""
Returns a dictionary of prompts for the class.
"""
prompts = {}
for _, value in self._get_prompts().items():
prompts.update({value.name: value})
return prompts
|
set_prompts
Sets the prompts for the class.
Raises:
| Type |
Description |
ValueError
|
If there is no prompt of that name, or the value is not a
ragas.prompt.BasePrompt.
|
Source code in src/ragas/prompt/mixin.py
| def set_prompts(self, **prompts: BasePrompt) -> None:
"""
Sets the prompts for the class.
Raises
------
ValueError
If there is no prompt of that name, or the value is not a
``ragas.prompt.BasePrompt``.
"""
available_prompts = self.get_prompts()
name_to_var = {v.name: k for k, v in self._get_prompts().items()}
for key, value in prompts.items():
if key not in available_prompts:
raise ValueError(
f"Prompt with name '{key}' does not exist. Use get_prompts() to see available prompts."
)
if not isinstance(value, BasePrompt):
raise ValueError(
f"Prompt with name '{key}' must be an instance of 'ragas.prompt.BasePrompt'"
)
setattr(self, name_to_var[key], value)
|
adapt_prompts
async
Adapts the prompts in the class to the given language and using the given LLM.
Notes
Make sure you use the best available LLM for adapting the prompts and then save and load the prompts using
save_prompts and load_prompts
methods.
Source code in src/ragas/prompt/mixin.py
| async def adapt_prompts(
self,
language: str,
llm: InstructorBaseRagasLLM,
adapt_instruction: bool = False,
) -> t.Dict[str, BasePrompt]:
"""
Adapts the prompts in the class to the given language and using the given LLM.
Notes
-----
Make sure you use the best available LLM for adapting the prompts and then save and load the prompts using
[save_prompts][ragas.prompt.mixin.PromptMixin.save_prompts] and [load_prompts][ragas.prompt.mixin.PromptMixin.load_prompts]
methods.
"""
prompts = self.get_prompts()
adapted_prompts = {}
for name, prompt in prompts.items():
adapted_prompt = await prompt.adapt(language, llm, adapt_instruction)
adapted_prompts[name] = adapted_prompt
return adapted_prompts
|
save_prompts
Saves the prompts to a directory in the format of {name}_{language}.json
Source code in src/ragas/prompt/mixin.py
| def save_prompts(self, path: str):
"""
Saves the prompts to a directory in the format of {name}_{language}.json
"""
# check if path is valid
if not os.path.exists(path):
raise ValueError(f"Path {path} does not exist")
prompts = self.get_prompts()
for prompt_name, prompt in prompts.items():
# hash_hex = f"0x{hash(prompt) & 0xFFFFFFFFFFFFFFFF:016x}"
if self.name == "":
file_name = os.path.join(path, f"{prompt_name}_{prompt.language}.json")
else:
file_name = os.path.join(
path, f"{self.name}_{prompt_name}_{prompt.language}.json"
)
prompt.save(file_name)
|
load_prompts
load_prompts(path: str, language: Optional[str] = None)
Loads the prompts from a path. File should be in the format of {name}_{language}.json
Source code in src/ragas/prompt/mixin.py
| def load_prompts(self, path: str, language: t.Optional[str] = None):
"""
Loads the prompts from a path. File should be in the format of {name}_{language}.json
"""
# check if path is valid
if not os.path.exists(path):
raise ValueError(f"Path {path} does not exist")
# check if language is supported, defaults to english
if language is None:
language = "english"
logger.info(
"Language not specified, loading prompts for default language: %s",
language,
)
loaded_prompts = {}
for prompt_name, prompt in self.get_prompts().items():
if self.name == "":
file_name = os.path.join(path, f"{prompt_name}_{language}.json")
else:
file_name = os.path.join(
path, f"{self.name}_{prompt_name}_{language}.json"
)
loaded_prompt = prompt.__class__.load(file_name)
loaded_prompts[prompt_name] = loaded_prompt
return loaded_prompts
|
Metrics Collections Prompts
Modern metrics in Ragas use specialized prompt classes. Each metric module contains:
- Input Model: Defines what data the prompt needs (e.g.,
StatementGeneratorInput)
- Output Model: Defines the expected LLM response structure (e.g.,
StatementGeneratorOutput)
- Prompt Class: Inherits from
BasePrompt to generate the prompt string with examples and instructions
Example: Faithfulness Metric Prompts
from ragas.metrics.collections.faithfulness.util import (
StatementGeneratorInput,
StatementGeneratorPrompt,
)
# The prompt class combines input/output models with instructions and examples
prompt = StatementGeneratorPrompt()
# Create input data
input_data = StatementGeneratorInput(
question="What is the capital of France?",
answer="The capital of France is Paris.",
)
# Generate the prompt string for the LLM
prompt_string = prompt.to_string(input_data)
# The LLM's reply is parsed into StatementGeneratorOutput (a list of statements)
Available Metric Prompts
See the individual metric documentation for details on their prompts:
Customization
For detailed guidance on customizing prompts for metrics, see Modifying prompts in metrics.