Skip to content

LLMs

InstructorBaseRagasLLM

Bases: ABC

Base class for LLMs using the Instructor library pattern.

Provider calls are retried with exponential backoff on transient errors (rate limits, timeouts, 5xx) per run_config (#73): the RunConfig passed to metrics_experiment / TestsetGenerator for calls made inside that run, else this LLM's set_run_config, else RunConfig() defaults. A run's config never sticks to the LLM (#76).

set_run_config

set_run_config(run_config: RunConfig) -> None

Retry/backoff settings for this LLM's provider calls.

Source code in src/ragas/llms/base.py
def set_run_config(self, run_config: RunConfig) -> None:
    """Retry/backoff settings for this LLM's provider calls."""
    self._run_config = run_config

generate abstractmethod

generate(
    prompt: LLMInput,
    response_model: Type[InstructorTypeVar],
    temperature: Optional[float] = None,
    sample_index: Optional[int] = None,
) -> InstructorTypeVar

Generate a response using the configured LLM.

For async clients, this will run the async method in the appropriate event loop.

prompt is a string, sent as one user message, or a list of OpenAI-format chat messages (content may be a list of text and image_url parts) sent as given; system_prompt is prepended to either unless the messages carry their own system message. Both go through the same retries, cache, rate limit, tracing and usage recording (#214).

temperature overrides the configured temperature for this call, for metrics that deliberately draw several samples (strictness). sample_index does not reach the provider; it only gives each sample its own cache entry so a cache doesn't collapse N samples into one.

Source code in src/ragas/llms/base.py
@abstractmethod
def generate(
    self,
    prompt: LLMInput,
    response_model: t.Type[InstructorTypeVar],
    temperature: t.Optional[float] = None,
    sample_index: t.Optional[int] = None,
) -> InstructorTypeVar:
    """Generate a response using the configured LLM.

    For async clients, this will run the async method in the appropriate event loop.

    ``prompt`` is a string, sent as one user message, or a list of
    OpenAI-format chat messages (content may be a list of text and
    ``image_url`` parts) sent as given; ``system_prompt`` is prepended to
    either unless the messages carry their own system message. Both go
    through the same retries, cache, rate limit, tracing and usage
    recording (#214).

    ``temperature`` overrides the configured temperature for this call, for
    metrics that deliberately draw several samples (``strictness``).
    ``sample_index`` does not reach the provider; it only gives each sample
    its own cache entry so a cache doesn't collapse N samples into one.
    """

agenerate abstractmethod async

agenerate(
    prompt: LLMInput,
    response_model: Type[InstructorTypeVar],
    temperature: Optional[float] = None,
    sample_index: Optional[int] = None,
) -> InstructorTypeVar

Asynchronously generate a response using the configured LLM.

See generate for temperature and sample_index.

Source code in src/ragas/llms/base.py
@abstractmethod
async def agenerate(
    self,
    prompt: LLMInput,
    response_model: t.Type[InstructorTypeVar],
    temperature: t.Optional[float] = None,
    sample_index: t.Optional[int] = None,
) -> InstructorTypeVar:
    """Asynchronously generate a response using the configured LLM.

    See ``generate`` for ``temperature`` and ``sample_index``.
    """

InstructorLLM

InstructorLLM(
    client: Any,
    model: str,
    provider: str,
    model_args: Optional[InstructorModelArgs] = None,
    cache: Optional[CacheInterface] = None,
    sync_client: Optional[Any] = None,
    reasoning: Optional[bool] = None,
    is_async: Optional[bool] = None,
    **kwargs,
)

Bases: CachedCopyable, InstructorBaseRagasLLM

LLM wrapper using the Instructor library for structured outputs.

llm_factory() builds these over LiteLLM (#182). You can also construct one directly around your own instructor client (e.g. instructor.from_provider(...)); the provider rules below still apply, but LiteLLM's drop_params and the LiteLLM-only defaults don't.

Source code in src/ragas/llms/base.py
def __init__(
    self,
    client: t.Any,
    model: str,
    provider: str,
    model_args: t.Optional[InstructorModelArgs] = None,
    cache: t.Optional[CacheInterface] = None,
    sync_client: t.Optional[t.Any] = None,
    reasoning: t.Optional[bool] = None,
    is_async: t.Optional[bool] = None,
    **kwargs,
):
    self.client = client
    # Sync counterpart of an async ``client`` over the same credentials,
    # for providers whose SDK object serves both (LiteLLM, google-genai).
    # generate() then calls it directly rather than hopping onto a
    # thread with a fresh event loop.
    self.sync_client = sync_client
    self.model = model
    self.provider = provider

    # Use deterministic defaults if no model_args provided
    if model_args is None:
        model_args = InstructorModelArgs()

    # None: detect from the model ID. True/False: the caller knows better,
    # e.g. an Azure deployment whose name says nothing about the model.
    self.reasoning = reasoning
    # Arguments the caller chose, as opposed to InstructorModelArgs
    # defaults: a default may be dropped where a provider rejects it, a
    # caller's choice is sent as given.
    self._explicit_args = set(model_args.model_fields_set) | set(kwargs)

    # Convert to dict and merge with any additional kwargs
    self.model_args = {**model_args.model_dump(), **kwargs}

    # Extract system_prompt separately (not passed to LLM API)
    self.system_prompt = self.model_args.pop("system_prompt", None)

    # A collector you attach yourself. Scoring runs and TestsetGenerator
    # don't set this: their collector is run-scoped (see
    # _active_usage_collector).
    self.usage_collector: t.Optional[t.Any] = None

    self.cache = cache

    # Whether ``client`` is async: llm_factory() knows; for a client of
    # your own, its ``chat.completions.create`` says.
    self.is_async = _has_async_create(client) if is_async is None else is_async

    self._wrap_with_cache()

generate

generate(
    prompt: LLMInput,
    response_model: Type[InstructorTypeVar],
    temperature: Optional[float] = None,
    sample_index: Optional[int] = None,
) -> InstructorTypeVar

Generate a response using the configured LLM.

For async clients, this will run the async method in the appropriate event loop. See InstructorBaseRagasLLM.generate for temperature/sample_index.

Source code in src/ragas/llms/base.py
def generate(
    self,
    prompt: LLMInput,
    response_model: t.Type[InstructorTypeVar],
    temperature: t.Optional[float] = None,
    sample_index: t.Optional[int] = None,
) -> InstructorTypeVar:
    """Generate a response using the configured LLM.

    For async clients, this will run the async method in the appropriate event loop.
    See ``InstructorBaseRagasLLM.generate`` for ``temperature``/``sample_index``.
    """
    rc = self.run_config
    with traced_call(
        prompt,
        lambda: self._messages(prompt),
        response_model,
        self.model,
        self.provider,
    ) as run:
        if self.sync_client is not None:
            result = call_with_retry(
                rc,
                self._call_sync,
                self.sync_client,
                prompt,
                response_model,
                temperature,
            )
        elif self.is_async:
            result = run_sync(
                acall_with_retry(
                    rc, self._call_async, prompt, response_model, temperature
                )
            )
        else:
            result = call_with_retry(
                rc,
                self._call_sync,
                self.client,
                prompt,
                response_model,
                temperature,
            )
        end_call(run, result)
        return result

agenerate async

agenerate(
    prompt: LLMInput,
    response_model: Type[InstructorTypeVar],
    temperature: Optional[float] = None,
    sample_index: Optional[int] = None,
) -> InstructorTypeVar

Asynchronously generate a response using the configured LLM.

A synchronous client is run on a worker thread, so every collections metric (all of which are async) works with OpenAI(), Anthropic() and other sync clients (#68). See InstructorBaseRagasLLM.generate for temperature/sample_index.

Source code in src/ragas/llms/base.py
async def agenerate(
    self,
    prompt: LLMInput,
    response_model: t.Type[InstructorTypeVar],
    temperature: t.Optional[float] = None,
    sample_index: t.Optional[int] = None,
) -> InstructorTypeVar:
    """Asynchronously generate a response using the configured LLM.

    A synchronous client is run on a worker thread, so every collections
    metric (all of which are async) works with ``OpenAI()``,
    ``Anthropic()`` and other sync clients (#68).
    See ``InstructorBaseRagasLLM.generate`` for ``temperature``/``sample_index``.
    """
    rc = self.run_config
    with traced_call(
        prompt,
        lambda: self._messages(prompt),
        response_model,
        self.model,
        self.provider,
    ) as run:
        if not self.is_async:
            result = await asyncio.to_thread(
                call_with_retry,
                rc,
                self._call_sync,
                self.client,
                prompt,
                response_model,
                temperature,
            )
        else:
            result = await acall_with_retry(
                rc, self._call_async, prompt, response_model, temperature
            )
        end_call(run, result)
        return result

llm_factory

llm_factory(
    model: str,
    provider: str = "openai",
    client: Optional[Any] = None,
    adapter: str = "auto",
    cache: Optional[CacheInterface] = None,
    mode: Optional[Mode] = None,
    rate_limit: Optional[RateLimit] = None,
    **kwargs: Any,
) -> InstructorBaseRagasLLM

Create an LLM for structured output. Every call goes through LiteLLM.

Address any LiteLLM-supported provider by model name; credentials come from the environment (OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY, ...) or from api_key= / api_base=.

Args: model: A LiteLLM model string: "gpt-6-luna", "anthropic/claude-sonnet-5-5", "gemini/gemini-3.6-flash", "ollama/qwen3.8:27b", "bedrock/<model id>", ... See https://docs.litellm.ai/docs/providers. provider: Prefixed onto a bare model when there is no client: llm_factory("llama3", provider="ollama") is "ollama/llama3". "openai" adds no prefix. client: Optional. An OpenAI / AsyncOpenAI (any OpenAI-compatible endpoint, via base_url) or AzureOpenAI / AsyncAzureOpenAI client, passed to LiteLLM with its settings (a copy of it: LiteLLM changes the object it is given), or a litellm.Router. Native SDK clients of other providers (Anthropic(), genai.Client(), Mistral, Groq, Cohere, boto3) are no longer accepted; use the provider's model string instead (#182). adapter: Deprecated and ignored: there is one path (#182). cache: Optional cache backend for LLM responses. mode: instructor mode. Defaults to Mode.JSON for OpenAI/Azure clients and for Claude, otherwise instructor's default. Native modes (Mode.ANTHROPIC_*, Mode.GENAI_*, ...) raise. rate_limit: Optional ragas.rate_limit.RateLimit: requests (and tokens) per minute. Every request this LLM sends waits for it, including retries; pass the same object to LLMs that share an API key (#171). **kwargs: Model arguments (temperature, top_p, max_tokens, system_prompt, reasoning, ...) and LiteLLM call arguments (api_key, api_base, extra_headers, ...).

Returns: An InstructorLLM with generate() and agenerate().

Examples: llm = llm_factory("gpt-6-luna") # OPENAI_API_KEY llm = llm_factory("anthropic/claude-sonnet-5-5") # ANTHROPIC_API_KEY llm = llm_factory("gemini/gemini-3.6-flash") # GEMINI_API_KEY llm = llm_factory("ollama/qwen3.8:27b") # local Ollama

# an OpenAI-compatible endpoint, with your own client settings
from openai import OpenAI
client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")
llm = llm_factory("qwen3.8:27b", client=client)
Source code in src/ragas/llms/factory.py
def llm_factory(
    model: str,
    provider: str = "openai",
    client: t.Optional[t.Any] = None,
    adapter: str = "auto",
    cache: t.Optional[CacheInterface] = None,
    mode: t.Optional[instructor.Mode] = None,
    rate_limit: t.Optional[RateLimit] = None,
    **kwargs: t.Any,
) -> InstructorBaseRagasLLM:
    """
    Create an LLM for structured output. Every call goes through LiteLLM.

    Address any LiteLLM-supported provider by model name; credentials come from
    the environment (``OPENAI_API_KEY``, ``ANTHROPIC_API_KEY``,
    ``GEMINI_API_KEY``, ...) or from ``api_key=`` / ``api_base=``.

    Args:
        model: A LiteLLM model string: ``"gpt-6-luna"``,
            ``"anthropic/claude-sonnet-5-5"``, ``"gemini/gemini-3.6-flash"``,
            ``"ollama/qwen3.8:27b"``, ``"bedrock/<model id>"``, ...
            See https://docs.litellm.ai/docs/providers.
        provider: Prefixed onto a bare ``model`` when there is no client:
            ``llm_factory("llama3", provider="ollama")`` is
            ``"ollama/llama3"``. ``"openai"`` adds no prefix.
        client: Optional. An ``OpenAI`` / ``AsyncOpenAI`` (any
            OpenAI-compatible endpoint, via ``base_url``) or ``AzureOpenAI`` /
            ``AsyncAzureOpenAI`` client, passed to LiteLLM with its settings
            (a copy of it: LiteLLM changes the object it is given), or a
            ``litellm.Router``. Native SDK clients of other providers
            (``Anthropic()``, ``genai.Client()``, Mistral, Groq, Cohere,
            boto3) are no longer accepted; use the provider's model string
            instead (#182).
        adapter: Deprecated and ignored: there is one path (#182).
        cache: Optional cache backend for LLM responses.
        mode: instructor mode. Defaults to ``Mode.JSON`` for OpenAI/Azure
            clients and for Claude, otherwise instructor's default. Native
            modes (``Mode.ANTHROPIC_*``, ``Mode.GENAI_*``, ...) raise.
        rate_limit: Optional ``ragas.rate_limit.RateLimit``: requests (and
            tokens) per minute. Every request this LLM sends waits for it,
            including retries; pass the same object to LLMs that share an
            API key (#171).
        **kwargs: Model arguments (``temperature``, ``top_p``, ``max_tokens``,
            ``system_prompt``, ``reasoning``, ...) and LiteLLM call
            arguments (``api_key``, ``api_base``, ``extra_headers``, ...).

    Returns:
        An ``InstructorLLM`` with ``generate()`` and ``agenerate()``.

    Examples:
        llm = llm_factory("gpt-6-luna")  # OPENAI_API_KEY
        llm = llm_factory("anthropic/claude-sonnet-5-5")  # ANTHROPIC_API_KEY
        llm = llm_factory("gemini/gemini-3.6-flash")  # GEMINI_API_KEY
        llm = llm_factory("ollama/qwen3.8:27b")  # local Ollama

        # an OpenAI-compatible endpoint, with your own client settings
        from openai import OpenAI
        client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")
        llm = llm_factory("qwen3.8:27b", client=client)
    """
    from ragas.llms.base import InstructorLLM

    if not model:
        raise ValueError("model parameter is required")
    if adapter != "auto":
        warnings.warn(
            f"llm_factory(adapter={adapter!r}) is deprecated and ignored: every "
            "call goes through LiteLLM (#182).",
            DeprecationWarning,
            stacklevel=2,
        )
    if mode is not None:
        message = _native_mode_error(mode)
        if message:
            raise ValueError(message)

    transport = _litellm_transport(model, provider, client)
    kind = transport.kind

    # Default mode: JSON for OpenAI/Azure clients (TOOLS mode returned {} for
    # Dict fields, vibrantlabsai/ragas#2490) and for Claude (instructor's
    # forced tool call fails on claude-sonnet-5-5, #184); otherwise
    # instructor's own default, as before.
    if mode is None and (
        kind in ("openai", "azure") or is_anthropic_model("", transport.model)
    ):
        mode = instructor.Mode.JSON

    completion, sync_completion = transport.completion, transport.sync_completion
    if transport.call_defaults:
        # bound per call; instructor's from_litellm(**kw) has its own `client`
        completion = functools.partial(completion, **transport.call_defaults)
        if sync_completion is not None:
            sync_completion = functools.partial(
                sync_completion, **transport.call_defaults
            )
    if rate_limit is not None:
        # paced below instructor, so its validation retries are counted too
        from ragas.rate_limit import limit_completion_calls

        completion = limit_completion_calls(completion, rate_limit)
        if sync_completion is not None:
            sync_completion = limit_completion_calls(sync_completion, rate_limit)
    llm = InstructorLLM(
        client=instructor.from_litellm(completion, **_mode_kwargs(mode)),
        model=transport.model,
        # "openai"/"azure" for those clients, "litellm" otherwise
        provider=kind if kind in ("openai", "azure") else "litellm",
        cache=cache,
        sync_client=(
            instructor.from_litellm(sync_completion, **_mode_kwargs(mode))
            if sync_completion is not None
            else None
        ),
        is_async=transport.is_async,
        **kwargs,
    )
    llm._litellm = True
    llm.rate_limit = rate_limit
    if kind in ("openai", "azure"):
        llm.source_client = client
    return llm