View module source on GitHub

​
TestRunBuilder

@final
class TestRunBuilder(Generic[T])
Builder for test runs.

​
__init__

def __init__(base_url: str, api_key: str, name: str, workspace_id: str,
             evaluators: List[Union[str, BaseEvaluator]])
Constructor

​
with_data_structure

def with_data_structure(data: T) -> "TestRunBuilder[T]"
Set the data structure for the test run Arguments:
NameTypeDescription
dataTThe data structure to use
Returns:
NameDescription
[TestRunBuilder](/sdk/python/references/test_runs/test_run_builder)[T]The current TestRunBuilder instance for method chaining

​
with_data

def with_data(data: Data) -> "TestRunBuilder[T]"
Set the data for the test run Arguments:
NameTypeDescription
dataDataValue[T]The data to use
Returns:
NameDescription
[TestRunBuilder](/sdk/python/references/test_runs/test_run_builder)[T]The current TestRunBuilder instance for method chaining

​
with_evaluators

def with_evaluators(
        *evaluators: Union[str, BaseEvaluator]) -> "TestRunBuilder[T]"
Add evaluators to the test run Arguments:
NameTypeDescription
*evaluatorsstrThe evaluators to add
Returns:
NameDescription
[TestRunBuilder](/sdk/python/references/test_runs/test_run_builder)[T]The current TestRunBuilder instance for method chaining

​
with_human_evaluation_config

def with_human_evaluation_config(
        config: HumanEvaluationConfig) -> "TestRunBuilder[T]"
Set the human evaluation configuration for the test run Arguments:
NameTypeDescription
configHumanEvaluationConfigThe human evaluation configuration to use
Returns:
NameDescription
[TestRunBuilder](/sdk/python/references/test_runs/test_run_builder)[T]The current TestRunBuilder instance for method chaining

​
with_workflow_id

def with_workflow_id(
        workflow_id: str,
        context_to_evaluate: Optional[str] = None) -> "TestRunBuilder[T]"
Set the workflow ID for the test run. Optionally, you can also set the context to evaluate for the workflow. (Note: setting the context to evaluate will end up overriding the CONTEXT_TO_EVALUATE dataset column value) Arguments:
NameTypeDescription
workflow_idstrThe ID of the workflow to use
context_to_evaluateOptional[str]The context to evaluate for the workflow (variable name essentially).
Returns:
NameDescription
[TestRunBuilder](/sdk/python/references/test_runs/test_run_builder)[T]The current TestRunBuilder instance for method chaining
Raises:
  • ValueError - If a prompt version ID, prompt chain version ID or output function is already set for this run builder

​
with_prompt_version_id

def with_prompt_version_id(
        prompt_version_id: str,
        context_to_evaluate: Optional[str] = None) -> "TestRunBuilder[T]"
Set the prompt version ID for the test run. Optionally, you can also set the context to evaluate for the prompt. (Note: setting the context to evaluate will end up overriding the CONTEXT_TO_EVALUATE dataset column value) Arguments:
NameTypeDescription
prompt_version_idstrThe ID of the prompt version to use
context_to_evaluateOptional[str]The context to evaluate for the prompt (variable name essentially).
Returns:
NameDescription
[TestRunBuilder](/sdk/python/references/test_runs/test_run_builder)[T]The current TestRunBuilder instance for method chaining
Raises:
  • ValueError - If a workflow ID, prompt chain version ID or output function is already set for this run builder

​
with_prompt_chain_version_id

def with_prompt_chain_version_id(
        prompt_chain_version_id: str,
        context_to_evaluate: Optional[str] = None) -> "TestRunBuilder[T]"
Set the prompt chain version ID for the test run. Optionally, you can also set the context to evaluate for the prompt chain. (Note: setting the context to evaluate will end up overriding the CONTEXT_TO_EVALUATE dataset column value) Arguments:
NameTypeDescription
prompt_chain_version_idstrThe ID of the prompt chain version to use
context_to_evaluateOptional[str]The context to evaluate for the prompt chain (variable name essentially).
Returns:
NameDescription
[TestRunBuilder](/sdk/python/references/test_runs/test_run_builder)[T]The current TestRunBuilder instance for method chaining
Raises:
  • ValueError - If a workflow ID, prompt version ID or output function is already set for this run builder

​
yields_output

def yields_output(
    output_function: Callable[[LocalData], Union[YieldedOutput,
                                                 Awaitable[YieldedOutput]]]
) -> "TestRunBuilder[T]"
Set the output function for the test run Arguments:
NameTypeDescription
output_functionCallable[[T], Union[[YieldedOutput](/sdk/python/references/models/test_run), Awaitable[YieldedOutput]]]The output function to use
Returns:
NameDescription
[TestRunBuilder](/sdk/python/references/test_runs/test_run_builder)[T]The current TestRunBuilder instance for method chaining
Raises:
  • ValueError - If a workflow ID, prompt chain version ID or prompt version ID is already set for this run builder

​
with_concurrency

def with_concurrency(concurrency: int) -> "TestRunBuilder[T]"
Set the concurrency level for the test run Arguments:
NameTypeDescription
concurrencyintThe concurrency level to use
Returns:
NameDescription
[TestRunBuilder](/sdk/python/references/test_runs/test_run_builder)[T]The current TestRunBuilder instance for method chaining

​
with_logger

def with_logger(logger: TestRunLogger) -> "TestRunBuilder[T]"
Set the logger for the test run Arguments:
NameTypeDescription
loggerTestRunLoggerThe logger to use
Returns:
NameDescription
[TestRunBuilder](/sdk/python/references/test_runs/test_run_builder)[T]The current TestRunBuilder instance for method chaining

​
run

def run(timeout_in_minutes: Optional[int] = 10) -> Optional[RunResult]
Run the test Arguments:
NameTypeDescription
timeout_in_minutesOptional[int]The timeout in minutes. Defaults to 10.
Returns:
NameDescription
[RunResult](/sdk/python/references/models/test_run)The result of the test run