TestRunBuilder

View module source on GitHub

​

TestRunBuilder

@final
class TestRunBuilder(Generic[T])

Builder for test runs.

​

__init__

def __init__(base_url: str, api_key: str, name: str, workspace_id: str,
             evaluators: List[Union[str, BaseEvaluator]])

Constructor

​

with_data_structure

def with_data_structure(data: T) -> "TestRunBuilder[T]"

Set the data structure for the test run

Arguments:

Returns:

​

with_data

def with_data(data: Data) -> "TestRunBuilder[T]"

Set the data for the test run

Arguments:

Returns:

​

with_evaluators

def with_evaluators(
        *evaluators: Union[str, BaseEvaluator]) -> "TestRunBuilder[T]"

Add evaluators to the test run

Arguments:

Returns:

​

with_human_evaluation_config

def with_human_evaluation_config(
        config: HumanEvaluationConfig) -> "TestRunBuilder[T]"

Set the human evaluation configuration for the test run

Arguments:

Returns:

​

with_workflow_id

def with_workflow_id(
        workflow_id: str,
        context_to_evaluate: Optional[str] = None) -> "TestRunBuilder[T]"

Set the workflow ID for the test run. Optionally, you can also set the context to evaluate for the workflow. (Note: setting the context to evaluate will end up overriding the CONTEXT_TO_EVALUATE dataset column value)

Arguments:

Returns:

Raises:

• ValueError - If a prompt version ID, prompt chain version ID or output function is already set for this run builder

​

with_prompt_version_id

def with_prompt_version_id(
        prompt_version_id: str,
        context_to_evaluate: Optional[str] = None) -> "TestRunBuilder[T]"

Set the prompt version ID for the test run. Optionally, you can also set the context to evaluate for the prompt. (Note: setting the context to evaluate will end up overriding the CONTEXT_TO_EVALUATE dataset column value)

Arguments:

Returns:

Raises:

• ValueError - If a workflow ID, prompt chain version ID or output function is already set for this run builder

​

with_prompt_chain_version_id

def with_prompt_chain_version_id(
        prompt_chain_version_id: str,
        context_to_evaluate: Optional[str] = None) -> "TestRunBuilder[T]"

Set the prompt chain version ID for the test run. Optionally, you can also set the context to evaluate for the prompt chain. (Note: setting the context to evaluate will end up overriding the CONTEXT_TO_EVALUATE dataset column value)

Arguments:

Returns:

Raises:

• ValueError - If a workflow ID, prompt version ID or output function is already set for this run builder

​

yields_output

def yields_output(
    output_function: Callable[[LocalData], Union[YieldedOutput,
                                                 Awaitable[YieldedOutput]]]
) -> "TestRunBuilder[T]"

Set the output function for the test run

Arguments:

Returns:

Raises:

• ValueError - If a workflow ID, prompt chain version ID or prompt version ID is already set for this run builder

​

with_concurrency

def with_concurrency(concurrency: int) -> "TestRunBuilder[T]"

Set the concurrency level for the test run

Arguments:

Returns:

​

with_logger

def with_logger(logger: TestRunLogger) -> "TestRunBuilder[T]"

Set the logger for the test run

Arguments:

Returns:

​

run

def run(timeout_in_minutes: Optional[int] = 10) -> Optional[RunResult]

Run the test

Arguments:

Returns: