diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 2ec662eded..9b43054f34 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -501,6 +501,10 @@ class EvaluationPrompt(_common.BaseModel): prompt_template_data: Optional[PromptTemplateData] = Field( default=None, description="""Prompt template data.""" ) + user_scenario: Optional[evals_types.UserScenario] = Field( + default=None, + description="""User scenario to help simulate multi-turn agent run results.""", + ) class EvaluationPromptDict(TypedDict, total=False): @@ -515,6 +519,9 @@ class EvaluationPromptDict(TypedDict, total=False): prompt_template_data: Optional[PromptTemplateDataDict] """Prompt template data.""" + user_scenario: Optional[evals_types.UserScenario] + """User scenario to help simulate multi-turn agent run results.""" + EvaluationPromptOrDict = Union[EvaluationPrompt, EvaluationPromptDict] @@ -1843,6 +1850,12 @@ class EvaluationRunInferenceConfig(_common.BaseModel): default=None, description="""The fully qualified name of the publisher model or endpoint to use for inference.""", ) + user_simulator_config: Optional[evals_types.UserSimulatorConfig] = Field( + default=None, + description="""Used for multi-turn agent run. + Contains configuration for a user simulator that + uses an LLM to generate messages on behalf of the user.""", + ) class EvaluationRunInferenceConfigDict(TypedDict, total=False): @@ -1857,6 +1870,11 @@ class EvaluationRunInferenceConfigDict(TypedDict, total=False): model: Optional[str] """The fully qualified name of the publisher model or endpoint to use for inference.""" + user_simulator_config: Optional[evals_types.UserSimulatorConfig] + """Used for multi-turn agent run. + Contains configuration for a user simulator that + uses an LLM to generate messages on behalf of the user.""" + EvaluationRunInferenceConfigOrDict = Union[ EvaluationRunInferenceConfig, EvaluationRunInferenceConfigDict @@ -13546,6 +13564,11 @@ class EvalRunInferenceConfig(_common.BaseModel): generate_content_config: Optional[genai_types.GenerateContentConfig] = Field( default=None, description="""The config for the generate content call.""" ) + user_simulator_config: Optional[evals_types.UserSimulatorConfig] = Field( + default=None, + description="""Configuration for user simulation in multi-turn agent scraping. If provided, and the dataset contains + conversation plans, user simulation will be triggered.""", + ) class EvalRunInferenceConfigDict(TypedDict, total=False): @@ -13560,6 +13583,10 @@ class EvalRunInferenceConfigDict(TypedDict, total=False): generate_content_config: Optional[genai_types.GenerateContentConfigDict] """The config for the generate content call.""" + user_simulator_config: Optional[evals_types.UserSimulatorConfig] + """Configuration for user simulation in multi-turn agent scraping. If provided, and the dataset contains + conversation plans, user simulation will be triggered.""" + EvalRunInferenceConfigOrDict = Union[EvalRunInferenceConfig, EvalRunInferenceConfigDict] diff --git a/vertexai/_genai/types/evals.py b/vertexai/_genai/types/evals.py index 4384150674..970286cd1d 100644 --- a/vertexai/_genai/types/evals.py +++ b/vertexai/_genai/types/evals.py @@ -768,3 +768,67 @@ class SessionInputDict(TypedDict, total=False): SessionInputOrDict = Union[SessionInput, SessionInputDict] + + +class UserScenario(_common.BaseModel): + """User scenario to help simulate multi-turn agent run results.""" + + starting_prompt: Optional[str] = Field( + default=None, + description="""The prompt that starts the conversation between the simulated user and the agent under test.""", + ) + conversation_plan: Optional[str] = Field( + default=None, + description="""The plan for the conversation, used to drive the multi-turn agent run and generate the simulated agent evaluation dataset.""", + ) + + +class UserScenarioDict(TypedDict, total=False): + """User scenario to help simulate multi-turn agent run results.""" + + starting_prompt: Optional[str] + """The prompt that starts the conversation between the simulated user and the agent under test.""" + + conversation_plan: Optional[str] + """The plan for the conversation, used to drive the multi-turn agent run and generate the simulated agent evaluation dataset.""" + + +UserScenarioOrDict = Union[UserScenario, UserScenarioDict] + + +class UserSimulatorConfig(_common.BaseModel): + """Configuration for a user simulator that uses an LLM to generate messages.""" + + model_name: Optional[str] = Field( + default=None, + description="""The model name to get next user message for multi-turn agent run.""", + ) + model_configuration: Optional[genai_types.GenerateContentConfig] = Field( + default=None, description="""The configuration for the model.""" + ) + max_turn: Optional[int] = Field( + default=None, + description="""Maximum number of invocations allowed by the multi-turn agent + running. This property allows us to stop a run-off conversation + where the agent and the user simulator get into a never ending loop. + The initial fixed prompt is also counted as an invocation.""", + ) + + +class UserSimulatorConfigDict(TypedDict, total=False): + """Configuration for a user simulator that uses an LLM to generate messages.""" + + model_name: Optional[str] + """The model name to get next user message for multi-turn agent run.""" + + model_configuration: Optional[genai_types.GenerateContentConfigDict] + """The configuration for the model.""" + + max_turn: Optional[int] + """Maximum number of invocations allowed by the multi-turn agent + running. This property allows us to stop a run-off conversation + where the agent and the user simulator get into a never ending loop. + The initial fixed prompt is also counted as an invocation.""" + + +UserSimulatorConfigOrDict = Union[UserSimulatorConfig, UserSimulatorConfigDict]