chore: GenAI Client(evals): Update SDK class to implement multi-turn agent scraping in evals.run_inference function

vertexai/_genai/types/common.py

@@ -501,6 +501,10 @@ class EvaluationPrompt(_common.BaseModel):
     prompt_template_data: Optional[PromptTemplateData] = Field(
         default=None, description="""Prompt template data."""
     )
+    user_scenario: Optional[evals_types.UserScenario] = Field(
+        default=None,
+        description="""User scenario to help simulate multi-turn agent run results.""",
+    )
 
 
 class EvaluationPromptDict(TypedDict, total=False):
@@ -515,6 +519,9 @@ class EvaluationPromptDict(TypedDict, total=False):
     prompt_template_data: Optional[PromptTemplateDataDict]
     """Prompt template data."""
 
+    user_scenario: Optional[evals_types.UserScenario]
+    """User scenario to help simulate multi-turn agent run results."""
+
 
 EvaluationPromptOrDict = Union[EvaluationPrompt, EvaluationPromptDict]
 
@@ -1843,6 +1850,12 @@ class EvaluationRunInferenceConfig(_common.BaseModel):
         default=None,
         description="""The fully qualified name of the publisher model or endpoint to use for inference.""",
     )
+    user_simulator_config: Optional[evals_types.UserSimulatorConfig] = Field(
+        default=None,
+        description="""Used for multi-turn agent run.
+      Contains configuration for a user simulator that
+      uses an LLM to generate messages on behalf of the user.""",
+    )
 
 
 class EvaluationRunInferenceConfigDict(TypedDict, total=False):
@@ -1857,6 +1870,11 @@ class EvaluationRunInferenceConfigDict(TypedDict, total=False):
     model: Optional[str]
     """The fully qualified name of the publisher model or endpoint to use for inference."""
 
+    user_simulator_config: Optional[evals_types.UserSimulatorConfig]
+    """Used for multi-turn agent run.
+      Contains configuration for a user simulator that
+      uses an LLM to generate messages on behalf of the user."""
+
 
 EvaluationRunInferenceConfigOrDict = Union[
     EvaluationRunInferenceConfig, EvaluationRunInferenceConfigDict
@@ -13546,6 +13564,11 @@ class EvalRunInferenceConfig(_common.BaseModel):
     generate_content_config: Optional[genai_types.GenerateContentConfig] = Field(
         default=None, description="""The config for the generate content call."""
     )
+    user_simulator_config: Optional[evals_types.UserSimulatorConfig] = Field(
+        default=None,
+        description="""Configuration for user simulation in multi-turn agent scraping. If provided, and the dataset contains
+      conversation plans, user simulation will be triggered.""",
+    )
 
 
 class EvalRunInferenceConfigDict(TypedDict, total=False):
@@ -13560,6 +13583,10 @@ class EvalRunInferenceConfigDict(TypedDict, total=False):
     generate_content_config: Optional[genai_types.GenerateContentConfigDict]
     """The config for the generate content call."""
 
+    user_simulator_config: Optional[evals_types.UserSimulatorConfig]
+    """Configuration for user simulation in multi-turn agent scraping. If provided, and the dataset contains
+      conversation plans, user simulation will be triggered."""
+
 
 EvalRunInferenceConfigOrDict = Union[EvalRunInferenceConfig, EvalRunInferenceConfigDict]
 

vertexai/_genai/types/evals.py

@@ -768,3 +768,67 @@ class SessionInputDict(TypedDict, total=False):
 
 
 SessionInputOrDict = Union[SessionInput, SessionInputDict]
+
+
+class UserScenario(_common.BaseModel):
+    """User scenario to help simulate multi-turn agent run results."""
+
+    starting_prompt: Optional[str] = Field(
+        default=None,
+        description="""The prompt that starts the conversation between the simulated user and the agent under test.""",
+    )
+    conversation_plan: Optional[str] = Field(
+        default=None,
+        description="""The plan for the conversation, used to drive the multi-turn agent run and generate the simulated agent evaluation dataset.""",
+    )
+
+
+class UserScenarioDict(TypedDict, total=False):
+    """User scenario to help simulate multi-turn agent run results."""
+
+    starting_prompt: Optional[str]
+    """The prompt that starts the conversation between the simulated user and the agent under test."""
+
+    conversation_plan: Optional[str]
+    """The plan for the conversation, used to drive the multi-turn agent run and generate the simulated agent evaluation dataset."""
+
+
+UserScenarioOrDict = Union[UserScenario, UserScenarioDict]
+
+
+class UserSimulatorConfig(_common.BaseModel):
+    """Configuration for a user simulator that uses an LLM to generate messages."""
+
+    model_name: Optional[str] = Field(
+        default=None,
+        description="""The model name to get next user message for multi-turn agent run.""",
+    )
+    model_configuration: Optional[genai_types.GenerateContentConfig] = Field(
+        default=None, description="""The configuration for the model."""
+    )
+    max_turn: Optional[int] = Field(
+        default=None,
+        description="""Maximum number of invocations allowed by the multi-turn agent
+      running. This property allows us to stop a run-off conversation
+      where the agent and the user simulator get into a never ending loop.
+      The initial fixed prompt is also counted as an invocation.""",
+    )
+
+
+class UserSimulatorConfigDict(TypedDict, total=False):
+    """Configuration for a user simulator that uses an LLM to generate messages."""
+
+    model_name: Optional[str]
+    """The model name to get next user message for multi-turn agent run."""
+
+    model_configuration: Optional[genai_types.GenerateContentConfigDict]
+    """The configuration for the model."""
+
+    max_turn: Optional[int]
+    """Maximum number of invocations allowed by the multi-turn agent
+      running. This property allows us to stop a run-off conversation
+      where the agent and the user simulator get into a never ending loop.
+      The initial fixed prompt is also counted as an invocation."""
+
+
+UserSimulatorConfigOrDict = Union[UserSimulatorConfig, UserSimulatorConfigDict]