From f05e8f494c0a15626ea5afd353d28ee9abdd1ebc Mon Sep 17 00:00:00 2001 From: ftnext Date: Sun, 8 Feb 2026 15:01:24 +0900 Subject: [PATCH 1/2] feat(cli): auto-discover test_config.json for single eval file in adk eval When `--config_file_path` is omitted, `adk eval` now auto-discovers `test_config.json` only for single eval-file input by looking in the same directory as the eval file. If no adjacent `test_config.json` is found, behavior remains unchanged and the built-in default evaluation criteria are used. --- src/google/adk/cli/cli_tools_click.py | 35 +++++++++++++++-- .../cli/utils/test_cli_tools_click.py | 38 +++++++++++++++++++ 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/src/google/adk/cli/cli_tools_click.py b/src/google/adk/cli/cli_tools_click.py index e4fb70c70c..9f597dd5e8 100644 --- a/src/google/adk/cli/cli_tools_click.py +++ b/src/google/adk/cli/cli_tools_click.py @@ -661,6 +661,30 @@ def wrapper(*args, **kwargs): return decorator +def _resolve_eval_config_file_path( + config_file_path: Optional[str], + eval_set_file_or_id_to_evals: dict[str, list[str]], +) -> Optional[str]: + """Returns config file path for eval command. + + If `config_file_path` is provided, it is used as-is. If omitted and evals are + loaded from files, this returns `/test_config.json` for the + first eval set file. Otherwise, returns None. + """ + if config_file_path: + return config_file_path + + if not eval_set_file_or_id_to_evals: + return None + + first_eval_set = next(iter(eval_set_file_or_id_to_evals)) + if os.path.exists(first_eval_set): + eval_set_dir = os.path.dirname(first_eval_set) + return os.path.join(eval_set_dir, "test_config.json") + + return None + + @main.command("eval", cls=HelpfulCommand) @feature_options() @click.argument( @@ -770,10 +794,6 @@ def cli_eval( except ModuleNotFoundError as mnf: raise click.ClickException(MISSING_EVAL_DEPENDENCIES_MESSAGE) from mnf - eval_config = get_evaluation_criteria_or_default(config_file_path) - print(f"Using evaluation criteria: {eval_config}") - eval_metrics = get_eval_metrics_from_config(eval_config) - root_agent = get_root_agent(agent_module_file_path) app_name = os.path.basename(agent_module_file_path) agents_dir = os.path.dirname(agent_module_file_path) @@ -793,6 +813,13 @@ def cli_eval( eval_set_file_or_id_to_evals = parse_and_get_evals_to_run( eval_set_file_path_or_id ) + resolved_config_file_path = _resolve_eval_config_file_path( + config_file_path=config_file_path, + eval_set_file_or_id_to_evals=eval_set_file_or_id_to_evals, + ) + eval_config = get_evaluation_criteria_or_default(resolved_config_file_path) + print(f"Using evaluation criteria: {eval_config}") + eval_metrics = get_eval_metrics_from_config(eval_config) # Check if the first entry is a file that exists, if it does then we assume # rest of the entries are also files. We enforce this assumption in the if diff --git a/tests/unittests/cli/utils/test_cli_tools_click.py b/tests/unittests/cli/utils/test_cli_tools_click.py index 5977f90dd7..1fb616a688 100644 --- a/tests/unittests/cli/utils/test_cli_tools_click.py +++ b/tests/unittests/cli/utils/test_cli_tools_click.py @@ -110,6 +110,44 @@ def test_validate_exclusive_blocks_multiple() -> None: cli_tools_click.validate_exclusive(ctx, param2, "resume.json") +def test_resolve_eval_config_file_path_prefers_explicit_path( + tmp_path: Path, +) -> None: + eval_set_file = tmp_path / "sample.test.json" + eval_set_file.touch() + explicit_config = tmp_path / "explicit_config.json" + + resolved_path = cli_tools_click._resolve_eval_config_file_path( + config_file_path=str(explicit_config), + eval_set_file_or_id_to_evals={str(eval_set_file): []}, + ) + + assert resolved_path == str(explicit_config) + + +def test_resolve_eval_config_file_path_uses_test_config_next_to_eval_file( + tmp_path: Path, +) -> None: + eval_set_file = tmp_path / "sample.test.json" + eval_set_file.touch() + + resolved_path = cli_tools_click._resolve_eval_config_file_path( + config_file_path=None, + eval_set_file_or_id_to_evals={str(eval_set_file): []}, + ) + + assert resolved_path == str(tmp_path / "test_config.json") + + +def test_resolve_eval_config_file_path_returns_none_for_eval_set_id() -> None: + resolved_path = cli_tools_click._resolve_eval_config_file_path( + config_file_path=None, + eval_set_file_or_id_to_evals={"eval_set_id": []}, + ) + + assert resolved_path is None + + # cli create def test_cli_create_cmd_invokes_run_cmd( tmp_path: Path, monkeypatch: pytest.MonkeyPatch From f34a1aac4e53c4605331a6fdfde781b30dc1a09b Mon Sep 17 00:00:00 2001 From: ftnext Date: Sun, 8 Feb 2026 15:07:53 +0900 Subject: [PATCH 2/2] feat(cli): limit test_config auto-discovery to single eval file input --- src/google/adk/cli/cli_tools_click.py | 7 +++++-- .../cli/utils/test_cli_tools_click.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/google/adk/cli/cli_tools_click.py b/src/google/adk/cli/cli_tools_click.py index 9f597dd5e8..4bcfc4445c 100644 --- a/src/google/adk/cli/cli_tools_click.py +++ b/src/google/adk/cli/cli_tools_click.py @@ -668,8 +668,8 @@ def _resolve_eval_config_file_path( """Returns config file path for eval command. If `config_file_path` is provided, it is used as-is. If omitted and evals are - loaded from files, this returns `/test_config.json` for the - first eval set file. Otherwise, returns None. + loaded from a single file, this returns + `/test_config.json`. Otherwise, returns None. """ if config_file_path: return config_file_path @@ -677,6 +677,9 @@ def _resolve_eval_config_file_path( if not eval_set_file_or_id_to_evals: return None + if len(eval_set_file_or_id_to_evals) != 1: + return None + first_eval_set = next(iter(eval_set_file_or_id_to_evals)) if os.path.exists(first_eval_set): eval_set_dir = os.path.dirname(first_eval_set) diff --git a/tests/unittests/cli/utils/test_cli_tools_click.py b/tests/unittests/cli/utils/test_cli_tools_click.py index 1fb616a688..e4e6e9527f 100644 --- a/tests/unittests/cli/utils/test_cli_tools_click.py +++ b/tests/unittests/cli/utils/test_cli_tools_click.py @@ -148,6 +148,25 @@ def test_resolve_eval_config_file_path_returns_none_for_eval_set_id() -> None: assert resolved_path is None +def test_resolve_eval_config_file_path_returns_none_for_multiple_eval_files( + tmp_path: Path, +) -> None: + eval_set_file_1 = tmp_path / "sample_1.test.json" + eval_set_file_2 = tmp_path / "sample_2.test.json" + eval_set_file_1.touch() + eval_set_file_2.touch() + + resolved_path = cli_tools_click._resolve_eval_config_file_path( + config_file_path=None, + eval_set_file_or_id_to_evals={ + str(eval_set_file_1): [], + str(eval_set_file_2): [], + }, + ) + + assert resolved_path is None + + # cli create def test_cli_create_cmd_invokes_run_cmd( tmp_path: Path, monkeypatch: pytest.MonkeyPatch