205 lines
7.9 KiB
Python
205 lines
7.9 KiB
Python
# SPDX-FileCopyrightText: 2026 Art Sea LLC
|
|
#
|
|
# SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from band_aider.ai_client.completion_client import CompletionClient, OpenAIConfig
|
|
from band_aider.judgement.broad_or_narrow_check import (
|
|
BroadOrNarrowCheck,
|
|
BroadOrNarrowCheckFactory,
|
|
)
|
|
from band_aider.judgement.prompt_formatter import MarkdownPromptFormatterBuilder
|
|
|
|
|
|
class _Client(CompletionClient):
|
|
"""_Client returns one configured JSON payload."""
|
|
|
|
def __init__(self, payload: dict[str, object] | BaseException | list[dict[str, object]]) -> None:
|
|
"""Bind an explicit classification response or correction sequence."""
|
|
self.payload = payload
|
|
self.prompts: list[str] = []
|
|
|
|
@property
|
|
def inner(self) -> CompletionClient | None:
|
|
return None
|
|
|
|
def complete(self, prompt: str, logger=None) -> str:
|
|
"""Record correction prompts and return the next declared classification."""
|
|
self.prompts.append(prompt)
|
|
if isinstance(self.payload, BaseException):
|
|
raise self.payload
|
|
if isinstance(self.payload, list):
|
|
return json.dumps(self.payload.pop(0))
|
|
return json.dumps(self.payload)
|
|
|
|
@property
|
|
def openai_configs(self) -> tuple[OpenAIConfig, ...]:
|
|
return ()
|
|
|
|
|
|
def test_broad_or_narrow_check_marks_broad_requests_true() -> None:
|
|
"""Verify broad requests return True and retain the model reason."""
|
|
|
|
client = _Client(
|
|
{
|
|
"scope": "broad",
|
|
"broad_request": True,
|
|
"strict_request": False,
|
|
"anchors": [],
|
|
"reason": "Architecture cleanup is vague and does not name symbols.",
|
|
}
|
|
)
|
|
check = BroadOrNarrowCheck(changes=[], prompt="Clean up the architecture.", client=client, prompt_formatter_builder=MarkdownPromptFormatterBuilder())
|
|
|
|
assert check() is True
|
|
assert check.policy.scope == "broad"
|
|
assert check.policy.anchors == ()
|
|
assert check.reason == "Architecture cleanup is vague and does not name symbols."
|
|
assert "Examples of broad requests" in client.prompts[0]
|
|
assert client.prompts[0].index("Strict scope:") < client.prompts[0].index("Narrow scope:")
|
|
assert client.prompts[0].index("Narrow scope:") < client.prompts[0].index("Broad scope:")
|
|
|
|
|
|
def test_broad_or_narrow_check_marks_narrow_requests_false() -> None:
|
|
"""Verify narrow spec-driven requests return False."""
|
|
|
|
client = _Client(
|
|
{
|
|
"scope": "narrow",
|
|
"broad_request": False,
|
|
"strict_request": False,
|
|
"anchors": [],
|
|
"reason": "The request names a concrete method and behavior.",
|
|
}
|
|
)
|
|
check = BroadOrNarrowCheck(
|
|
changes=[],
|
|
prompt="Update FileParser.parse_stream to skip empty files.",
|
|
client=client,
|
|
prompt_formatter_builder=MarkdownPromptFormatterBuilder(),
|
|
)
|
|
|
|
assert check() is False
|
|
assert check.policy.scope == "narrow"
|
|
assert check.reason == "The request names a concrete method and behavior."
|
|
|
|
|
|
def test_broad_or_narrow_check_marks_strict_requests_false_with_anchors() -> None:
|
|
"""Verify strict requests return False while retaining concrete anchors."""
|
|
|
|
client = _Client(
|
|
{
|
|
"scope": "strict",
|
|
"broad_request": False,
|
|
"strict_request": True,
|
|
"reason": "The request explicitly identifies concrete implementation anchors.",
|
|
"anchors": ["cli.py", "terminal-ui", "DEBUG", "pydevd"],
|
|
}
|
|
)
|
|
check = BroadOrNarrowCheck(
|
|
changes=[],
|
|
prompt="modify cli.py to not use ui regardless of terminal-ui flagging if debug is set because pydevd does not work well in the subprocess",
|
|
client=client,
|
|
prompt_formatter_builder=MarkdownPromptFormatterBuilder(),
|
|
)
|
|
|
|
assert check() is False
|
|
assert check.policy.scope == "strict"
|
|
assert check.policy.anchors == ("cli.py", "terminal-ui", "DEBUG", "pydevd")
|
|
|
|
|
|
def test_broad_or_narrow_check_corrects_malformed_and_deduplicates_valid_anchors() -> None:
|
|
"""Malformed anchors need a new judgement before valid duplicates can normalize."""
|
|
|
|
client = _Client(
|
|
[{
|
|
"scope": "strict",
|
|
"broad_request": False,
|
|
"strict_request": True,
|
|
"reason": "Anchored.",
|
|
"anchors": ["cli.py", "", "cli.py", 1, "DEBUG", "ui", "py"],
|
|
}, {"scope": "strict", "broad_request": False, "strict_request": True,
|
|
"reason": "Anchored.", "anchors": ["cli.py", "cli.py", "DEBUG"]}]
|
|
)
|
|
check = BroadOrNarrowCheck(changes=[], prompt="Fix cli.py DEBUG.", client=client, prompt_formatter_builder=MarkdownPromptFormatterBuilder())
|
|
|
|
check()
|
|
|
|
assert check.policy.scope == "strict"
|
|
assert check.policy.anchors == ("cli.py", "DEBUG")
|
|
assert len(client.prompts) == 2
|
|
assert "anchors[3]" in client.prompts[1]
|
|
|
|
|
|
def test_broad_or_narrow_check_corrects_strict_short_anchors() -> None:
|
|
"""An unusable strict selection needs an explicit corrected broad judgement."""
|
|
|
|
client = _Client([
|
|
{"scope": "strict", "broad_request": False, "strict_request": True, "reason": "Short anchors.", "anchors": ["ui", "py"]},
|
|
{"scope": "broad", "broad_request": True, "strict_request": False, "reason": "No usable anchors.", "anchors": []},
|
|
])
|
|
check = BroadOrNarrowCheck(changes=[], prompt="Fix ui.", client=client, prompt_formatter_builder=MarkdownPromptFormatterBuilder())
|
|
|
|
assert check() is True
|
|
assert check.policy.scope == "broad"
|
|
assert check.policy.anchors == ()
|
|
assert len(client.prompts) == 2
|
|
|
|
|
|
def test_broad_or_narrow_check_corrects_strict_without_anchors() -> None:
|
|
"""Malformed anchor types cannot implicitly authorize a broad search."""
|
|
|
|
client = _Client([
|
|
{"scope": "strict", "broad_request": False, "strict_request": True, "reason": "No valid anchors.", "anchors": [1, ""]},
|
|
{"scope": "broad", "broad_request": True, "strict_request": False, "reason": "No usable anchors.", "anchors": []},
|
|
])
|
|
check = BroadOrNarrowCheck(changes=[], prompt="Fix it.", client=client, prompt_formatter_builder=MarkdownPromptFormatterBuilder())
|
|
|
|
assert check() is True
|
|
assert check.policy.scope == "broad"
|
|
assert check.policy.anchors == ()
|
|
assert len(client.prompts) == 2
|
|
|
|
|
|
def test_broad_or_narrow_check_corrects_unrecognized_scope() -> None:
|
|
"""Unknown enums require a corrected judgement rather than a default decision."""
|
|
|
|
client = _Client([
|
|
{"scope": "extra_narrow", "broad_request": False, "strict_request": False, "reason": "Unknown.", "anchors": ["cli.py"]},
|
|
{"scope": "broad", "broad_request": True, "strict_request": False, "reason": "Broad request.", "anchors": []},
|
|
])
|
|
check = BroadOrNarrowCheck(changes=[], prompt="Fix cli.py.", client=client, prompt_formatter_builder=MarkdownPromptFormatterBuilder())
|
|
|
|
assert check() is True
|
|
assert check.policy.scope == "broad"
|
|
assert check.policy.anchors == ()
|
|
assert len(client.prompts) == 2
|
|
|
|
|
|
def test_broad_or_narrow_check_exceptions_normalize_to_broad_and_log(caplog) -> None:
|
|
"""Verify classifier execution errors default to broad and are logged."""
|
|
|
|
client = _Client(RuntimeError("boom"))
|
|
check = BroadOrNarrowCheck(changes=[], prompt="Fix cli.py.", client=client, prompt_formatter_builder=MarkdownPromptFormatterBuilder())
|
|
|
|
assert check() is True
|
|
assert check.policy.scope == "broad"
|
|
assert "Search policy classification failed closed to broad" in caplog.text
|
|
|
|
|
|
def test_broad_or_narrow_check_factory_builds_prompt_backed_check() -> None:
|
|
"""Verify the factory seeds one check with the supplied goal."""
|
|
|
|
client = _Client({"scope": "narrow", "broad_request": False, "reason": "specific"})
|
|
|
|
check = BroadOrNarrowCheckFactory(
|
|
client,
|
|
MarkdownPromptFormatterBuilder(),
|
|
).create("Add thresholding to NarrowParser at 0.72.")
|
|
|
|
assert isinstance(check, BroadOrNarrowCheck)
|
|
assert check.prompt == "Add thresholding to NarrowParser at 0.72."
|
|
assert check.client is client
|