261 lines
11 KiB
Python
261 lines
11 KiB
Python
# SPDX-FileCopyrightText: 2026 Art Sea LLC
|
|
#
|
|
# SPDX-License-Identifier: AGPL-3.0-only
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import cast
|
|
|
|
import pytest
|
|
|
|
from band_aider.context.class_meta import ClassMeta
|
|
from band_aider.context.memento import Memento
|
|
from band_aider.context.validation_utils import MechanicalValidationError, ValidationError
|
|
from band_aider.judgement.code_based_reasoning import CodeBasedReasoning
|
|
from band_aider.judgement.mechanical_check import MechanicalCheck
|
|
from tests.band_aider_loop_impl import _file_manager
|
|
|
|
|
|
def test_deleted_block_preserves_exact_baseline_and_rechecks_current_absence() -> None:
|
|
"""Deletion evidence survives a changed implementation but becomes invalid on restoration."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = manager.require_src("src/foo.py")
|
|
previous = "def foo():\n\treturn old()\n"
|
|
gate = CodeBasedReasoning(file_manager=manager, files=[source], baseline={source.location: previous})
|
|
citation = "<DELETED>" + previous + "</DELETED>"
|
|
|
|
assert gate([citation, "return 1"])
|
|
assert not gate([citation.replace("\t", " ")])
|
|
assert not gate([citation.replace("\n", "\r\n")])
|
|
assert not gate(["<DELETED>never_existed()</DELETED>"])
|
|
manager.repo_exec.write_text(source.resolve_path(manager), previous)
|
|
assert not gate([citation])
|
|
assert previous in gate.render_baseline()
|
|
|
|
|
|
@pytest.mark.parametrize("citation", [
|
|
"<DELETED></DELETED>", "<DELETED> \n </DELETED>",
|
|
"<DELETED>def foo():", "def foo():</DELETED>",
|
|
" <DELETED>def foo():</DELETED>", "<DELETED><DELETED>def foo():</DELETED></DELETED>",
|
|
])
|
|
def test_deleted_citation_rejects_malformed_tags(citation: str) -> None:
|
|
"""Tag errors cannot bypass either ordinary citations or exact historical verification."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = manager.require_src("src/foo.py")
|
|
gate = CodeBasedReasoning(file_manager=manager, files=[source], baseline={source.location: "def foo():\n"})
|
|
assert not gate([citation])
|
|
|
|
|
|
def test_deleted_citation_cannot_borrow_another_files_baseline() -> None:
|
|
"""Historical attribution stays in the caller's selected file scope."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = manager.require_src("src/foo.py")
|
|
gate = CodeBasedReasoning(file_manager=manager, files=[source], baseline={"src/other.py": "removed()"})
|
|
assert not gate(["<DELETED>removed()</DELETED>"])
|
|
|
|
|
|
def test_failed_live_read_cannot_establish_deleted_evidence() -> None:
|
|
"""A missing file read is not a successful observation of absent source text."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = ClassMeta("src/unavailable.py", "", [], {})
|
|
gate = CodeBasedReasoning(file_manager=manager, files=[source], baseline={source.location: "removed()\n"})
|
|
|
|
assert not gate(["<DELETED>removed()\n</DELETED>"])
|
|
assert "Cannot retrieve live code" in gate.validation.stringify()
|
|
|
|
|
|
def test_current_code_supersedes_snapshot_and_is_read_again() -> None:
|
|
"""The same citation changes validity with live code, independently of its checkpoint."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = manager.require_src("src/foo.py")
|
|
path = source.resolve_path(manager)
|
|
manager.repo_exec.write_text(path, "return memento.capture()\n")
|
|
Memento(path, manager.repo_exec).capture()
|
|
gate = CodeBasedReasoning(file_manager=manager, files=[source])
|
|
|
|
assert isinstance(gate, MechanicalCheck)
|
|
assert gate(["return memento.capture()"])
|
|
manager.repo_exec.write_text(path, "return memento\n")
|
|
assert not gate(["return memento.capture()"])
|
|
assert "reason[0]" in gate.validation.stringify()
|
|
assert gate(["return memento"])
|
|
assert manager.repo_exec.read_text(Memento(path, manager.repo_exec).path) == "return memento.capture()\n"
|
|
|
|
|
|
@pytest.mark.parametrize("reasons", (
|
|
[], [""], [" \t"], ["return"], ["+ return 1"], ["return 1"],
|
|
["return 1\nreturn 2"], ["return 1", "return missing()"],
|
|
))
|
|
def test_all_reasons_must_be_complete_current_code_lines(reasons: list[str]) -> None:
|
|
"""Empty, partial, altered, multiline, or mixed citations cannot prove a verdict."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
gate = CodeBasedReasoning(file_manager=manager, files=[manager.require_src("src/foo.py")])
|
|
|
|
assert not gate(reasons)
|
|
assert not gate.validation.valid
|
|
assert gate.changes == [(gate, False)]
|
|
|
|
|
|
def test_boundary_whitespace_is_ignored_without_rewriting_code() -> None:
|
|
"""Indentation around an exact line is tolerated while the original file is preserved."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
gate = CodeBasedReasoning(file_manager=manager, files=[manager.require_src("src/foo.py")])
|
|
|
|
assert gate(["\treturn 1 ", "def foo():"])
|
|
assert manager.repo_exec.read_text(manager.source_path("src/foo.py")) == "def foo():\n return 1\n"
|
|
|
|
|
|
def test_other_files_cannot_supply_missing_citations() -> None:
|
|
"""A line in an undeclared file does not establish code in the supplied scope."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
manager.repo_exec.write_text(manager.source_path("src/other.py"), "return memento.capture()\n")
|
|
gate = CodeBasedReasoning(file_manager=manager, files=[manager.require_src("src/foo.py")])
|
|
|
|
assert not gate(["return memento.capture()"])
|
|
assert "reason[0]" in gate.validation.stringify()
|
|
assert all(isinstance(error, MechanicalValidationError) for error in gate.validation.errors)
|
|
|
|
|
|
def test_shape_failure_remains_structural_while_code_failure_is_mechanical() -> None:
|
|
"""Malformed response shape and deterministic repository evidence retain distinct validation kinds."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
gate = CodeBasedReasoning(file_manager=manager, files=[manager.require_src("src/foo.py")])
|
|
|
|
assert not gate([1]) # type: ignore[list-item]
|
|
assert all(type(error) is ValidationError for error in gate.validation.errors)
|
|
assert not gate(["missing()"])
|
|
assert all(isinstance(error, MechanicalValidationError) for error in gate.validation.errors)
|
|
|
|
|
|
def test_caller_can_supply_test_code_without_an_alpha_dependency() -> None:
|
|
"""The reusable check follows any supplied ClassMeta's repository location."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
gate = CodeBasedReasoning(file_manager=manager, files=[manager.require_test("tests/test_foo.py")])
|
|
|
|
assert gate(["assert foo() == 1"])
|
|
assert not gate(["return 1"])
|
|
|
|
|
|
@pytest.mark.parametrize("exists", (True, False))
|
|
def test_empty_or_missing_files_cannot_supply_citations(exists: bool) -> None:
|
|
"""Neither empty code nor an absent file provides an acceptance fallback."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = ClassMeta("src/empty.py", "", [], {})
|
|
# Distinguish a present empty file from a location that was never created.
|
|
if exists:
|
|
manager.repo_exec.write_text(source.resolve_path(manager), "")
|
|
gate = CodeBasedReasoning(file_manager=manager, files=[source])
|
|
|
|
assert not gate(["return 1"])
|
|
assert not gate([])
|
|
|
|
|
|
def test_both_citation_buckets_follow_current_source() -> None:
|
|
"""Changing live code invalidates both formerly supported presence and absence claims."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = manager.require_src("src/foo.py")
|
|
gate = CodeBasedReasoning(
|
|
file_manager=manager, files=[source],
|
|
evidence_lines=["> return old()", " return 1"],
|
|
)
|
|
|
|
assert gate(["return 1"], negative_reason=["return old()"])
|
|
assert gate(["return 1"], negative_reason=["> return old()"])
|
|
assert not gate(["return 1"], negative_reason=["return invented()"])
|
|
assert "supplied traceback" in gate.validation.stringify()
|
|
assert not gate(["return 1"], negative_reason=["> return 1"])
|
|
assert "still present" in gate.validation.stringify()
|
|
manager.repo_exec.write_text(source.resolve_path(manager), "return old()\n")
|
|
assert not gate(["return 1"], negative_reason=["return old()"])
|
|
assert "reason[0]" in gate.validation.stringify()
|
|
assert "negative_reason[0]" in gate.validation.stringify()
|
|
assert gate(["return old()"])
|
|
|
|
|
|
def test_traceback_pointer_does_not_hide_live_source_operator() -> None:
|
|
"""A C++ continuation present in live source cannot disappear through traceback normalization."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = manager.require_src("src/foo.py")
|
|
manager.repo_exec.write_text(source.resolve_path(manager), "stream\n >> value;\n")
|
|
gate = CodeBasedReasoning(
|
|
file_manager=manager, files=[source], evidence_lines=[">> value;"],
|
|
)
|
|
|
|
assert not gate(["stream"], negative_reason=[">> value;"])
|
|
assert "negative_reason[0]" in gate.validation.stringify()
|
|
assert "still present in live code" in gate.validation.stringify()
|
|
|
|
|
|
@pytest.mark.parametrize("reason, negative_reason, errors", (
|
|
([], ["return old()"], ("reason:",)),
|
|
(["invented()"], ["return old()"], ("reason[0]",)),
|
|
(["return 1"], ["invented()"], ("negative_reason[0]",)),
|
|
(["return 1"], ["return 1"], ("negative_reason[0]",)),
|
|
(["return 1"], ["", "return old()\nreturn 1"], ("negative_reason[0]", "negative_reason[1]")),
|
|
([1], ["return 1"], ("reason[0]", "negative_reason[0]")),
|
|
(["invented()"], [1], ("reason[0]", "negative_reason[0]")),
|
|
))
|
|
def test_neither_bucket_can_conceal_invalid_citations(
|
|
reason: object, negative_reason: object, errors: tuple[str, ...],
|
|
) -> None:
|
|
"""Missing, malformed, or unsupported evidence is rejected even when the other bucket is valid."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
gate = CodeBasedReasoning(
|
|
file_manager=manager, files=[manager.require_src("src/foo.py")],
|
|
evidence_lines=["return old()", "return 1"],
|
|
)
|
|
|
|
assert not gate(cast(list[str], reason), negative_reason=cast(list[str], negative_reason))
|
|
assert all(map(lambda field: field in gate.validation.stringify(), errors))
|
|
assert gate.changes == [(gate, False)]
|
|
|
|
|
|
@pytest.mark.parametrize("evidence", ([], [""], [" \t"], ["> "]))
|
|
def test_empty_failure_evidence_cannot_prove_negative_citations(evidence: list[str]) -> None:
|
|
"""No failure and blank display markers cannot establish resolved work."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
gate = CodeBasedReasoning(
|
|
file_manager=manager, files=[manager.require_src("src/foo.py")],
|
|
evidence_lines=evidence,
|
|
)
|
|
|
|
assert gate(["return 1"])
|
|
assert gate(["return 1"], negative_reason=[])
|
|
assert not gate(["return 1"], negative_reason=["return old()"])
|
|
assert "negative_reason" in gate.validation.stringify()
|
|
assert not gate(["return 1"], negative_reason=[""])
|
|
assert not gate([], negative_reason=[])
|
|
|
|
|
|
@pytest.mark.parametrize("scope", ("missing", "unresolved", "empty"))
|
|
def test_unreadable_or_unresolved_scope_cannot_prove_absence(scope: str) -> None:
|
|
"""A failure to obtain live source is not a successfully observed absence."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = ClassMeta("src/missing.py", "", [], {})
|
|
# An invalid content domain exercises ClassMeta path resolution rather than file IO.
|
|
if scope == "unresolved":
|
|
source.content_type = None # type: ignore[assignment]
|
|
gate = CodeBasedReasoning(
|
|
file_manager=manager, files=[] if scope == "empty" else [source],
|
|
evidence_lines=["return old()"],
|
|
)
|
|
|
|
assert not gate(["return 1"], negative_reason=["return old()"])
|
|
assert "scope" in gate.validation.stringify() or "Cannot retrieve live code" in gate.validation.stringify()
|