233 lines
12 KiB
Python
233 lines
12 KiB
Python
"""Fresh remediation verification uses only nominal language and history contracts."""
|
|
|
|
from band_aider.judgement.test_runner_strategy import TestRunOutcome
|
|
from dataclasses import replace
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import Sequence
|
|
|
|
import pytest
|
|
|
|
from band_aider.context import provenance
|
|
from band_aider.context.edited_test_inventory import EditedTestInventory
|
|
from band_aider.context.failing_test import FailingTest
|
|
from band_aider.context.validation_receipts import ValidationReceipt, ValidationReceipts
|
|
from band_aider.context.provenance import DeadEndError, Direction, Phase, Provenance
|
|
from band_aider.context.working_backlog import FailedQAItem, InspectionBacklog, TestingBacklog as BetaWork
|
|
from band_aider.judgement.edit_request import AiderEditRequest
|
|
from band_aider.judgement.remediation_check import RemediationCheck
|
|
from band_aider.judgement.test_runner_strategy import TestRunResult
|
|
from band_aider.strategies.compiler import CompileAttempt, CompileError, Compiler, CompilerFactory, RemediationCompiler
|
|
from band_aider.strategies.language_strategy_set import LanguageStrategySet
|
|
from tests.band_aider_loop_impl import _file_manager
|
|
|
|
|
|
class _ForeignCompiler(RemediationCompiler):
|
|
"""Provide a non-Python nominal compiler and observe inventory teardown.
|
|
|
|
``compiler = _ForeignCompiler(result)`` binds an independent execution witness.
|
|
"""
|
|
|
|
def __init__(self, result: TestRunResult | CompileError) -> None:
|
|
"""Bind one execution result without syntax checks or filesystem access."""
|
|
|
|
self.result = result
|
|
self.closed = False
|
|
self.calls = 0
|
|
self.inventory = EditedTestInventory(self._close)
|
|
|
|
def _close(self) -> None:
|
|
"""Expose the semantic teardown event to the test."""
|
|
|
|
self.closed = True
|
|
|
|
def compile(self, paths: Sequence[Path]) -> list[CompileError]:
|
|
"""Reject any attempt to substitute whole compilation for test execution."""
|
|
|
|
raise AssertionError("Remediation verification must invoke recheck, not compile")
|
|
|
|
def recheck(self) -> TestRunResult | CompileError:
|
|
"""Record actual use of the language-neutral execution contract."""
|
|
|
|
self.calls += 1
|
|
return self.result
|
|
|
|
def get_blackboard(self) -> EditedTestInventory:
|
|
"""Return the inventory owned by this temporary compiler."""
|
|
|
|
return self.inventory
|
|
|
|
|
|
class _Factory(CompilerFactory):
|
|
"""Supply a nominal compiler without discovering or importing a language.
|
|
|
|
``factory.build(attempt)`` records the actual abstract-factory input.
|
|
"""
|
|
|
|
def __init__(self, compiler: _ForeignCompiler) -> None:
|
|
"""Retain the compiler and received attempt for contract assertions."""
|
|
|
|
self.compiler = compiler
|
|
self.attempts: list[CompileAttempt] = []
|
|
|
|
def build(self, attempt: CompileAttempt) -> Compiler:
|
|
"""Bind the request through the existing creation seam."""
|
|
|
|
self.attempts.append(attempt)
|
|
return self.compiler
|
|
|
|
|
|
def test_current_named_receipt_reuses_execution_without_rechecking(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A covering current execution supersedes the old display without another run."""
|
|
manager = _file_manager(Path("/testbed"))
|
|
test = manager.require_test("tests/test_foo.py")
|
|
result = TestRunResult("foreign runner Foo.test_foo", TestRunOutcome.Success, 0, "passed current", test_name="Foo.test_foo")
|
|
receipt = ValidationReceipt.from_result(test, result, test_definition="test definition", generation=1)
|
|
compiler = _ForeignCompiler(CompileError("must not recheck"))
|
|
monkeypatch.setattr(LanguageStrategySet, "instance", lambda *_args, **_kwargs: LanguageStrategySet({CompilerFactory: _Factory(compiler)}))
|
|
request = AiderEditRequest("repair", [], [], is_remediation=True, remediation_originator_test=test,
|
|
failure_test_location=test.location, failure_test_name="Foo.test_foo", failure_error_message="old failure",
|
|
validation_receipts=ValidationReceipts((receipt,), generation=1))
|
|
|
|
assert RemediationCheck(request, manager, logging.getLogger(__name__))()
|
|
assert compiler.calls == 0
|
|
assert request.validation_view().failure_error_message == ""
|
|
assert request.failure_error_message == "old failure"
|
|
|
|
|
|
def test_fresh_pass_without_history_uses_factory_and_releases_inventory(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A passing test needs no duplicate provenance and must not commit borrowed source."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = manager.require_src("src/foo.py")
|
|
test = manager.require_test("tests/test_foo.py")
|
|
memento = source.set_or_get_memento(manager)
|
|
original = memento.pre
|
|
manager.repo_exec.write_text(memento.file_path, "changed candidate\n")
|
|
request = AiderEditRequest(
|
|
authoritative_prompt="repair", editable_files=[source.resolve_path(manager)],
|
|
readonly_files=[test.resolve_path(manager)], is_remediation=True,
|
|
remediation_originator_test=test, failure_error_message="old error",
|
|
failure_stack_trace=["tests/test_foo.py:2"],
|
|
)
|
|
compiler = _ForeignCompiler(TestRunResult("foreign test runner", TestRunOutcome.Success, 0, "passed"))
|
|
factory = _Factory(compiler)
|
|
monkeypatch.setattr(LanguageStrategySet, "instance", lambda *_args, **_kwargs: LanguageStrategySet({CompilerFactory: factory}))
|
|
|
|
assert RemediationCheck(request, manager, logging.getLogger(__name__))()
|
|
assert compiler.calls == 1
|
|
assert compiler.closed
|
|
assert factory.attempts[0].request is request
|
|
assert factory.attempts[0].candidate_mementos == ()
|
|
assert memento.pre == original
|
|
assert manager.repo_exec.read_text(memento.file_path) == "changed candidate\n"
|
|
assert manager.repo_exec.exists(memento.path)
|
|
|
|
|
|
@pytest.mark.parametrize("result", (
|
|
TestRunResult("container test", TestRunOutcome.Failure, 1, "different duration", ["outer", "inner"]),
|
|
CompileError("Could not establish fresh test execution"),
|
|
))
|
|
def test_same_failure_or_diagnostic_keeps_existing_judgment(monkeypatch: pytest.MonkeyPatch, result: TestRunResult | CompileError) -> None:
|
|
"""Transport/raw-output movement and compiler diagnostics cannot redirect repair."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = manager.require_src("src/foo.py")
|
|
test = manager.require_test("tests/test_foo.py")
|
|
request = AiderEditRequest(
|
|
authoritative_prompt="repair", editable_files=[source.resolve_path(manager)], readonly_files=[test.resolve_path(manager)],
|
|
is_remediation=True, remediation_originator_test=test,
|
|
failure_error_message="Test command failed with return code 1.", failure_stack_trace=["outer", "inner"],
|
|
failure_raw_output="old duration",
|
|
)
|
|
compiler = _ForeignCompiler(result)
|
|
monkeypatch.setattr(LanguageStrategySet, "instance", lambda *_args, **_kwargs: LanguageStrategySet({CompilerFactory: _Factory(compiler)}))
|
|
assert not RemediationCheck(request, manager, logging.getLogger(__name__))()
|
|
assert compiler.closed
|
|
assert request.failure_raw_output == "old duration"
|
|
|
|
|
|
@pytest.mark.parametrize("history", (False, True))
|
|
def test_stale_failure_uses_only_admitted_historical_rollforward(
|
|
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, history: bool,
|
|
) -> None:
|
|
"""One-character drift revives two parents with complete guidance or stays local."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = manager.require_src("src/foo.py")
|
|
test = manager.require_test("tests/test_foo.py")
|
|
old = FailingTest(test, ["outer", "inner"], "Test command failed with return code 1.", "old raw λ\n")
|
|
request = AiderEditRequest(
|
|
authoritative_prompt="repair", editable_files=[source.resolve_path(manager)], readonly_files=[test.resolve_path(manager)],
|
|
is_remediation=True, remediation_originator_test=test,
|
|
failure_error_message=old.error_message, failure_stack_trace=list(old.stack_trace), failure_raw_output=old.raw_output,
|
|
)
|
|
result = TestRunResult("foreign test", TestRunOutcome.Failure, 1, "new raw λ\n", ["outer", "inner!"])
|
|
compiler = _ForeignCompiler(result)
|
|
monkeypatch.setattr(LanguageStrategySet, "instance", lambda *_args, **_kwargs: LanguageStrategySet({CompilerFactory: _Factory(compiler)}))
|
|
graph = Provenance.open(repo_root=manager.repo_root, repo_exec=manager.repo_exec)
|
|
item = FailedQAItem(source, "current repair", failed_test=old)
|
|
# The two-parent target is deliberately Alpha, proving this is not an implicit Beta handoff.
|
|
if history:
|
|
root = graph.start(Phase.ALPHA, InspectionBacklog(source, "original inspection"))
|
|
parent = graph.track_provenance(root, Phase.BETA, BetaWork(test, "test"))
|
|
graph.track_provenance(parent, Phase.ALPHA, item)
|
|
else:
|
|
graph.start(Phase.ALPHA, item)
|
|
with provenance.bind(graph), provenance.item_scope(Phase.ALPHA, item), caplog.at_level(logging.WARNING):
|
|
# Only retained historical routes may escape the mechanical check.
|
|
if history:
|
|
with pytest.raises(DeadEndError) as caught:
|
|
RemediationCheck(request, manager, logging.getLogger(__name__))()
|
|
error = caught.value
|
|
assert error.direction is Direction.ROLLFORWARD
|
|
assert old.render() in error.message
|
|
assert FailingTest.from_result(test, result).render() in error.message
|
|
assert "rolls back" in error.justification
|
|
phase, revived = graph.revive(error)
|
|
assert phase is Phase.ALPHA
|
|
assert revived.summary == "original inspection"
|
|
assert revived.future_guidance[-1].message == error.message
|
|
else:
|
|
assert not RemediationCheck(request, manager, logging.getLogger(__name__))()
|
|
assert "no admitted ROLLFORWARD" in caplog.text
|
|
assert "Stale remediation failure" in caplog.text
|
|
assert compiler.calls == 1
|
|
assert compiler.closed
|
|
|
|
|
|
@pytest.mark.parametrize("missing", ({"remediation_originator_test": None}, {"failure_error_message": ""}, {"failure_stack_trace": []}))
|
|
def test_missing_evidence_cannot_claim_a_pass(monkeypatch: pytest.MonkeyPatch, missing: dict) -> None:
|
|
"""Incomplete remediation remains in correction before any language execution."""
|
|
|
|
manager = _file_manager(Path("/testbed"))
|
|
source = manager.require_src("src/foo.py")
|
|
test = manager.require_test("tests/test_foo.py")
|
|
request = AiderEditRequest(
|
|
authoritative_prompt="repair", editable_files=[source.resolve_path(manager)], readonly_files=[test.resolve_path(manager)],
|
|
is_remediation=True, remediation_originator_test=test, failure_error_message="error", failure_stack_trace=["frame"],
|
|
)
|
|
compiler = _ForeignCompiler(TestRunResult("foreign test", TestRunOutcome.Success, 0, "passed"))
|
|
factory = _Factory(compiler)
|
|
monkeypatch.setattr(LanguageStrategySet, "instance", lambda *_args, **_kwargs: LanguageStrategySet({CompilerFactory: factory}))
|
|
assert not RemediationCheck(replace(request, **missing), manager, logging.getLogger(__name__))()
|
|
assert factory.attempts == []
|
|
|
|
|
|
@pytest.mark.parametrize("outcome,code", ((TestRunOutcome.Timeout, 124), (TestRunOutcome.Success, 0)))
|
|
def test_timeout_gate_reuses_current_error_scope_receipt(monkeypatch: pytest.MonkeyPatch, outcome, code) -> None:
|
|
"""Error-scope execution governs timeout discharge without a second invocation."""
|
|
manager = _file_manager(Path("/testbed"))
|
|
test = manager.require_test("tests/test_foo.py")
|
|
result = TestRunResult("pytest file", outcome, code, "current output", run_info="current run details")
|
|
receipt = ValidationReceipt.from_result(test, result, test_definition="source", generation=1, file_scope=True)
|
|
compiler = _ForeignCompiler(CompileError("must not recheck"))
|
|
monkeypatch.setattr(LanguageStrategySet, "instance", lambda *_args, **_kwargs: LanguageStrategySet({CompilerFactory: _Factory(compiler)}))
|
|
request = AiderEditRequest("repair", [], [], is_remediation=True, remediation_originator_test=test,
|
|
failure_test_location=test.location, failure_outcome=TestRunOutcome.Timeout,
|
|
failure_error_message="TIMEOUT", failure_run_info="prior run details",
|
|
validation_receipts=ValidationReceipts((receipt,), generation=1))
|
|
|
|
assert RemediationCheck(request, manager, logging.getLogger(__name__))() is (code == 0)
|
|
assert compiler.calls == 0
|
|
assert request.validation_view().failure_run_info == ("" if code == 0 else "current run details")
|