163 lines
9.1 KiB
Python
163 lines
9.1 KiB
Python
"""Execution receipts preserve identity, observations, and candidate freshness."""
|
|
|
|
from band_aider.judgement.test_runner_strategy import TestRunOutcome
|
|
import pytest
|
|
|
|
from band_aider.context.class_meta import ClassMeta, ContentType
|
|
from band_aider.context.failing_test import FailingTest
|
|
from band_aider.context.validation_receipts import ValidationReceipt, ValidationReceipts
|
|
from band_aider.judgement.edit_request import AiderEditRequest
|
|
from band_aider.judgement.test_runner_strategy import TestRunResult
|
|
|
|
|
|
@pytest.mark.parametrize("passed", [True, False])
|
|
def test_receipt_retains_executor_signature_streams_identity_and_definition(passed: bool) -> None:
|
|
"""Success is still evidence, with exact stdout/stderr and no fabricated failure."""
|
|
test = ClassMeta("tests/test_builder.py", "", [], {}, content_type=ContentType.Test)
|
|
command = "python -m pytest tests/test_builder.py::TestBuilder::test_build"
|
|
stdout, stderr = " stdout payload\n\n", "stderr payload\n"
|
|
result = TestRunResult(command, TestRunOutcome.Success if passed else TestRunOutcome.Failure, 0 if passed else 1, stdout + stderr,
|
|
["tests/test_builder.py:7: AssertionError"] if not passed else [],
|
|
stdout=stdout, stderr=stderr, test_name="TestBuilder.test_build")
|
|
definition = " def test_build(self):\n assert builder.build()\n"
|
|
|
|
receipt = ValidationReceipt.from_result(test, result, test_definition=definition, generation=2)
|
|
rendered = ValidationReceipts((receipt,), generation=2).render()
|
|
|
|
assert receipt.test_name == "TestBuilder.test_build"
|
|
assert receipt.test.location == "tests/test_builder.py"
|
|
assert receipt.call_signature == command
|
|
assert receipt.passed is passed
|
|
assert receipt.result.stdout == stdout
|
|
assert receipt.result.stderr == stderr
|
|
assert receipt.test_definition == definition
|
|
assert command in rendered and definition in rendered
|
|
assert stdout in rendered and stderr in rendered
|
|
# Only an unsuccessful invocation supplies failure-owned evidence.
|
|
if passed:
|
|
assert receipt.failure is None
|
|
assert "PASS" in rendered
|
|
else:
|
|
assert receipt.failure is not None
|
|
assert receipt.failure.test_name == "TestBuilder.test_build"
|
|
assert receipt.failure.raw_output == result.output
|
|
assert "FAIL" in rendered
|
|
|
|
|
|
@pytest.mark.parametrize("last_passed", [True, False])
|
|
def test_latest_receipt_replaces_earlier_result_for_the_same_test(last_passed: bool) -> None:
|
|
"""The last retained execution wins in both directions, not just for failures."""
|
|
test = ClassMeta("tests/test_builder.py", "", [], {}, content_type=ContentType.Test)
|
|
first = ValidationReceipt.from_result(test,
|
|
TestRunResult("pytest old", TestRunOutcome.Success if not last_passed else TestRunOutcome.Failure, int(last_passed), "old output", test_name="TestBuilder.test_build"),
|
|
test_definition="old definition", generation=1)
|
|
last = ValidationReceipt.from_result(test,
|
|
TestRunResult("pytest new", TestRunOutcome.Success if last_passed else TestRunOutcome.Failure, int(not last_passed), "new output", test_name="TestBuilder.test_build"),
|
|
test_definition="new definition", generation=2)
|
|
|
|
receipts = ValidationReceipts((first, last), generation=2)
|
|
|
|
assert receipts.receipts == (last,)
|
|
assert receipts.has_current_pass is last_passed
|
|
assert "old output" not in receipts.render()
|
|
|
|
|
|
def test_same_method_name_in_two_files_remains_two_receipts() -> None:
|
|
"""Method names alone cannot identify a test execution."""
|
|
left = ClassMeta("tests/test_a.py", "", [], {}, content_type=ContentType.Test)
|
|
right = ClassMeta("tests/test_b.py", "", [], {}, content_type=ContentType.Test)
|
|
result = TestRunResult("pytest", TestRunOutcome.Success, 0, "passed", test_name="TestShared.test_value")
|
|
a = ValidationReceipt.from_result(left, result, test_definition="left definition", generation=1)
|
|
b = ValidationReceipt.from_result(right, result, test_definition="right definition", generation=1)
|
|
|
|
receipts = ValidationReceipts((a, b), generation=1)
|
|
|
|
assert receipts.receipts == (a, b)
|
|
|
|
|
|
@pytest.mark.parametrize("generation,name,superseded", [(1, "test_value", False), (2, "test_value", True), (2, "", False), (2, "test_other", False)])
|
|
def test_only_current_exact_identity_supersedes_historical_failure(generation: int, name: str, superseded: bool) -> None:
|
|
"""An earlier pass, file-wide result, or another method cannot erase an obligation."""
|
|
test = ClassMeta("tests/test_a.py", "", [], {}, content_type=ContentType.Test)
|
|
receipt = ValidationReceipt.from_result(test,
|
|
TestRunResult("pytest", TestRunOutcome.Success, 0, "passed", test_name=name),
|
|
test_definition="def test_value():\n pass\n", generation=generation)
|
|
failure = FailingTest(test, ["historical trace"], "historical failure", test_name="test_value")
|
|
receipts = ValidationReceipts((receipt,), generation=2)
|
|
request = AiderEditRequest("goal", [], [], failure_test_location=test.location,
|
|
failure_test_name="test_value", failure_error_message=failure.error_message,
|
|
failure_stack_trace=list(failure.stack_trace), validation_receipts=receipts)
|
|
|
|
projected = request.validation_view()
|
|
|
|
assert (receipts.current_for(failure.test.location, failure.test_name) is not None) is superseded
|
|
assert projected.failure_error_message == ("" if superseded else "historical failure")
|
|
assert request.failure_error_message == "historical failure"
|
|
assert receipts.has_current_pass is (generation == 2 and bool(name))
|
|
|
|
|
|
def test_empty_collection_is_not_a_passing_receipt() -> None:
|
|
"""The runner's legacy no-tests success flag does not prove a test executed."""
|
|
test = ClassMeta("tests/test_a.py", "", [], {}, content_type=ContentType.Test)
|
|
receipt = ValidationReceipt.from_result(test,
|
|
TestRunResult("pytest", TestRunOutcome.Success, 5, "no tests ran", test_name="test_value"),
|
|
test_definition="", generation=1)
|
|
|
|
receipts = ValidationReceipts((receipt,), generation=1)
|
|
|
|
assert not receipt.passed
|
|
assert not receipts.has_current_pass
|
|
assert "no tests ran" in receipts.render()
|
|
|
|
|
|
def test_current_failure_replaces_display_not_captured_history() -> None:
|
|
"""Supersession applies to failed executions too, with raw evidence unchanged."""
|
|
test = ClassMeta("tests/test_a.py", "", [], {}, content_type=ContentType.Test)
|
|
receipt = ValidationReceipt.from_result(test,
|
|
TestRunResult("pytest exact", TestRunOutcome.Failure, 1, "new raw", ["new trace"], test_name="test_value"),
|
|
test_definition="def test_value(): pass", generation=2)
|
|
request = AiderEditRequest("repair", [], [], failure_test_location=test.location, failure_test_name="test_value",
|
|
failure_error_message="old failure", failure_stack_trace=["old trace"], failure_raw_output="old raw",
|
|
validation_receipts=ValidationReceipts((receipt,), generation=2))
|
|
|
|
view = request.validation_view()
|
|
|
|
assert view.failure_raw_output == "new raw"
|
|
assert view.failure_stack_trace == ["new trace"]
|
|
assert request.failure_raw_output == "old raw"
|
|
|
|
|
|
def test_receipt_snapshots_mutable_runner_results() -> None:
|
|
"""A later adapter mutation cannot rewrite an execution already observed."""
|
|
test = ClassMeta("tests/test_a.py", "", [], {}, content_type=ContentType.Test)
|
|
result = TestRunResult("pytest first", TestRunOutcome.Failure, 1, "first output", ["first trace"], stdout="stdout", stderr="stderr", test_name="test_value")
|
|
receipt = ValidationReceipt.from_result(test, result, test_definition="def test_value(): pass", generation=1)
|
|
|
|
result.output = "second output"
|
|
result.stack_trace.append("second trace")
|
|
result.test_name = "test_other"
|
|
|
|
assert receipt.result.output == "first output"
|
|
assert receipt.result.stack_trace == ["first trace"]
|
|
assert receipt.test_name == "test_value"
|
|
|
|
|
|
@pytest.mark.parametrize("outcome,code", ((TestRunOutcome.Timeout, 124), (TestRunOutcome.Success, 0)))
|
|
def test_timeout_receipt_projects_current_evidence_in_existing_error_scope(outcome, code):
|
|
"""Fresh file observations replace old timeout details; timing does not confer success."""
|
|
from band_aider.judgement.edit_request import AiderFailureMode, AiderEditRequestMask
|
|
|
|
test = ClassMeta("tests/test_value.py", "", [], {}, content_type=ContentType.Test)
|
|
result = TestRunResult("pytest tests/test_value.py", outcome, code, "fresh output", run_info="fresh invocation details")
|
|
receipt = ValidationReceipt.from_result(test, result, test_definition="test definition", generation=1, file_scope=True)
|
|
receipts = ValidationReceipts((receipt,), generation=1)
|
|
request = AiderEditRequest("repair", [], [], failure_test_location=test.location, failure_outcome=TestRunOutcome.Timeout,
|
|
failure_error_message="old timeout", failure_run_info="old invocation details", validation_receipts=receipts)
|
|
|
|
view = request.validation_view()
|
|
|
|
assert "fresh invocation details" in receipts.render()
|
|
assert view.failure_run_info == ("" if code == 0 else "fresh invocation details")
|
|
assert request.failure_run_info == "old invocation details"
|
|
assert receipt.passed is (code == 0)
|
|
assert AiderEditRequestMask(view, AiderFailureMode.STATIC).projected().failure_run_info == ""
|