366 lines
22 KiB
Python
366 lines
22 KiB
Python
"""Independent applicability must constrain localization before candidate-aware semantic review."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from band_aider.context.provenance import FutureGuidance
|
|
from band_aider.judgement.edit_request import AiderEditRequest
|
|
from band_aider.judgement.code_editor import EditorRecord
|
|
from band_aider.judgement.prompt_localization import PromptLocalization
|
|
from tests.band_aider_loop_impl import _file_manager
|
|
from tests.support.completion_clients import ScriptedCompletionClient
|
|
|
|
|
|
pytestmark = pytest.mark.prompt_localization
|
|
|
|
|
|
@pytest.mark.parametrize(('field', 'incomplete', 'missing', 'highlight'), (
|
|
('relevant cot', [2, 2], 3, 'Expose resolution UNRESOLVED.'),
|
|
('relevant backlog guidance', [], 5, 'Repair the missing resolution.'),
|
|
('relevant future guidance', [], 7, 'Verify both factory branches.'),
|
|
('relevant code', [[8, 8]], 9, 'Include and cite the missing original source'),
|
|
))
|
|
def test_independent_requirements_reject_missing_coverage_before_semantic_review(
|
|
field: str, incomplete: list[int] | list[list[int]], missing: int, highlight: str) -> None:
|
|
"""Missing goal/guidance/code identities cannot be hidden by duplicate counts or later PASS verdicts."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'Keep construction inert.\nExpose resolution UNRESOLVED.'
|
|
original = ('COT\nKeep construction inert.\nExpose resolution UNRESOLVED.\n'
|
|
'BACKLOG GUIDANCE\nRepair the missing resolution.\n'
|
|
'FUTURE GUIDANCE\nVerify both factory branches.\nFILE: src/foo.py\nreturn 1')
|
|
required = {'relevant cot': [2, 3], 'relevant backlog guidance': [5],
|
|
'relevant future guidance': [7], 'relevant code': [[8, 9]], 'relevant stack': []}
|
|
complete = {'prompt': 'CANDIDATE: Keep construction inert. Expose resolution UNRESOLVED. '
|
|
'Repair the missing resolution and verify both factory branches.\n{{SOURCE}}',
|
|
'relevant cot': [2, 3], 'relevant backlog guidance': [5],
|
|
'relevant future guidance': [7], 'relevant code': [[8, 9]]}
|
|
bad = dict(complete, **{field: incomplete})
|
|
passed = {'outcome': 'pass', 'reason': 'The complete instruction covers all fixed requirements.', 'instruction': ''}
|
|
explanation = {'summary': 'The instruction must connect the lifecycle requirement to its supporting input. '
|
|
'Reconsider the missing citation separately from whether the prose already covers it.'}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (required, [bad], explanation, [complete], passed, passed))))
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager,
|
|
request=AiderEditRequest(manager.prompt, [Path('src/foo.py')], []), owner='Editor').localize(original)
|
|
|
|
assert len(client.prompts) == 6
|
|
assert 'CANDIDATE:' not in client.prompts[0]
|
|
assert 'RETURNED LOCALIZED PROMPTS' not in client.prompts[0]
|
|
assert sum('BEGIN LOCALIZATION APPLICABILITY' in prompt for prompt in client.prompts) == 1
|
|
assert 'BEGIN LOCALIZATION COVERAGE FEEDBACK' in client.prompts[2]
|
|
assert json.dumps([bad]) in client.prompts[2]
|
|
assert 'Summarize in your own words' in client.prompts[2]
|
|
assert 'do not write replacement localized prose' in client.prompts[2]
|
|
feedback = client.prompts[3].split('BEGIN LOCALIZATION VALIDATION ERROR')[1]
|
|
assert f'{field}: Missing independently required input lines [{missing}]' in feedback
|
|
assert highlight in feedback
|
|
assert explanation['summary'] in feedback
|
|
assert 'BEGIN LOCALIZATION ALIGNMENT REVIEW' in client.prompts[4]
|
|
assert 'Expose resolution UNRESOLVED.' in result
|
|
assert result.endswith('FILE: src/foo.py\nreturn 1')
|
|
|
|
|
|
def test_coverage_accepts_supersets_and_unions_across_prompt_items() -> None:
|
|
"""Items may share obligations and retain extra source without failing the fixed subset comparison."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'phase INERT\nresolution UNRESOLVED\nKeep path resolution.'
|
|
original = manager.prompt + '\nBACKLOG: preserve descriptor selection\nFUTURE: verify both branches\nFILE: src/foo.py\nreturn 1'
|
|
required = {'relevant cot': [1, 2], 'relevant backlog guidance': [4],
|
|
'relevant future guidance': [5], 'relevant code': [[7, 7]], 'relevant stack': []}
|
|
candidate = [
|
|
{'prompt': 'Verify phase INERT. Preserve descriptor selection.', 'relevant cot': [1],
|
|
'relevant backlog guidance': [4], 'relevant code': [[6, 7]]},
|
|
{'prompt': 'Verify resolution UNRESOLVED, both branches and path resolution.', 'relevant cot': [2, 3],
|
|
'relevant future guidance': [5]},
|
|
]
|
|
passed = {'outcome': 'pass', 'reason': 'Both prompts substantively cover their claimed obligations.', 'instruction': ''}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (required, candidate, passed, passed))))
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager,
|
|
request=AiderEditRequest(manager.prompt, [], []), owner='Inspection').localize(original)
|
|
|
|
assert 'phase INERT' in result and 'resolution UNRESOLVED' in result
|
|
assert 'FILE: src/foo.py\nreturn 1' in result
|
|
assert len(client.prompts) == 4
|
|
|
|
|
|
def test_claimed_cot_line_cannot_override_missing_goal_in_instruction_text() -> None:
|
|
"""A candidate claiming the right IDs still regenerates when detail review finds missing substance."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'phase INERT\nresolution UNRESOLVED'
|
|
required = {'relevant cot': [1, 2], 'relevant code': [], 'relevant stack': [],
|
|
'relevant backlog guidance': [], 'relevant future guidance': []}
|
|
bad = [{'prompt': 'Verify phase INERT.', 'relevant cot': [1, 2]}]
|
|
good = [{'prompt': 'Verify phase INERT and resolution UNRESOLVED.', 'relevant cot': [1, 2]}]
|
|
passed = {'outcome': 'pass', 'reason': 'No contradiction.', 'instruction': ''}
|
|
failed = {'outcome': 'fail', 'reason': 'Input line 2 is claimed but not addressed.',
|
|
'instruction': 'Highlight resolution UNRESOLVED in the actual instruction.'}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (required, bad, passed, failed, good, passed, passed))))
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager,
|
|
request=AiderEditRequest(manager.prompt, [], []), owner='Inspection').localize(manager.prompt)
|
|
|
|
assert result == 'Verify phase INERT and resolution UNRESOLVED.'
|
|
assert 'Highlight resolution UNRESOLVED' in client.prompts[4]
|
|
assert all('"relevant cot": [1, 2]' in client.prompts[index] for index in (2, 3, 5, 6))
|
|
assert 'REQUIRED COVERAGE' not in client.prompts[1]
|
|
assert sum('BEGIN LOCALIZATION APPLICABILITY' in prompt for prompt in client.prompts) == 1
|
|
assert len(client.prompts) == 7
|
|
|
|
|
|
@pytest.mark.parametrize('bad', (
|
|
[{'prompt': 'Repair the failure.\n ValueError: bad', 'relevant cot': [1], 'relevant stack': [[4, 4]]}],
|
|
[{'prompt': 'Repair the failure.\nTraceback:\n File x.py:4\n ValueError: wrong',
|
|
'relevant cot': [1], 'relevant stack': [[2, 4]]}],
|
|
[{'prompt': 'Repair the failure.\nTraceback:\n File x.py:4', 'relevant cot': [1], 'relevant stack': [[2, 3]]},
|
|
{'prompt': 'Also inspect:\n ValueError: bad', 'relevant stack': [[4, 4]]}],
|
|
[{'prompt': 'Repair the failure.\nTraceback:\n File x.py:4\n ValueError: bad', 'relevant cot': [1]}],
|
|
[{'prompt': 'Repair the failure.\n{{SOURCE}}', 'relevant cot': [1], 'relevant code': [[2, 4]]},
|
|
{'prompt': 'Traceback:\n File x.py:4', 'relevant stack': [[2, 3]]},
|
|
{'prompt': ' ValueError: bad', 'relevant stack': [[4, 4]]}],
|
|
))
|
|
def test_required_trace_must_be_complete_exact_uninterrupted_and_cited(bad: list[dict[str, object]]) -> None:
|
|
"""Partial, paraphrased, split or uncited traces cannot pass through semantic similarity."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'Repair the failure.'
|
|
original = 'Repair the failure.\nTraceback:\n File x.py:4\n ValueError: bad\n'
|
|
required = {'relevant cot': [1], 'relevant code': [], 'relevant stack': [[2, 4]],
|
|
'relevant backlog guidance': [], 'relevant future guidance': []}
|
|
good = [{'prompt': 'Repair the failure.\nTraceback:\n File x.py:4\n ValueError: bad',
|
|
'relevant cot': [1], 'relevant stack': [[2, 4]]}]
|
|
passed = {'outcome': 'pass', 'reason': 'The task addresses the preserved failure.', 'instruction': ''}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (required, bad, good, passed, passed))))
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager,
|
|
request=AiderEditRequest(manager.prompt, [], []), owner='Editor').localize(original)
|
|
|
|
assert 'LOCALIZATION VALIDATION ERROR' in client.prompts[2]
|
|
assert 'BEGIN LOCALIZATION ALIGNMENT REVIEW' in client.prompts[3]
|
|
assert result == good[0]['prompt']
|
|
assert len(client.prompts) == 5
|
|
|
|
|
|
@pytest.mark.parametrize('invalid', (
|
|
{}, {'relevant cot': [0]}, {'relevant cot': [99]}, {'relevant cot': [True]},
|
|
{'relevant code': [[1]]}, {'relevant code': [[2, 1]]}, {'relevant stack': [[1, 99]]},
|
|
{'relevant code': [1, 1]}, {'relevant stack': [1, 1]},
|
|
{'relevant future guidance': [1.0]},
|
|
))
|
|
def test_invalid_applicability_is_corrected_before_any_localization(invalid: dict[str, object]) -> None:
|
|
"""Malformed or invented requirement coordinates stay with their selector instead of burdening generation."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'goal'
|
|
required = {'relevant cot': [1], 'relevant code': [], 'relevant stack': [],
|
|
'relevant backlog guidance': [], 'relevant future guidance': []}
|
|
wrong = dict(required, **invalid) if invalid else {}
|
|
candidate = [{'prompt': 'Address goal.', 'relevant cot': [1]}]
|
|
passed = {'outcome': 'pass', 'reason': 'Goal addressed.', 'instruction': ''}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (wrong, required, candidate, passed, passed))))
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager,
|
|
request=AiderEditRequest(manager.prompt, [], []), owner='Inspection').localize('goal')
|
|
|
|
assert result == 'Address goal.'
|
|
assert 'BEGIN LOCALIZATION APPLICABILITY' in client.prompts[1]
|
|
assert 'LOCALIZATION VALIDATION ERROR' in client.prompts[1]
|
|
assert 'BEGIN PROMPT LOCALIZATION' in client.prompts[2]
|
|
assert len(client.prompts) == 5
|
|
|
|
|
|
@pytest.mark.parametrize(('field', 'value'), (
|
|
('relevant cot', [0]), ('relevant cot', [True]), ('relevant backlog guidance', [99]),
|
|
('relevant future guidance', [1.0]),
|
|
))
|
|
def test_invalid_candidate_authority_is_corrected_without_reselecting_requirements(field: str, value: list[object]) -> None:
|
|
"""Producer annotations must use valid input identities while applicability remains stationary."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'goal'
|
|
required = {'relevant cot': [1], 'relevant code': [], 'relevant stack': [],
|
|
'relevant backlog guidance': [], 'relevant future guidance': []}
|
|
bad = [{'prompt': 'Address goal.', field: value}]
|
|
good = [{'prompt': 'Address goal.', 'relevant cot': [1]}]
|
|
passed = {'outcome': 'pass', 'reason': 'Goal addressed.', 'instruction': ''}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (required, bad, good, passed, passed))))
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager,
|
|
request=AiderEditRequest(manager.prompt, [], []), owner='Inspection').localize('goal')
|
|
|
|
assert result == 'Address goal.'
|
|
assert sum('BEGIN LOCALIZATION APPLICABILITY' in prompt for prompt in client.prompts) == 1
|
|
assert 'LOCALIZATION VALIDATION ERROR' in client.prompts[2]
|
|
assert len(client.prompts) == 5
|
|
|
|
|
|
def test_external_cot_and_backlog_are_numbered_without_changing_source_final_byte() -> None:
|
|
"""Applicability and generation share appended authority coordinates while source remains exact."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'phase INERT\nresolution UNRESOLVED'
|
|
original = 'FILE: src/foo.py\nreturn 1'
|
|
required = {'relevant cot': [5, 6], 'relevant code': [[1, 2]], 'relevant stack': [],
|
|
'relevant backlog guidance': [10], 'relevant future guidance': []}
|
|
candidate = [{'prompt': 'Verify phase INERT and resolution UNRESOLVED; repair resolution.\n{{SOURCE}}',
|
|
'relevant code': [[1, 2]], 'relevant cot': [5, 6], 'relevant backlog guidance': [10]}]
|
|
passed = {'outcome': 'pass', 'reason': 'All required authority is addressed.', 'instruction': ''}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (required, candidate, passed, passed))))
|
|
request = AiderEditRequest(manager.prompt, [Path('src/foo.py')], [], backlog_guidance=['Repair resolution.'])
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager, request=request, owner='Editor').localize(original)
|
|
|
|
assert result.endswith(original)
|
|
assert not result.endswith('\n')
|
|
assert all('5 | phase INERT\n6 | resolution UNRESOLVED' in prompt for prompt in client.prompts)
|
|
assert all('10 | Repair resolution.' in prompt for prompt in client.prompts)
|
|
|
|
|
|
def test_opted_in_localization_receives_complete_work_logs_without_additional_calls() -> None:
|
|
"""The existing four helpers see retained edits; their localized instruction survives raw-log reset."""
|
|
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'goal'
|
|
request = AiderEditRequest('goal', [], [], editor_records=[EditorRecord('type repair', 'before', 'after')])
|
|
required = {'relevant cot': [1], 'relevant code': [], 'relevant stack': [],
|
|
'relevant backlog guidance': [7, 10, 13], 'relevant future guidance': []}
|
|
candidate = [{'prompt': 'Address goal while preserving the completed type repair.', 'relevant cot': [1],
|
|
'relevant backlog guidance': [7, 10, 13]}]
|
|
passed = {'outcome': 'pass', 'reason': 'The retained type repair is accounted for.', 'instruction': ''}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (required, candidate, passed, passed))))
|
|
|
|
localized = PromptLocalization(client=client, file_manager=manager, request=request, owner='Editor').localize('goal')
|
|
request.backlog_guidance = [localized]
|
|
request.editor_records = []
|
|
|
|
assert len(client.prompts) == 4
|
|
assert all('Work Logs' in prompt and '7 | type repair' in prompt and '13 | after' in prompt
|
|
for prompt in client.prompts)
|
|
assert request.guidance == ['Address goal while preserving the completed type repair.']
|
|
|
|
|
|
def test_future_guidance_coordinates_are_shared_before_and_after_localization() -> None:
|
|
"""The separate applicability call sees supplied future guidance before any summary exists."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'goal'
|
|
future = FutureGuidance(origin_message_id='later', addressed_message_id='earlier',
|
|
message='Verify both branches.', justification='Preserve goal.')
|
|
required = {'relevant cot': [1], 'relevant code': [], 'relevant stack': [],
|
|
'relevant backlog guidance': [], 'relevant future guidance': [7, 8]}
|
|
candidate = [{'prompt': 'Address goal and verify both branches.', 'relevant cot': [1],
|
|
'relevant future guidance': [7, 8]}]
|
|
passed = {'outcome': 'pass', 'reason': 'Goal and future verification are covered.', 'instruction': ''}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (required, candidate, passed, passed))))
|
|
request = AiderEditRequest(manager.prompt, [], [], future_guidance=(future,))
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager, request=request, owner='Inspection').localize(
|
|
'goal\nFILE: src/foo.py\nreturn 1\n')
|
|
|
|
assert result == 'Address goal and verify both branches.'
|
|
assert all('7 | Verify both branches.\n8 | Preserve goal.' in prompt for prompt in client.prompts)
|
|
assert len(client.prompts) == 4
|
|
|
|
|
|
def test_blank_stack_selection_is_repaired_before_requirements_are_frozen() -> None:
|
|
"""An impossible empty diagnostic belongs to selector correction, not endless producer retries."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'goal'
|
|
required = {'relevant cot': [1], 'relevant code': [], 'relevant stack': [],
|
|
'relevant backlog guidance': [], 'relevant future guidance': []}
|
|
blank = dict(required, **{'relevant stack': [[2, 2]]})
|
|
candidate = [{'prompt': 'Address goal.', 'relevant cot': [1]}]
|
|
passed = {'outcome': 'pass', 'reason': 'Goal addressed.', 'instruction': ''}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (blank, required, candidate, passed, passed))))
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager,
|
|
request=AiderEditRequest(manager.prompt, [], []), owner='Inspection').localize('goal\n\n')
|
|
|
|
assert result == 'Address goal.'
|
|
assert 'Select complete nonblank diagnostic blocks' in client.prompts[1]
|
|
assert 'BEGIN LOCALIZATION APPLICABILITY' in client.prompts[1]
|
|
assert len(client.prompts) == 5
|
|
|
|
|
|
def test_independent_generation_ignores_selector_answer_and_merges_source_ranges() -> None:
|
|
"""Independent answers can group evidence differently without leaking selections or duplicating code."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'Repair both failures.'
|
|
original = ('Repair both failures.\nclass A:\n pass\nTraceback:\nError: first\n'
|
|
'class B:\n pass\nError: second')
|
|
required = {'relevant code': [[2, 3], [6, 7]], 'relevant stack': [[4, 5], [8, 8]],
|
|
'relevant cot': [1], 'relevant backlog guidance': [], 'relevant future guidance': []}
|
|
candidate = [{'prompt': 'Inspect both failures and reason about their repairs.\n'
|
|
'Traceback:\nError: first\nError: second\n{{SOURCE}}',
|
|
'relevant code': [[7, 7], [2, 3], [6, 7], [2, 2]],
|
|
'relevant stack': [[4, 4], [5, 5], [8, 8]], 'relevant cot': [1]}]
|
|
passed = {'outcome': 'pass', 'reason': 'Both obligations and failures are preserved.', 'instruction': ''}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (required, candidate, passed, passed))))
|
|
changed = dict(required, **{'relevant code': [[2, 2], [6, 7]]})
|
|
other = ScriptedCompletionClient(tuple(map(json.dumps, (changed, candidate, passed, passed))))
|
|
request = AiderEditRequest(manager.prompt, [], [])
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager, request=request, owner='Editor').localize(original)
|
|
repeated = PromptLocalization(client=other, file_manager=manager, request=request, owner='Editor').localize(original)
|
|
|
|
assert result == repeated == ('Inspect both failures and reason about their repairs.\n'
|
|
'Traceback:\nError: first\nError: second\n'
|
|
'class A:\n pass\n\n\nclass B:\n pass\n')
|
|
assert len(client.prompts) == len(other.prompts) == 4
|
|
assert client.prompts[0] == other.prompts[0]
|
|
assert client.prompts[1] == other.prompts[1]
|
|
assert json.dumps(required) not in client.prompts[1]
|
|
assert 'REQUIRED COVERAGE' not in client.prompts[1]
|
|
assert 'Inspect both failures and reason' not in client.prompts[0]
|
|
assert client.prompts[2] != other.prompts[2]
|
|
|
|
|
|
@pytest.mark.parametrize('ranges', ([[2, 3], [2, 3]], [[2, 2], [4, 4]]))
|
|
def test_overlaps_and_gaps_cannot_hide_missing_source_lines(ranges: list[list[int]]) -> None:
|
|
"""Coverage is a subset comparison of identities, never a count or the endpoints' enclosing span."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'Inspect the function.'
|
|
original = 'Inspect the function.\ndef foo():\n value = 1\n return value\n'
|
|
required = {'relevant code': [[2, 4]], 'relevant stack': [], 'relevant cot': [1],
|
|
'relevant backlog guidance': [], 'relevant future guidance': []}
|
|
bad = [{'prompt': 'Inspect foo.\n{{SOURCE}}', 'relevant code': ranges, 'relevant cot': [1]}]
|
|
good = [{'prompt': 'Inspect foo.\n{{SOURCE}}', 'relevant code': [[2, 3], [3, 4]], 'relevant cot': [1]}]
|
|
passed = {'outcome': 'pass', 'reason': 'The whole function is available.', 'instruction': ''}
|
|
explanation = {'summary': 'Part of the function is missing from the selected source. '
|
|
'Reconsider what source the inspection needs to understand the operation.'}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps, (required, bad, explanation, good, passed, passed))))
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager,
|
|
request=AiderEditRequest(manager.prompt, [], []), owner='Inspection').localize(original)
|
|
|
|
assert result == 'Inspect foo.\ndef foo():\n value = 1\n return value\n'
|
|
assert 'relevant code: Missing independently required input lines' in client.prompts[3]
|
|
assert 'BEGIN LOCALIZATION ALIGNMENT REVIEW' in client.prompts[4]
|
|
assert sum('BEGIN LOCALIZATION APPLICABILITY' in prompt for prompt in client.prompts) == 1
|
|
assert len(client.prompts) == 6
|
|
|
|
|
|
def test_feedback_correction_and_disagreement_cannot_waive_missing_coverage() -> None:
|
|
"""The explanation has its own correction boundary and no authority to admit the candidate."""
|
|
manager = _file_manager(Path('/testbed'))
|
|
manager.prompt = 'Verify phase.\nVerify resolution.'
|
|
required = {'relevant cot': [1, 2], 'relevant code': [], 'relevant stack': [],
|
|
'relevant backlog guidance': [], 'relevant future guidance': []}
|
|
bad = [{'prompt': 'Verify phase and resolution.', 'relevant cot': [1]}]
|
|
good = [{'prompt': 'Verify phase and resolution.', 'relevant cot': [1, 2]}]
|
|
empty = {'summary': ''}
|
|
misleading = {'summary': 'The prose already covers resolution; no instruction change seems necessary.'}
|
|
passed = {'outcome': 'pass', 'reason': 'Both obligations are covered.', 'instruction': ''}
|
|
client = ScriptedCompletionClient(tuple(map(json.dumps,
|
|
(required, bad, empty, misleading, good, passed, passed))))
|
|
|
|
result = PromptLocalization(client=client, file_manager=manager,
|
|
request=AiderEditRequest(manager.prompt, [], []), owner='Inspection').localize(manager.prompt)
|
|
|
|
assert result == 'Verify phase and resolution.'
|
|
assert len(client.prompts) == 7
|
|
assert 'BEGIN LOCALIZATION COVERAGE FEEDBACK' in client.prompts[3]
|
|
assert 'Summarize the missing obligations' in client.prompts[3]
|
|
assert 'relevant cot: Missing independently required input lines [2]' in client.prompts[4]
|
|
assert misleading['summary'] in client.prompts[4]
|
|
assert 'BEGIN LOCALIZATION ALIGNMENT REVIEW' in client.prompts[5]
|
|
assert sum('BEGIN LOCALIZATION APPLICABILITY' in prompt for prompt in client.prompts) == 1
|