From 6233cfdf088d76db3c3b391acbb684169869c559 Mon Sep 17 00:00:00 2001 From: Punleuk Oum Date: Fri, 27 Feb 2026 09:48:09 -0800 Subject: [PATCH 1/6] test(conflict-resolution): split unit and llm integration tests - Extract unit tests from tests/llm/test_conflict_resolution.py into tests/unit/test_conflict_resolution.py - Register 'llm' pytest marker in pyproject.toml for LLM integration tests --- justfile | 6 + pyproject.toml | 3 +- tests/llm/test_conflict_resolution.py | 503 +++++-------------------- tests/unit/test_conflict_resolution.py | 438 +++++++++++++++++++++ 4 files changed, 537 insertions(+), 413 deletions(-) create mode 100644 tests/unit/test_conflict_resolution.py diff --git a/justfile b/justfile index d1dc7419..1c95a97f 100644 --- a/justfile +++ b/justfile @@ -2,6 +2,12 @@ default: @just --list +# Format staged files and commit — avoids pre-commit stash conflicts with ruff auto-fixes +commit message: + uv run ruff format . + git add -u + git commit -m "{{message}}" + image := "claude-sandbox" env_file := "sandbox/myenv" sandbox_dir := "sandbox" diff --git a/pyproject.toml b/pyproject.toml index 3802ef4d..119f5d1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,8 @@ addopts = "--ignore=explorations -m 'not phoenix'" markers = [ "e2e", "unit", - "phoenix" + "phoenix", + "llm" ] anyio_mode = "auto" diff --git a/tests/llm/test_conflict_resolution.py b/tests/llm/test_conflict_resolution.py index 41fa6271..b5c2cff4 100644 --- a/tests/llm/test_conflict_resolution.py +++ b/tests/llm/test_conflict_resolution.py @@ -1,438 +1,117 @@ -"""Tests for conflict resolution functionality.""" +"""LLM-based tests for conflict resolution. + +These tests call the real LLM and verify that the conflict resolution prompt +produces semantically correct diffs. They are slow and require a configured +LLM backend, so they are marked `llm` and excluded from the default run. + +Run with: uv run pytest -m llm +""" -import json from datetime import datetime -from unittest.mock import Mock, patch import pytest -from kaizen.llm.conflict_resolution.conflict_resolution import ( - resolve_conflicts, - get_update_entities_messages, -) -from kaizen.schema.conflict_resolution import SimpleEntity +from kaizen.llm.conflict_resolution.conflict_resolution import resolve_conflicts from kaizen.schema.core import RecordedEntity -# ============================================================================= -# Fixtures -# ============================================================================= - - -@pytest.fixture -def sample_recorded_entities(): - """Create sample RecordedEntity objects for testing.""" - return [ +@pytest.mark.llm +def test_add_to_empty_store(): + """All incoming entities should be ADDed when the store is empty.""" + new = [ RecordedEntity( - id="entity_1", + id="g1", type="guideline", - content="Always use type hints in Python", - metadata={"source": "code_review", "priority": "high"}, + content="Always use type hints in Python function signatures.", + metadata={}, created_at=datetime.now(), ), RecordedEntity( - id="entity_2", - type="guideline", - content="Write unit tests for all functions", - metadata={"source": "best_practices", "priority": "medium"}, - created_at=datetime.now(), + id="g2", type="guideline", content="Prefer f-strings over .format() or % formatting.", metadata={}, created_at=datetime.now() ), ] + updates = {u.id: u for u in resolve_conflicts([], new)} + assert updates["g1"].event == "ADD" + assert updates["g2"].event == "ADD" -@pytest.fixture -def sample_new_recorded_entities(): - """Create sample new RecordedEntity objects for testing.""" - return [ +@pytest.mark.llm +def test_none_for_duplicate_and_equivalent(): + """Exact duplicates and semantic paraphrases should not produce new ADDs.""" + old = [ RecordedEntity( - id="new_entity_1", - type="guideline", - content="Use descriptive variable names", - metadata={"source": "code_review", "priority": "high"}, - created_at=datetime.now(), + id="g1", type="fact", content="Always use type hints in Python function signatures.", metadata={}, created_at=datetime.now() ), + RecordedEntity(id="g2", type="fact", content="Likes cheese pizza", metadata={}, created_at=datetime.now()), ] - - -@pytest.fixture -def mock_llm_response_add(): - """Mock LLM response for ADD operation.""" - return json.dumps( - { - "entities": [ - { - "id": "entity_1", - "type": "guideline", - "content": "Always use type hints in Python", - "event": "NONE", - }, - { - "id": "entity_2", - "type": "guideline", - "content": "Write unit tests for all functions", - "event": "NONE", - }, - { - "id": "new_entity_1", - "type": "guideline", - "content": "Use descriptive variable names", - "event": "ADD", - }, - ] - } - ) - - -@pytest.fixture -def mock_llm_response_update(): - """Mock LLM response for UPDATE operation.""" - return json.dumps( - { - "entities": [ - { - "id": "entity_1", - "type": "guideline", - "content": "Always use type hints and docstrings in Python", - "event": "UPDATE", - "old_entity": "Always use type hints in Python", - }, - { - "id": "entity_2", - "type": "guideline", - "content": "Write unit tests for all functions", - "event": "NONE", - }, - ] - } - ) - - -@pytest.fixture -def mock_llm_response_delete(): - """Mock LLM response for DELETE operation.""" - return json.dumps( - { - "entities": [ - { - "id": "entity_1", - "type": "guideline", - "content": "Always use type hints in Python", - "event": "NONE", - }, - { - "id": "entity_2", - "type": "guideline", - "content": "Write unit tests for all functions", - "event": "DELETE", - }, - ] - } - ) - - -@pytest.fixture -def mock_llm_response_with_markdown(): - """Mock LLM response wrapped in markdown code block.""" - return """```json -{ - "entities": [ - { - "id": "entity_1", - "type": "guideline", - "content": "Test content", - "event": "NONE" - } - ] -} -```""" - - -# ============================================================================= -# SimpleEntity.from_recorded_entities() Tests -# ============================================================================= - - -@pytest.mark.unit -def test_from_recorded_entities_basic(sample_recorded_entities): - """Test basic conversion from RecordedEntity to SimpleEntity.""" - simple_entities = SimpleEntity.from_recorded_entities(sample_recorded_entities) - - assert len(simple_entities) == 2 - assert simple_entities[0].id == "entity_1" - assert simple_entities[0].type == "guideline" - assert simple_entities[0].content == "Always use type hints in Python" - assert simple_entities[1].id == "entity_2" - - # Test conversion with empty list. - simple_entities = SimpleEntity.from_recorded_entities([]) - assert simple_entities == [] - - # Test that different content types are preserved. - entities = [ + new = [ + # Exact duplicate RecordedEntity( - id="1", - type="test", - content="string content", - metadata={}, - created_at=datetime.now(), - ), - RecordedEntity( - id="2", - type="test", - content={"key": "value"}, - metadata={}, - created_at=datetime.now(), + id="g1_dup", type="fact", content="Always use type hints in Python function signatures.", metadata={}, created_at=datetime.now() ), + # Semantic paraphrase — same meaning, slightly different wording + RecordedEntity(id="g2_dup", type="fact", content="Loves cheese pizza", metadata={}, created_at=datetime.now()), + ] + updates = {u.id: u for u in resolve_conflicts(old, new)} + assert updates["g1"].event == "NONE" + assert updates["g2"].event == "NONE" + for u in updates.values(): + assert u.event != "ADD" + + +@pytest.mark.llm +def test_update_preserves_id_and_captures_old_content(): + """An enriched incoming entity should UPDATE the existing one, keeping its ID and recording old_entity.""" + old = [ + RecordedEntity(id="g1", type="fact", content="User likes to play cricket", metadata={}, created_at=datetime.now()), + RecordedEntity(id="g2", type="fact", content="User is a software engineer", metadata={}, created_at=datetime.now()), + ] + new = [ + # Richer version of g1 RecordedEntity( - id="3", - type="test", - content=["item1", "item2"], - metadata={}, - created_at=datetime.now(), + id="n1", type="fact", content="Loves to play cricket with friends on weekends", metadata={}, created_at=datetime.now() ), ] - - simple_entities = SimpleEntity.from_recorded_entities(entities) - - assert isinstance(simple_entities[0].content, str) - assert isinstance(simple_entities[1].content, dict) - assert isinstance(simple_entities[2].content, list) - - -# ============================================================================= -# get_update_entities_messages() Tests -# ============================================================================= - - -@pytest.mark.unit -def test_get_update_entities_messages_default_prompt(): - """Test prompt generation with default template.""" - old_entities = [SimpleEntity(id="1", type="guideline", content="Old content")] - new_entities = [SimpleEntity(id="2", type="guideline", content="New content")] - - prompt = get_update_entities_messages(old_entities, new_entities) - - assert "Old content" in prompt - assert "New content" in prompt - assert "ADD" in prompt - assert "UPDATE" in prompt - assert "DELETE" in prompt - assert "NONE" in prompt - assert '"id"' in prompt - assert '"type"' in prompt - assert '"content"' in prompt - - # Test prompt generation with custom template. - custom_prompt = "Custom instructions for entity management" - - prompt = get_update_entities_messages(old_entities, new_entities, custom_prompt) - - assert "Custom instructions for entity management" in prompt - assert "Old content" in prompt - assert "New content" in prompt - - # Test prompt generation with empty old entities list. - old_entities = [] - new_entities = [SimpleEntity(id="1", type="guideline", content="New content")] - - prompt = get_update_entities_messages(old_entities, new_entities) - - assert "Currently contains no entities" in prompt - assert "New content" in prompt - - -# ============================================================================= -# resolve_conflicts() Tests -# ============================================================================= - - -@pytest.mark.unit -@patch("kaizen.llm.conflict_resolution.conflict_resolution.completion") -def test_resolve_conflicts_event_types( - mock_completion, - sample_recorded_entities, - sample_new_recorded_entities, - mock_llm_response_add, - mock_llm_response_update, - mock_llm_response_delete, -): - """Test successful conflict resolution with ADD, UPDATE, DELETE, and NONE operations.""" - # Test ADD operation - mock_response = Mock() - mock_response.choices = [Mock()] - mock_response.choices[0].message.content = mock_llm_response_add - mock_completion.return_value = mock_response - - result = resolve_conflicts( - sample_recorded_entities, - sample_new_recorded_entities, - ) - - assert len(result) == 3 - assert result[0].event == "NONE" - assert result[1].event == "NONE" - assert result[2].event == "ADD" - assert result[2].id == "new_entity_1" - # Verify metadata was assigned for ADD operation - assert result[2].metadata == {"source": "code_review", "priority": "high"} - - # Test UPDATE operation - mock_response.choices[0].message.content = mock_llm_response_update - mock_completion.return_value = mock_response - - result = resolve_conflicts( - sample_recorded_entities, - sample_recorded_entities, - ) - - assert len(result) == 2 - assert result[0].event == "UPDATE" - assert result[0].old_entity == "Always use type hints in Python" - assert "docstrings" in result[0].content - assert result[1].event == "NONE" - # Verify UPDATE operation doesn't get metadata reassigned - assert result[0].metadata == {} - - # Test DELETE operation - mock_response.choices[0].message.content = mock_llm_response_delete - mock_completion.return_value = mock_response - - result = resolve_conflicts( - sample_recorded_entities, - sample_recorded_entities, - ) - - assert len(result) == 2 - assert result[0].event == "NONE" - assert result[1].event == "DELETE" - - -@pytest.mark.unit -@patch("kaizen.llm.conflict_resolution.conflict_resolution.completion") -def test_resolve_conflicts_response_parsing( - mock_completion, - sample_recorded_entities, - mock_llm_response_with_markdown, -): - """Test markdown cleaning and JSON parsing of LLM responses.""" - # Test that markdown code blocks are properly cleaned - mock_response = Mock() - mock_response.choices = [Mock()] - mock_response.choices[0].message.content = mock_llm_response_with_markdown - mock_completion.return_value = mock_response - - result = resolve_conflicts( - sample_recorded_entities, - sample_recorded_entities, - ) - - assert len(result) == 1 - assert result[0].event == "NONE" - - # Test handling of malformed JSON response - mock_response.choices[0].message.content = '{"entities": [invalid json}' - mock_completion.return_value = mock_response - - with pytest.raises(Exception): - resolve_conflicts( - sample_recorded_entities, - sample_recorded_entities, - ) - - # Test handling of response missing 'entities' key - mock_response.choices[0].message.content = json.dumps({"wrong_key": []}) - mock_completion.return_value = mock_response - - with pytest.raises(Exception): - resolve_conflicts( - sample_recorded_entities, - sample_recorded_entities, - ) - - -@pytest.mark.unit -@patch("kaizen.llm.conflict_resolution.conflict_resolution.completion") -def test_resolve_conflicts_retry_logic( - mock_completion, - sample_recorded_entities, - sample_new_recorded_entities, - mock_llm_response_add, -): - """Test retry logic when LLM calls fail.""" - # Test retry on JSON parsing error - mock_response_fail = Mock() - mock_response_fail.choices = [Mock()] - mock_response_fail.choices[0].message.content = "invalid json" - - mock_response_success = Mock() - mock_response_success.choices = [Mock()] - mock_response_success.choices[0].message.content = mock_llm_response_add - - mock_completion.side_effect = [ - mock_response_fail, - mock_response_fail, - mock_response_success, + updates = {u.id: u for u in resolve_conflicts(old, new)} + assert updates["g1"].event == "UPDATE" + assert updates["g1"].id == "g1" # ID must not change + assert updates["g1"].old_entity is not None # old content must be recorded + assert "cricket" in updates["g1"].old_entity + assert updates["g2"].event == "NONE" + + +@pytest.mark.llm +def test_delete_contradicted_fact(): + """A directly contradicting incoming entity should DELETE the old one.""" + old = [ + RecordedEntity(id="g1", type="fact", content="Name is John", metadata={}, created_at=datetime.now()), + RecordedEntity(id="g2", type="fact", content="Loves cheese pizza", metadata={}, created_at=datetime.now()), ] - - result = resolve_conflicts( - sample_recorded_entities, - sample_new_recorded_entities, - ) - - # Verify it succeeded after retries - assert len(result) == 3 - assert mock_completion.call_count == 3 - - # Test that exception is raised after max retries - mock_completion.reset_mock() - mock_completion.side_effect = Exception() - - with pytest.raises(Exception, match="Failed to resolve conflicts after 3 attempts"): - resolve_conflicts( - sample_recorded_entities, - sample_recorded_entities, - ) - - # Verify it tried 3 times - assert mock_completion.call_count == 3 - - -@pytest.mark.unit -@patch("kaizen.llm.conflict_resolution.conflict_resolution.completion") -def test_resolve_conflicts_edge_cases( - mock_completion, - sample_recorded_entities, - sample_new_recorded_entities, - mock_llm_response_add, -): - """Test edge cases like empty lists and custom prompts.""" - # Test conflict resolution with empty entity lists - mock_response = Mock() - mock_response.choices = [Mock()] - mock_response.choices[0].message.content = json.dumps({"entities": []}) - mock_completion.return_value = mock_response - - result = resolve_conflicts([], []) - assert result == [] - - # Test conflict resolution with custom prompt template - mock_response.choices[0].message.content = mock_llm_response_add - mock_completion.return_value = mock_response - - custom_prompt = "Custom conflict resolution instructions" - - result = resolve_conflicts( - sample_recorded_entities, - sample_new_recorded_entities, - custom_update_entities_prompt=custom_prompt, - ) - - # Verify the call was made with custom prompt - call_args = mock_completion.call_args - assert custom_prompt in call_args[1]["messages"][0]["content"] - assert len(result) == 3 - - # Test that LLM settings are properly used - assert "model" in call_args[1] - assert "messages" in call_args[1] - assert "custom_llm_provider" in call_args[1] + new = [RecordedEntity(id="n1", type="fact", content="Dislikes cheese pizza", metadata={}, created_at=datetime.now())] + updates = {u.id: u for u in resolve_conflicts(old, new)} + assert updates["g2"].event == "DELETE" + assert updates["g1"].event == "NONE" + + +@pytest.mark.llm +def test_mixed_add_update_delete_none(): + """A realistic batch: ADD new info, UPDATE enriched info, DELETE contradicted info, NONE for unchanged.""" + old = [ + RecordedEntity(id="g1", type="fact", content="I really like cheese pizza", metadata={}, created_at=datetime.now()), + RecordedEntity(id="g2", type="fact", content="User is a software engineer", metadata={}, created_at=datetime.now()), + RecordedEntity(id="g3", type="fact", content="User likes to play cricket", metadata={}, created_at=datetime.now()), + ] + new = [ + RecordedEntity( + id="n1", type="fact", content="Loves chicken pizza", metadata={}, created_at=datetime.now() + ), # contradicts / updates g1 + RecordedEntity( + id="n2", type="fact", content="Loves to play cricket with friends", metadata={}, created_at=datetime.now() + ), # enriches g3 + RecordedEntity(id="n3", type="fact", content="Name is John", metadata={}, created_at=datetime.now()), # brand new + ] + updates = {u.id: u for u in resolve_conflicts(old, new)} + assert updates["g1"].event == "UPDATE" + assert updates["g2"].event == "NONE" + assert updates["g3"].event == "UPDATE" + assert updates["n3"].event == "ADD" diff --git a/tests/unit/test_conflict_resolution.py b/tests/unit/test_conflict_resolution.py new file mode 100644 index 00000000..41fa6271 --- /dev/null +++ b/tests/unit/test_conflict_resolution.py @@ -0,0 +1,438 @@ +"""Tests for conflict resolution functionality.""" + +import json +from datetime import datetime +from unittest.mock import Mock, patch + +import pytest + +from kaizen.llm.conflict_resolution.conflict_resolution import ( + resolve_conflicts, + get_update_entities_messages, +) +from kaizen.schema.conflict_resolution import SimpleEntity +from kaizen.schema.core import RecordedEntity + + +# ============================================================================= +# Fixtures +# ============================================================================= + + +@pytest.fixture +def sample_recorded_entities(): + """Create sample RecordedEntity objects for testing.""" + return [ + RecordedEntity( + id="entity_1", + type="guideline", + content="Always use type hints in Python", + metadata={"source": "code_review", "priority": "high"}, + created_at=datetime.now(), + ), + RecordedEntity( + id="entity_2", + type="guideline", + content="Write unit tests for all functions", + metadata={"source": "best_practices", "priority": "medium"}, + created_at=datetime.now(), + ), + ] + + +@pytest.fixture +def sample_new_recorded_entities(): + """Create sample new RecordedEntity objects for testing.""" + return [ + RecordedEntity( + id="new_entity_1", + type="guideline", + content="Use descriptive variable names", + metadata={"source": "code_review", "priority": "high"}, + created_at=datetime.now(), + ), + ] + + +@pytest.fixture +def mock_llm_response_add(): + """Mock LLM response for ADD operation.""" + return json.dumps( + { + "entities": [ + { + "id": "entity_1", + "type": "guideline", + "content": "Always use type hints in Python", + "event": "NONE", + }, + { + "id": "entity_2", + "type": "guideline", + "content": "Write unit tests for all functions", + "event": "NONE", + }, + { + "id": "new_entity_1", + "type": "guideline", + "content": "Use descriptive variable names", + "event": "ADD", + }, + ] + } + ) + + +@pytest.fixture +def mock_llm_response_update(): + """Mock LLM response for UPDATE operation.""" + return json.dumps( + { + "entities": [ + { + "id": "entity_1", + "type": "guideline", + "content": "Always use type hints and docstrings in Python", + "event": "UPDATE", + "old_entity": "Always use type hints in Python", + }, + { + "id": "entity_2", + "type": "guideline", + "content": "Write unit tests for all functions", + "event": "NONE", + }, + ] + } + ) + + +@pytest.fixture +def mock_llm_response_delete(): + """Mock LLM response for DELETE operation.""" + return json.dumps( + { + "entities": [ + { + "id": "entity_1", + "type": "guideline", + "content": "Always use type hints in Python", + "event": "NONE", + }, + { + "id": "entity_2", + "type": "guideline", + "content": "Write unit tests for all functions", + "event": "DELETE", + }, + ] + } + ) + + +@pytest.fixture +def mock_llm_response_with_markdown(): + """Mock LLM response wrapped in markdown code block.""" + return """```json +{ + "entities": [ + { + "id": "entity_1", + "type": "guideline", + "content": "Test content", + "event": "NONE" + } + ] +} +```""" + + +# ============================================================================= +# SimpleEntity.from_recorded_entities() Tests +# ============================================================================= + + +@pytest.mark.unit +def test_from_recorded_entities_basic(sample_recorded_entities): + """Test basic conversion from RecordedEntity to SimpleEntity.""" + simple_entities = SimpleEntity.from_recorded_entities(sample_recorded_entities) + + assert len(simple_entities) == 2 + assert simple_entities[0].id == "entity_1" + assert simple_entities[0].type == "guideline" + assert simple_entities[0].content == "Always use type hints in Python" + assert simple_entities[1].id == "entity_2" + + # Test conversion with empty list. + simple_entities = SimpleEntity.from_recorded_entities([]) + assert simple_entities == [] + + # Test that different content types are preserved. + entities = [ + RecordedEntity( + id="1", + type="test", + content="string content", + metadata={}, + created_at=datetime.now(), + ), + RecordedEntity( + id="2", + type="test", + content={"key": "value"}, + metadata={}, + created_at=datetime.now(), + ), + RecordedEntity( + id="3", + type="test", + content=["item1", "item2"], + metadata={}, + created_at=datetime.now(), + ), + ] + + simple_entities = SimpleEntity.from_recorded_entities(entities) + + assert isinstance(simple_entities[0].content, str) + assert isinstance(simple_entities[1].content, dict) + assert isinstance(simple_entities[2].content, list) + + +# ============================================================================= +# get_update_entities_messages() Tests +# ============================================================================= + + +@pytest.mark.unit +def test_get_update_entities_messages_default_prompt(): + """Test prompt generation with default template.""" + old_entities = [SimpleEntity(id="1", type="guideline", content="Old content")] + new_entities = [SimpleEntity(id="2", type="guideline", content="New content")] + + prompt = get_update_entities_messages(old_entities, new_entities) + + assert "Old content" in prompt + assert "New content" in prompt + assert "ADD" in prompt + assert "UPDATE" in prompt + assert "DELETE" in prompt + assert "NONE" in prompt + assert '"id"' in prompt + assert '"type"' in prompt + assert '"content"' in prompt + + # Test prompt generation with custom template. + custom_prompt = "Custom instructions for entity management" + + prompt = get_update_entities_messages(old_entities, new_entities, custom_prompt) + + assert "Custom instructions for entity management" in prompt + assert "Old content" in prompt + assert "New content" in prompt + + # Test prompt generation with empty old entities list. + old_entities = [] + new_entities = [SimpleEntity(id="1", type="guideline", content="New content")] + + prompt = get_update_entities_messages(old_entities, new_entities) + + assert "Currently contains no entities" in prompt + assert "New content" in prompt + + +# ============================================================================= +# resolve_conflicts() Tests +# ============================================================================= + + +@pytest.mark.unit +@patch("kaizen.llm.conflict_resolution.conflict_resolution.completion") +def test_resolve_conflicts_event_types( + mock_completion, + sample_recorded_entities, + sample_new_recorded_entities, + mock_llm_response_add, + mock_llm_response_update, + mock_llm_response_delete, +): + """Test successful conflict resolution with ADD, UPDATE, DELETE, and NONE operations.""" + # Test ADD operation + mock_response = Mock() + mock_response.choices = [Mock()] + mock_response.choices[0].message.content = mock_llm_response_add + mock_completion.return_value = mock_response + + result = resolve_conflicts( + sample_recorded_entities, + sample_new_recorded_entities, + ) + + assert len(result) == 3 + assert result[0].event == "NONE" + assert result[1].event == "NONE" + assert result[2].event == "ADD" + assert result[2].id == "new_entity_1" + # Verify metadata was assigned for ADD operation + assert result[2].metadata == {"source": "code_review", "priority": "high"} + + # Test UPDATE operation + mock_response.choices[0].message.content = mock_llm_response_update + mock_completion.return_value = mock_response + + result = resolve_conflicts( + sample_recorded_entities, + sample_recorded_entities, + ) + + assert len(result) == 2 + assert result[0].event == "UPDATE" + assert result[0].old_entity == "Always use type hints in Python" + assert "docstrings" in result[0].content + assert result[1].event == "NONE" + # Verify UPDATE operation doesn't get metadata reassigned + assert result[0].metadata == {} + + # Test DELETE operation + mock_response.choices[0].message.content = mock_llm_response_delete + mock_completion.return_value = mock_response + + result = resolve_conflicts( + sample_recorded_entities, + sample_recorded_entities, + ) + + assert len(result) == 2 + assert result[0].event == "NONE" + assert result[1].event == "DELETE" + + +@pytest.mark.unit +@patch("kaizen.llm.conflict_resolution.conflict_resolution.completion") +def test_resolve_conflicts_response_parsing( + mock_completion, + sample_recorded_entities, + mock_llm_response_with_markdown, +): + """Test markdown cleaning and JSON parsing of LLM responses.""" + # Test that markdown code blocks are properly cleaned + mock_response = Mock() + mock_response.choices = [Mock()] + mock_response.choices[0].message.content = mock_llm_response_with_markdown + mock_completion.return_value = mock_response + + result = resolve_conflicts( + sample_recorded_entities, + sample_recorded_entities, + ) + + assert len(result) == 1 + assert result[0].event == "NONE" + + # Test handling of malformed JSON response + mock_response.choices[0].message.content = '{"entities": [invalid json}' + mock_completion.return_value = mock_response + + with pytest.raises(Exception): + resolve_conflicts( + sample_recorded_entities, + sample_recorded_entities, + ) + + # Test handling of response missing 'entities' key + mock_response.choices[0].message.content = json.dumps({"wrong_key": []}) + mock_completion.return_value = mock_response + + with pytest.raises(Exception): + resolve_conflicts( + sample_recorded_entities, + sample_recorded_entities, + ) + + +@pytest.mark.unit +@patch("kaizen.llm.conflict_resolution.conflict_resolution.completion") +def test_resolve_conflicts_retry_logic( + mock_completion, + sample_recorded_entities, + sample_new_recorded_entities, + mock_llm_response_add, +): + """Test retry logic when LLM calls fail.""" + # Test retry on JSON parsing error + mock_response_fail = Mock() + mock_response_fail.choices = [Mock()] + mock_response_fail.choices[0].message.content = "invalid json" + + mock_response_success = Mock() + mock_response_success.choices = [Mock()] + mock_response_success.choices[0].message.content = mock_llm_response_add + + mock_completion.side_effect = [ + mock_response_fail, + mock_response_fail, + mock_response_success, + ] + + result = resolve_conflicts( + sample_recorded_entities, + sample_new_recorded_entities, + ) + + # Verify it succeeded after retries + assert len(result) == 3 + assert mock_completion.call_count == 3 + + # Test that exception is raised after max retries + mock_completion.reset_mock() + mock_completion.side_effect = Exception() + + with pytest.raises(Exception, match="Failed to resolve conflicts after 3 attempts"): + resolve_conflicts( + sample_recorded_entities, + sample_recorded_entities, + ) + + # Verify it tried 3 times + assert mock_completion.call_count == 3 + + +@pytest.mark.unit +@patch("kaizen.llm.conflict_resolution.conflict_resolution.completion") +def test_resolve_conflicts_edge_cases( + mock_completion, + sample_recorded_entities, + sample_new_recorded_entities, + mock_llm_response_add, +): + """Test edge cases like empty lists and custom prompts.""" + # Test conflict resolution with empty entity lists + mock_response = Mock() + mock_response.choices = [Mock()] + mock_response.choices[0].message.content = json.dumps({"entities": []}) + mock_completion.return_value = mock_response + + result = resolve_conflicts([], []) + assert result == [] + + # Test conflict resolution with custom prompt template + mock_response.choices[0].message.content = mock_llm_response_add + mock_completion.return_value = mock_response + + custom_prompt = "Custom conflict resolution instructions" + + result = resolve_conflicts( + sample_recorded_entities, + sample_new_recorded_entities, + custom_update_entities_prompt=custom_prompt, + ) + + # Verify the call was made with custom prompt + call_args = mock_completion.call_args + assert custom_prompt in call_args[1]["messages"][0]["content"] + assert len(result) == 3 + + # Test that LLM settings are properly used + assert "model" in call_args[1] + assert "messages" in call_args[1] + assert "custom_llm_provider" in call_args[1] From 95d4de9afcb160869f14f426b7f8b87b6222ea52 Mon Sep 17 00:00:00 2001 From: Punleuk Oum Date: Fri, 27 Feb 2026 09:51:02 -0800 Subject: [PATCH 2/6] docs(agents): run ruff format before committing to avoid pre-commit stash conflicts --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 46899a74..3861b841 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,6 +66,6 @@ pre-commit install ## Coding Standards - Use Ruff for linting and formatting (configured in pyproject.toml) -- Run pre-commit hooks before committing +- Always run `uv run ruff format .` and `git add -u` before committing to avoid pre-commit stash conflicts with ruff auto-fixes - All new features need tests (unit + e2e where applicable) - Use uv to run Python commands, including pip. From 688c04908680228ba649b118d2d569609b70ad47 Mon Sep 17 00:00:00 2001 From: Punleuk Oum Date: Fri, 27 Feb 2026 09:51:57 -0800 Subject: [PATCH 3/6] docs(agents): add conventional commits format guidance for python-semantic-release --- AGENTS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 3861b841..7cc61f66 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,5 +67,11 @@ pre-commit install ## Coding Standards - Use Ruff for linting and formatting (configured in pyproject.toml) - Always run `uv run ruff format .` and `git add -u` before committing to avoid pre-commit stash conflicts with ruff auto-fixes +- Write commit messages in the Conventional Commits format expected by `python-semantic-release`: + - `feat(scope): description` — new feature, triggers a minor version bump + - `fix(scope): description` — bug fix, triggers a patch version bump + - `perf(scope): description` — performance improvement, triggers a patch version bump + - `test(scope): description`, `chore(scope): description`, `docs(scope): description`, etc. — no version bump + - Breaking changes: append `!` after the type/scope (e.g. `feat!:`) or add `BREAKING CHANGE:` in the footer - All new features need tests (unit + e2e where applicable) - Use uv to run Python commands, including pip. From 6eb66c9aa92e951116f9ab65179d5a91ac50e2f9 Mon Sep 17 00:00:00 2001 From: Punleuk Oum Date: Mon, 2 Mar 2026 11:49:51 -0800 Subject: [PATCH 4/6] parameterize conflict resolution test(llm)!: improve conflict resolution prompt clarity and add comprehensive test suite - Enhanced UPDATE operation to only trigger on concrete new information, not mere rewordings - Clarified DELETE operation to explicitly handle negation phrases like "no longer", "switched away from", "quit" - Improved NONE operation to properly handle paraphrases and synonyms - Fixed ADD operation to use retrieved entity ID instead of generating new ID - Added extensive examples for each operation type with realistic scenarios - Replaced old test file with comprehensive parametrized test suite covering 60+ scenarios --- .../prompts/conflict_resolution.jinja2 | 2 +- .../default_conflict_resolution.jinja2 | 122 +++- tests/llm/test_conflict_resolution.py | 117 ---- tests/llm/test_long_conflict_resolution.py | 629 ++++++++++++++++++ 4 files changed, 730 insertions(+), 140 deletions(-) delete mode 100644 tests/llm/test_conflict_resolution.py create mode 100644 tests/llm/test_long_conflict_resolution.py diff --git a/kaizen/llm/conflict_resolution/prompts/conflict_resolution.jinja2 b/kaizen/llm/conflict_resolution/prompts/conflict_resolution.jinja2 index 3d5e6496..79c7aeba 100644 --- a/kaizen/llm/conflict_resolution/prompts/conflict_resolution.jinja2 +++ b/kaizen/llm/conflict_resolution/prompts/conflict_resolution.jinja2 @@ -33,7 +33,7 @@ Follow the instruction mentioned below: - Do not return anything from the custom few shot prompts provided above. - If the current list of entities is empty, then you have to add the new retrieved entity to the list of entities. - You should return the updated entity in only JSON format as shown below. The entity ID and type should be the same if no changes are made. -- If there is an addition, generate a new ID and add the new event corresponding to it. +- If there is an addition, use the ID from the retrieved entity. - If there is a deletion, the object corresponding to that entity should be removed from the list of entities. - If there is an update, the ID key should remain the same and only the content needs to be updated. diff --git a/kaizen/llm/conflict_resolution/prompts/default_conflict_resolution.jinja2 b/kaizen/llm/conflict_resolution/prompts/default_conflict_resolution.jinja2 index 7ede9987..22abb842 100644 --- a/kaizen/llm/conflict_resolution/prompts/default_conflict_resolution.jinja2 +++ b/kaizen/llm/conflict_resolution/prompts/default_conflict_resolution.jinja2 @@ -47,10 +47,8 @@ Here are some guidelines on how to select which operation to perform: ] } -2. **Update**: If the retrieved entities contain information that is already present in the list of entities but the information is totally different, then you have to update it. -If the retrieved entity contains information that conveys the same thing as the elements present in the list of entities, then you have to keep the entity which has the most information. -Example (a) -- if the entity contains "User likes to play cricket" and the retrieved entity is "Loves to play cricket with friends", then update the list of entities with the retrieved facts. -Example (b) -- if the entity contains "Likes cheese pizza" and the retrieved entity is "Loves cheese pizza", then you do not need to update it because they convey the same information. +2. **Update**: Use UPDATE only when the new entity adds concrete new information that genuinely enriches an existing entity — for example, adding specific details, numbers, or qualifications that the old entity did not contain. Use UPDATE (not NONE) when the same topic now has different specific values — for example, a different deployment platform, a different tool, or a different location with new details, or additional preferences. If the new entity merely restates or paraphrases the old one (same meaning, different words), use NONE instead. Do NOT update an entity just because the wording changed. +Sometimes the new information found in a retrieved entity needs to be added to the old entity. If the direction is to update the list of entities, then you have to update it. Please keep in mind while updating you have to keep the same ID. Please note to return the IDs in the output from the input IDs only and do not generate any new ID. @@ -60,31 +58,45 @@ Please note to return the IDs in the output from the input IDs only and do not g { "id" : "0", "type": "fact", - "content" : "I really like cheese pizza" + "content" : "User is a software engineer" }, { "id" : "1", "type": "fact", - "content" : "User is a software engineer" + "content" : "User likes to play cricket" }, { "id" : "2", "type": "fact", - "content" : "User likes to play cricket" + "content" : "Likes racing" + }, + { + "id" : "3", + "type": "fact", + "content" : "His usual order is a double cheeseburger" } - ] - Retrieved entities: [ { "id" : "Retrieved_Fact_1", "type": "fact", - "content" : "Loves chicken pizza" + "content" : "User is a senior software engineer specializing in distributed systems" }, { "id" : "Retrieved_Fact_2", "type": "fact", - "content" : "Loves to play cricket with friends" - } + "content" : "Loves to play cricket with friends on weekends" + }, + { + "id" : "Retrieved_Fact_3", + "type": "fact", + "content" : "Loves racing" + }, + { + "id" : "Retrieved_Fact_4", + "type": "fact", + "content" : "Always requests his hamburger with grilled onions" + }, ] - New Entities: { @@ -92,28 +104,35 @@ Please note to return the IDs in the output from the input IDs only and do not g { "id" : "0", "type": "fact", - "content" : "Loves cheese and chicken pizza", + "content" : "User is a senior software engineer specializing in distributed systems", "event" : "UPDATE", - "old_entity" : "I really like cheese pizza" + "old_entity" : "User is a software engineer" }, { "id" : "1", "type": "fact", - "content" : "User is a software engineer", - "event" : "NONE" + "content" : "Loves to play cricket with friends on weekends", + "event" : "UPDATE", + "old_entity" : "User likes to play cricket" }, { "id" : "2", "type": "fact", - "content" : "Loves to play cricket with friends", + "content" : "Likes cheese pizza", + "event" : "NONE" + }, + { + "id" : "3", + "type": "fact", + "content" : "His usual order is a double cheeseburger with grilled onions", "event" : "UPDATE", - "old_entity" : "User likes to play cricket" + "old_entity": "His usual order is a double cheeseburger" } ] } -3. **Delete**: If the retrieved entities contain information that contradicts the information present in the list of entities, then you have to delete it. Or if the direction is to delete the entity, then you have to delete it. +3. **Delete**: If the retrieved entities contain information that directly contradicts an existing entity — making the old entity false or obsolete (e.g., a preference is reversed, a status is replaced, or the old information is no longer true) — then you have to delete it. Pay special attention to explicit negation phrases such as "no longer", "switched away from", "quit", "stopped using", "replaced" — these always indicate a DELETE. Or if the direction is to delete the entity, then you have to delete it. Please note to return the IDs in the output from the input IDs only and do not generate any new ID. - **Example**: - Old Entities: @@ -127,6 +146,11 @@ Please note to return the IDs in the output from the input IDs only and do not g "id" : "1", "type": "fact", "content" : "Loves cheese pizza" + }, + { + "id" : "2", + "type": "fact", + "content" : "User uses Jira for project management" } ] - Retrieved entities: @@ -135,6 +159,11 @@ Please note to return the IDs in the output from the input IDs only and do not g "id": "Retrieved_Fact_1", "type": "fact", "content": "Dislikes cheese pizza" + }, + { + "id": "Retrieved_Fact_2", + "type": "fact", + "content": "Team switched from Jira to Linear for project management" } ] - New Entities: @@ -151,11 +180,23 @@ Please note to return the IDs in the output from the input IDs only and do not g "type": "fact", "content" : "Loves cheese pizza", "event" : "DELETE" + }, + { + "id" : "2", + "type": "fact", + "content" : "User uses Jira for project management", + "event" : "DELETE" + }, + { + "id" : "Retrieved_Fact_2", + "type": "fact", + "content" : "Team switched from Jira to Linear for project management", + "event" : "ADD" } ] } -4. **No Change**: If the retrieved entities contain information that is already present in the list of entities, then you do not need to make any changes. +4. **No Change**: If the retrieved entities contain information that is already present in the list of old entities — including paraphrases, synonyms, or minor rewordings that convey the same meaning — then you do not need to make any changes. Also use NONE for any existing entity that is unrelated to the new entities. - **Example**: - Old Entities: [ @@ -167,7 +208,17 @@ Please note to return the IDs in the output from the input IDs only and do not g { "id" : "1", "type": "fact", - "content" : "Loves cheese pizza" + "content" : "Likes racing" + }, + { + "id" : "2", + "type": "fact", + "content" : "User enjoys running" + }, + { + "id" : "3", + "type": "fact", + "content" : "User works from home" } ] - Retrieved entities: @@ -176,6 +227,21 @@ Please note to return the IDs in the output from the input IDs only and do not g "id" : "Retrieved_Fact_1", "type": "fact", "content" : "Name is John" + }, + { + "id" : "Retrieved_Fact_2", + "type": "fact", + "content" : "Loves racing" + }, + { + "id" : "Retrieved_Fact_3", + "type": "fact", + "content" : "User likes to run" + }, + { + "id" : "Retrieved_Fact_4", + "type": "fact", + "content" : "The user is a remote worker" } ] - New Entities: @@ -190,8 +256,20 @@ Please note to return the IDs in the output from the input IDs only and do not g { "id" : "1", "type": "fact", - "content" : "Loves cheese pizza", + "content" : "Likes racing", + "event" : "NONE" + }, + { + "id" : "2", + "type": "fact", + "content" : "User enjoys running", + "event" : "NONE" + }, + { + "id" : "3", + "type": "fact", + "content" : "User works from home", "event" : "NONE" } ] - } \ No newline at end of file + } diff --git a/tests/llm/test_conflict_resolution.py b/tests/llm/test_conflict_resolution.py deleted file mode 100644 index b5c2cff4..00000000 --- a/tests/llm/test_conflict_resolution.py +++ /dev/null @@ -1,117 +0,0 @@ -"""LLM-based tests for conflict resolution. - -These tests call the real LLM and verify that the conflict resolution prompt -produces semantically correct diffs. They are slow and require a configured -LLM backend, so they are marked `llm` and excluded from the default run. - -Run with: uv run pytest -m llm -""" - -from datetime import datetime - -import pytest - -from kaizen.llm.conflict_resolution.conflict_resolution import resolve_conflicts -from kaizen.schema.core import RecordedEntity - - -@pytest.mark.llm -def test_add_to_empty_store(): - """All incoming entities should be ADDed when the store is empty.""" - new = [ - RecordedEntity( - id="g1", - type="guideline", - content="Always use type hints in Python function signatures.", - metadata={}, - created_at=datetime.now(), - ), - RecordedEntity( - id="g2", type="guideline", content="Prefer f-strings over .format() or % formatting.", metadata={}, created_at=datetime.now() - ), - ] - updates = {u.id: u for u in resolve_conflicts([], new)} - assert updates["g1"].event == "ADD" - assert updates["g2"].event == "ADD" - - -@pytest.mark.llm -def test_none_for_duplicate_and_equivalent(): - """Exact duplicates and semantic paraphrases should not produce new ADDs.""" - old = [ - RecordedEntity( - id="g1", type="fact", content="Always use type hints in Python function signatures.", metadata={}, created_at=datetime.now() - ), - RecordedEntity(id="g2", type="fact", content="Likes cheese pizza", metadata={}, created_at=datetime.now()), - ] - new = [ - # Exact duplicate - RecordedEntity( - id="g1_dup", type="fact", content="Always use type hints in Python function signatures.", metadata={}, created_at=datetime.now() - ), - # Semantic paraphrase — same meaning, slightly different wording - RecordedEntity(id="g2_dup", type="fact", content="Loves cheese pizza", metadata={}, created_at=datetime.now()), - ] - updates = {u.id: u for u in resolve_conflicts(old, new)} - assert updates["g1"].event == "NONE" - assert updates["g2"].event == "NONE" - for u in updates.values(): - assert u.event != "ADD" - - -@pytest.mark.llm -def test_update_preserves_id_and_captures_old_content(): - """An enriched incoming entity should UPDATE the existing one, keeping its ID and recording old_entity.""" - old = [ - RecordedEntity(id="g1", type="fact", content="User likes to play cricket", metadata={}, created_at=datetime.now()), - RecordedEntity(id="g2", type="fact", content="User is a software engineer", metadata={}, created_at=datetime.now()), - ] - new = [ - # Richer version of g1 - RecordedEntity( - id="n1", type="fact", content="Loves to play cricket with friends on weekends", metadata={}, created_at=datetime.now() - ), - ] - updates = {u.id: u for u in resolve_conflicts(old, new)} - assert updates["g1"].event == "UPDATE" - assert updates["g1"].id == "g1" # ID must not change - assert updates["g1"].old_entity is not None # old content must be recorded - assert "cricket" in updates["g1"].old_entity - assert updates["g2"].event == "NONE" - - -@pytest.mark.llm -def test_delete_contradicted_fact(): - """A directly contradicting incoming entity should DELETE the old one.""" - old = [ - RecordedEntity(id="g1", type="fact", content="Name is John", metadata={}, created_at=datetime.now()), - RecordedEntity(id="g2", type="fact", content="Loves cheese pizza", metadata={}, created_at=datetime.now()), - ] - new = [RecordedEntity(id="n1", type="fact", content="Dislikes cheese pizza", metadata={}, created_at=datetime.now())] - updates = {u.id: u for u in resolve_conflicts(old, new)} - assert updates["g2"].event == "DELETE" - assert updates["g1"].event == "NONE" - - -@pytest.mark.llm -def test_mixed_add_update_delete_none(): - """A realistic batch: ADD new info, UPDATE enriched info, DELETE contradicted info, NONE for unchanged.""" - old = [ - RecordedEntity(id="g1", type="fact", content="I really like cheese pizza", metadata={}, created_at=datetime.now()), - RecordedEntity(id="g2", type="fact", content="User is a software engineer", metadata={}, created_at=datetime.now()), - RecordedEntity(id="g3", type="fact", content="User likes to play cricket", metadata={}, created_at=datetime.now()), - ] - new = [ - RecordedEntity( - id="n1", type="fact", content="Loves chicken pizza", metadata={}, created_at=datetime.now() - ), # contradicts / updates g1 - RecordedEntity( - id="n2", type="fact", content="Loves to play cricket with friends", metadata={}, created_at=datetime.now() - ), # enriches g3 - RecordedEntity(id="n3", type="fact", content="Name is John", metadata={}, created_at=datetime.now()), # brand new - ] - updates = {u.id: u for u in resolve_conflicts(old, new)} - assert updates["g1"].event == "UPDATE" - assert updates["g2"].event == "NONE" - assert updates["g3"].event == "UPDATE" - assert updates["n3"].event == "ADD" diff --git a/tests/llm/test_long_conflict_resolution.py b/tests/llm/test_long_conflict_resolution.py new file mode 100644 index 00000000..11fef367 --- /dev/null +++ b/tests/llm/test_long_conflict_resolution.py @@ -0,0 +1,629 @@ +"""LLM-based tests for conflict resolution. + +These tests call the real LLM and verify that the conflict resolution prompt +produces semantically correct diffs. They are slow and require a configured +LLM backend, so they are marked `llm` and excluded from the default run. + +Run with: uv run pytest -m llm +""" + +from datetime import datetime +from typing import TypedDict + +import pytest + +from kaizen.llm.conflict_resolution.conflict_resolution import resolve_conflicts +from kaizen.schema.core import RecordedEntity + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _entity(entity_id: str, content: str, entity_type: str = "fact") -> RecordedEntity: + """Shorthand factory for a RecordedEntity.""" + return RecordedEntity(id=entity_id, type=entity_type, content=content, metadata={}, created_at=datetime.now()) + + +class ConflictScenario(TypedDict): + """A self-contained conflict-resolution scenario. + + Attributes: + label: Short human-readable name used as the pytest ID. + old: Entities already present in the store. + new: Entities just retrieved (to be reconciled against ``old``). + expect: Mapping of entity ID → expected event (or list of acceptable + events) that must appear in the ``resolve_conflicts`` result. + Use a list when the scenario is genuinely ambiguous and + multiple LLM answers are semantically valid. + """ + + label: str + old: list[RecordedEntity] + new: list[RecordedEntity] + expect: dict[str, str | list[str]] + + +# --------------------------------------------------------------------------- +# Individual conflict scenarios +# +# Each scenario is a ConflictScenario that is run directly as a parametrized +# test case. The LLM must correctly classify every entity listed in `expect`. +# --------------------------------------------------------------------------- + +CONFLICT_SCENARIOS: list[ConflictScenario] = [ + # ── ADD scenarios ────────────────────────────────────────────────────── + { + "label": "add_two_guidelines_to_empty_store", + "old": [], + "new": [ + _entity("g1", "Always use type hints in Python function signatures.", entity_type="guideline"), + _entity("g2", "Prefer f-strings over .format() or % formatting.", entity_type="guideline"), + ], + "expect": {"g1": "ADD", "g2": "ADD"}, + }, + { + "label": "add_name", + "old": [_entity("o1", "User is a software engineer")], + "new": [_entity("n1", "Name is Alice")], + "expect": {"n1": "ADD"}, + }, + { + "label": "add_hobby", + "old": [_entity("o2", "User likes hiking")], + "new": [_entity("n2", "User plays the guitar")], + "expect": {"n2": "ADD"}, + }, + { + "label": "add_language_preference", + "old": [_entity("o3", "Prefers Python over Java")], + "new": [_entity("n3", "Uses Rust for systems programming")], + "expect": {"n3": "ADD"}, + }, + { + "label": "add_diet", + "old": [_entity("o4", "User is vegetarian")], + "new": [_entity("n4", "User is lactose intolerant")], + "expect": {"n4": "ADD"}, + }, + { + "label": "add_location", + "old": [_entity("o5", "User lives in New York")], + "new": [_entity("n5", "User works remotely from home")], + "expect": {"n5": "ADD"}, + }, + { + "label": "add_guideline_type_hints", + "old": [], + "new": [_entity("n6", "Always use type hints in Python function signatures.", entity_type="guideline")], + "expect": {"n6": "ADD"}, + }, + { + "label": "add_guideline_fstrings", + "old": [], + "new": [_entity("n7", "Prefer f-strings over .format() or % formatting.", entity_type="guideline")], + "expect": {"n7": "ADD"}, + }, + { + "label": "add_guideline_docstrings", + "old": [], + "new": [_entity("n8", "Write docstrings for all public functions.", entity_type="guideline")], + "expect": {"n8": "ADD"}, + }, + { + "label": "add_preference_dark_mode", + # The new entity could be treated as a brand-new ADD (IDE-specific dark + # mode preference is new information) OR as a DELETE of the old light-mode + # preference followed by an ADD. Both are semantically valid. + "old": [_entity("o9", "User prefers light mode")], + "new": [_entity("n9", "User prefers dark mode in their IDE")], + "expect": {"n9": "ADD", "o9": ["DELETE", "NONE", "UPDATE"]}, + }, + { + "label": "add_skill_docker", + "old": [_entity("o10", "User knows Kubernetes")], + "new": [_entity("n10", "User is proficient with Docker")], + "expect": {"n10": "ADD"}, + }, + { + "label": "add_project_context", + "old": [_entity("o11", "Project uses PostgreSQL")], + "new": [_entity("n11", "Project also uses Redis for caching")], + "expect": {"n11": "ADD"}, + }, + { + "label": "add_team_size", + "old": [_entity("o12", "User works alone on side projects")], + "new": [_entity("n12", "User is part of a 5-person engineering team at work")], + "expect": {"n12": "ADD"}, + }, + { + "label": "add_coding_style", + "old": [_entity("o13", "User follows PEP 8", entity_type="guideline")], + "new": [_entity("n13", "User uses Black for auto-formatting", entity_type="guideline")], + "expect": {"n13": "ADD"}, + }, + { + "label": "add_testing_framework", + "old": [_entity("o14", "User writes unit tests")], + "new": [_entity("n14", "User uses pytest as the testing framework")], + "expect": {"n14": "ADD"}, + }, + { + "label": "add_cloud_provider", + "old": [_entity("o15", "User deploys on AWS")], + "new": [_entity("n15", "User also uses GCP for ML workloads")], + "expect": {"n15": "ADD"}, + }, + # ── NONE scenarios (duplicate / paraphrase) ──────────────────────────── + { + "label": "none_for_duplicate_and_paraphrase", + "old": [ + _entity("g1", "Always use type hints in Python function signatures."), + _entity("g2", "Likes cheese pizza"), + ], + "new": [ + _entity("g1_dup", "Always use type hints in Python function signatures."), + _entity("g2_dup", "Loves cheese pizza"), + ], + "expect": {"g1": "NONE", "g2": "NONE"}, + }, + { + "label": "none_exact_duplicate", + "old": [_entity("o20", "Always use type hints in Python function signatures.", entity_type="guideline")], + "new": [_entity("n20", "Always use type hints in Python function signatures.", entity_type="guideline")], + "expect": {"o20": "NONE"}, + }, + { + "label": "none_paraphrase_pizza", + "old": [_entity("o21", "Likes cheese pizza")], + "new": [_entity("n21", "Loves cheese pizza")], + "expect": {"o21": "NONE"}, + }, + { + "label": "none_paraphrase_engineer", + "old": [_entity("o22", "User is a software engineer")], + "new": [_entity("n22", "The user works as a software engineer")], + "expect": {"o22": "NONE"}, + }, + { + "label": "none_paraphrase_python", + "old": [_entity("o23", "Prefers Python for scripting", entity_type="guideline")], + "new": [_entity("n23", "Python is the preferred scripting language", entity_type="guideline")], + "expect": {"o23": "NONE"}, + }, + { + "label": "none_paraphrase_hiking", + "old": [_entity("o24", "User enjoys hiking on weekends")], + "new": [_entity("n24", "Likes to go hiking during the weekend")], + "expect": {"o24": "NONE"}, + }, + { + "label": "none_paraphrase_dark_mode", + "old": [_entity("o25", "User prefers dark mode in their editor")], + "new": [_entity("n25", "Prefers dark theme in the code editor")], + "expect": {"o25": "NONE"}, + }, + { + "label": "none_paraphrase_remote_work", + "old": [_entity("o26", "User works from home")], + "new": [_entity("n26", "The user is a remote worker")], + "expect": {"o26": "NONE"}, + }, + { + "label": "none_paraphrase_tests", + "old": [_entity("o27", "Write tests for all new features", entity_type="guideline")], + "new": [_entity("n27", "All new features should have tests", entity_type="guideline")], + "expect": {"o27": "NONE"}, + }, + { + "label": "none_paraphrase_git", + "old": [_entity("o28", "Use descriptive commit messages")], + "new": [_entity("n28", "Commit messages should be clear and descriptive")], + "expect": {"o28": "NONE"}, + }, + { + "label": "none_paraphrase_vegetarian", + "old": [_entity("o29", "User does not eat meat")], + "new": [_entity("n29", "User is vegetarian")], + # "does not eat meat" and "is vegetarian" are semantically equivalent, + # but some LLMs may treat "vegetarian" as more specific → UPDATE is also valid + "expect": {"o29": ["NONE", "UPDATE"]}, + }, + # ── UPDATE scenarios (enrichment) ───────────────────────────────────── + { + "label": "update_cricket_and_none_engineer", + "old": [ + _entity("g1", "User likes to play cricket"), + _entity("g2", "User is a software engineer"), + ], + "new": [_entity("n1", "Loves to play cricket with friends on weekends")], + "expect": {"g1": "UPDATE", "g2": "NONE"}, + }, + { + "label": "update_cricket_enriched", + "old": [_entity("o30", "User likes to play cricket")], + "new": [_entity("n30", "Loves to play cricket with friends on weekends")], + "expect": {"o30": "UPDATE"}, + }, + { + "label": "update_python_enriched", + "old": [_entity("o31", "User knows Python")], + "new": [_entity("n31", "User is an expert Python developer with 10 years of experience")], + "expect": {"o31": "UPDATE"}, + }, + { + "label": "update_location_enriched", + "old": [_entity("o32", "User lives in New York")], + "new": [_entity("n32", "User lives in Brooklyn, New York and commutes to Manhattan")], + "expect": {"o32": "UPDATE"}, + }, + { + "label": "update_job_enriched", + "old": [_entity("o33", "User is a software engineer")], + "new": [_entity("n33", "User is a senior software engineer specializing in distributed systems")], + "expect": {"o33": "UPDATE"}, + }, + { + "label": "update_diet_enriched", + "old": [_entity("o34", "User is vegetarian")], + "new": [_entity("n34", "User is a vegan who avoids all animal products")], + "expect": {"o34": "UPDATE"}, + }, + { + "label": "update_guideline_enriched", + "old": [_entity("o35", "Write tests", entity_type="guideline")], + "new": [_entity("n35", "Write unit and integration tests for all new features using pytest", entity_type="guideline")], + "expect": {"o35": "UPDATE"}, + }, + { + "label": "update_team_enriched", + "old": [_entity("o36", "User works in a team")], + "new": [_entity("n36", "User leads a team of 8 engineers across two time zones")], + "expect": {"o36": "UPDATE"}, + }, + { + "label": "update_database_enriched", + "old": [_entity("o37", "Project uses a relational database")], + "new": [_entity("n37", "Project uses PostgreSQL 15 with read replicas for high availability")], + "expect": {"o37": "UPDATE"}, + }, + { + "label": "update_hobby_enriched", + "old": [_entity("o38", "User plays guitar")], + "new": [_entity("n38", "User plays acoustic guitar and performs at local open-mic events")], + "expect": {"o38": "UPDATE"}, + }, + { + "label": "update_language_enriched", + "old": [_entity("o39", "User speaks English")], + "new": [_entity("n39", "User is fluent in English and conversational in Spanish")], + "expect": {"o39": "UPDATE"}, + }, + # ── DELETE scenarios (contradiction) ────────────────────────────────── + { + "label": "delete_pizza_and_none_name", + "old": [ + _entity("g1", "Name is John"), + _entity("g2", "Loves cheese pizza"), + ], + "new": [_entity("n1", "Dislikes cheese pizza")], + "expect": {"g2": "DELETE", "g1": "NONE"}, + }, + { + "label": "delete_pizza_contradiction", + "old": [_entity("o40", "Loves cheese pizza"), _entity("o41", "Name is Bob")], + "new": [_entity("n40", "Dislikes cheese pizza")], + "expect": {"o40": "DELETE", "o41": "NONE"}, + }, + { + "label": "delete_location_contradiction", + "old": [_entity("o42", "User lives in New York"), _entity("o43", "User is a developer")], + "new": [_entity("n42", "User moved to San Francisco")], + "expect": {"o42": "DELETE"}, + }, + { + "label": "delete_diet_contradiction", + "old": [_entity("o44", "User eats meat"), _entity("o45", "User likes burgers")], + "new": [_entity("n44", "User became vegan and no longer eats any animal products")], + # o44 is clearly contradicted. o45 ("likes burgers") is also + # contradicted by veganism, so DELETE is valid; but an LLM might + # consider it a separate, still-true historical preference → NONE. + "expect": {"o44": "DELETE", "o45": ["DELETE", "NONE"]}, + }, + { + "label": "delete_job_contradiction", + "old": [_entity("o46", "User is unemployed"), _entity("o47", "User is looking for a job")], + "new": [_entity("n46", "User started a new job as a data scientist")], + # o46 is clearly contradicted. o47 ("looking for a job") is also + # rendered obsolete, so DELETE is valid; NONE is also defensible if + # the LLM treats it as a historical fact. + "expect": {"o46": "DELETE", "o47": ["DELETE", "NONE"]}, + }, + { + "label": "delete_preference_contradiction", + "old": [_entity("o48", "User prefers tabs for indentation", entity_type="guideline"), _entity("o49", "User uses vim")], + "new": [_entity("n48", "User switched to spaces for indentation", entity_type="guideline")], + # A preference reversal on the same setting may be treated as UPDATE or DELETE. + "expect": {"o48": ["DELETE", "UPDATE"]}, + }, + { + "label": "delete_language_contradiction", + "old": [_entity("o50", "User dislikes JavaScript"), _entity("o51", "User uses Python for everything")], + "new": [_entity("n50", "User now enjoys writing TypeScript for frontend work")], + # o50: TypeScript is a JS superset, so whether this contradicts a JS + # dislike depends on LLM domain knowledge → DELETE or NONE are both valid. + # o51 ("Python for everything") may be partially contradicted → DELETE or NONE. + "expect": {"o50": ["DELETE", "NONE"], "o51": ["DELETE", "NONE"]}, + }, + { + "label": "delete_relationship_contradiction", + "old": [_entity("o52", "User is single"), _entity("o53", "User lives alone")], + "new": [_entity("n52", "User got married last year")], + # o52 is clearly contradicted. o53 ("lives alone") is likely also + # contradicted by marriage, but an LLM might keep it as NONE. + "expect": {"o52": "DELETE", "o53": ["DELETE", "NONE"]}, + }, + { + "label": "delete_os_contradiction", + "old": [_entity("o54", "User uses Windows as their primary OS"), _entity("o55", "User is familiar with PowerShell")], + "new": [_entity("n54", "User switched to macOS as their primary operating system")], + # o54: switching OS is a same-setting change → DELETE or UPDATE are both valid. + # o55 (PowerShell familiarity) is still technically true even on macOS → NONE is most likely, but DELETE is + # also defensible. + "expect": {"o54": ["DELETE", "UPDATE"], "o55": ["NONE", "DELETE"]}, + }, + { + "label": "delete_framework_contradiction", + "old": [_entity("o56", "User prefers Django for web development"), _entity("o57", "User knows SQL")], + "new": [_entity("n56", "User switched to FastAPI and no longer uses Django")], + # o56 is clearly contradicted. o57 ("knows SQL") is unrelated to the + # framework switch → NONE, but an LLM might DELETE it too. + "expect": {"o56": "DELETE", "o57": ["NONE", "DELETE"]}, + }, + { + "label": "delete_sport_contradiction", + "old": [_entity("o58", "User plays football"), _entity("o59", "User watches NFL games")], + "new": [_entity("n58", "User quit football due to an injury and now only swims")], + # o58 is clearly contradicted → DELETE; but an LLM may also UPDATE it + # (the activity changed rather than the entity being wrong). + # o59 ("watches NFL games") may or may not be contradicted. + "expect": {"o58": ["DELETE", "UPDATE"], "o59": ["NONE", "DELETE"]}, + }, + # ── mixed scenarios ───────────── + { + "label": "mixed_update_none_update_add", + "old": [ + _entity("g1", "I really like cheese pizza"), + _entity("g2", "User is a software engineer"), + _entity("g3", "User likes to play cricket"), + ], + "new": [ + _entity("n1", "Loves chicken pizza"), + _entity("n2", "Loves to play cricket with friends"), + _entity("n3", "Name is John"), + ], + # g1: "cheese pizza" → "chicken pizza" is a change in preference. + # An LLM might UPDATE (different pizza type, still pizza lover) or + # DELETE (contradicts cheese preference) or even ADD (separate fact). + # g2: unrelated to any new entity → NONE. + # g3: enriched with more detail → UPDATE. + # n3: brand new fact → ADD. + "expect": {"g1": ["UPDATE", "DELETE", "ADD"], "g2": "NONE", "g3": "UPDATE", "n3": "ADD"}, + }, + # ── Mixed scenarios ──────────────────────────────────────────────────── + { + "label": "mixed_add_and_none", + "old": [ + _entity("m1", "User is a Python developer"), + _entity("m2", "User enjoys hiking"), + ], + "new": [ + _entity("n1", "User is a Python developer"), # paraphrase → NONE + _entity("n2", "User owns a dog"), # brand new → ADD + ], + "expect": {"m1": "NONE", "n2": "ADD"}, + }, + { + "label": "mixed_delete_and_add", + "old": [ + _entity("m1", "User prefers vim as their editor"), + _entity("m2", "User works on Linux"), + ], + "new": [ + _entity("n1", "User switched to VS Code and no longer uses vim"), # contradicts m1 → DELETE + _entity("n2", "User recently adopted macOS"), # brand new → ADD + ], + # m1 is clearly contradicted → DELETE. + # m2 ("works on Linux"): adopting macOS may or may not contradict + # working on Linux (dual-boot / WSL / VM are common) → NONE or DELETE. + "expect": {"m1": "DELETE", "m2": ["NONE", "DELETE"], "n2": "ADD"}, + }, + { + "label": "mixed_update_and_delete", + "old": [ + _entity("m1", "User knows some JavaScript"), + _entity("m2", "User dislikes TypeScript"), + ], + "new": [ + _entity("n1", "User is an expert JavaScript and TypeScript developer"), # enriches m1 → UPDATE, contradicts m2 → DELETE + ], + # m1 is enriched → UPDATE. + # m2 is contradicted → DELETE; but an LLM might also UPDATE it + # (the old dislike is superseded by expertise). + "expect": {"m1": "UPDATE", "m2": ["DELETE", "UPDATE"]}, + }, + { + "label": "mixed_add_update_none", + "old": [ + _entity("m1", "User drinks coffee every morning"), + _entity("m2", "User goes to the gym twice a week"), + _entity("m3", "User reads books"), + ], + "new": [ + _entity("n1", "User drinks two cups of coffee every morning before work"), # enriches m1 → UPDATE + _entity("n2", "User goes to the gym twice a week"), # exact duplicate → NONE + _entity("n3", "User recently started learning Spanish"), # brand new → ADD + ], + "expect": {"m1": "UPDATE", "m2": "NONE", "n3": "ADD"}, + }, + { + "label": "mixed_delete_update_none", + "old": [ + _entity("m1", "User uses MySQL for all projects"), + _entity("m2", "User deploys on Heroku"), + _entity("m3", "User writes backend code in Node.js"), + ], + "new": [ + _entity("n1", "User migrated all projects from MySQL to PostgreSQL"), # contradicts m1 → DELETE + _entity("n2", "User deploys on AWS using ECS"), # enriches m2 → UPDATE + _entity("n3", "User writes backend services in Node.js"), # paraphrase → NONE + ], + "expect": {"m1": "DELETE", "m2": "UPDATE", "m3": "NONE"}, + }, + { + "label": "mixed_all_four_events_guidelines", + "old": [ + _entity("m1", "Use tabs for indentation", entity_type="guideline"), + _entity("m2", "Write tests", entity_type="guideline"), + _entity("m3", "Use snake_case for variable names", entity_type="guideline"), + ], + "new": [ + _entity("n1", "Use spaces (4 per level) for indentation", entity_type="guideline"), # contradicts m1 → DELETE or UPDATE + _entity("n2", "Write unit and integration tests for all new features", entity_type="guideline"), # enriches m2 → UPDATE + _entity("n3", "Use snake_case for variable names", entity_type="guideline"), # exact duplicate → NONE + _entity("n4", "Run ruff before every commit", entity_type="guideline"), # brand new → ADD + ], + # m1: preference reversal on same setting → DELETE or UPDATE are both valid. + "expect": {"m1": ["DELETE", "UPDATE"], "m2": "UPDATE", "m3": "NONE", "n4": "ADD"}, + }, + { + "label": "mixed_personal_facts_all_four_events", + "old": [ + _entity("m1", "User is single"), + _entity("m2", "User lives in Boston"), + _entity("m3", "User has a cat"), + ], + "new": [ + _entity("n1", "User got engaged last month"), # contradicts m1 → DELETE or UPDATE + _entity("n2", "User lives in the South End neighborhood of Boston"), # enriches m2 → UPDATE + _entity("n3", "User has a cat"), # exact duplicate → NONE + _entity("n4", "User recently adopted a rescue dog"), # brand new → ADD + ], + # m1: relationship status change → DELETE or UPDATE are both valid. + "expect": {"m1": ["DELETE", "UPDATE"], "m2": "UPDATE", "m3": "NONE", "n4": "ADD"}, + }, + { + "label": "mixed_tech_stack_update_and_add", + "old": [ + _entity("m1", "Project uses React for the frontend"), + _entity("m2", "Project uses REST APIs"), + _entity("m3", "Project is deployed on a single server"), + ], + "new": [ + _entity("n1", "Project uses React 18 with TypeScript for the frontend"), # enriches m1 → UPDATE + _entity("n2", "Project uses REST APIs"), # exact duplicate → NONE + _entity("n3", "Project migrated to a Kubernetes cluster on AWS"), # contradicts m3 → DELETE + _entity("n4", "Project added GraphQL alongside the REST API"), # brand new → ADD + ], + "expect": {"m1": "UPDATE", "m2": "NONE", "m3": "DELETE", "n4": "ADD"}, + }, + { + "label": "mixed_career_facts_delete_and_update", + "old": [ + _entity("m1", "User is a junior developer"), + _entity("m2", "User works at a startup"), + _entity("m3", "User earns a modest salary"), + ], + "new": [ + _entity("n1", "User was promoted to senior developer"), # contradicts m1 → DELETE + _entity("n2", "User works at a fast-growing Series B startup in fintech"), # enriches m2 → UPDATE + _entity("n3", "User earns a modest salary"), # exact duplicate → NONE + ], + # m1: "junior" is contradicted by "senior" → DELETE; but an LLM might + # also UPDATE (the role changed rather than the entity being wrong). + "expect": {"m1": ["DELETE", "UPDATE"], "m2": "UPDATE", "m3": "NONE"}, + }, + { + "label": "mixed_hobbies_add_and_update", + "old": [ + _entity("m1", "User runs occasionally"), + _entity("m2", "User likes cooking"), + _entity("m3", "User watches documentaries"), + ], + "new": [ + _entity("n1", "User runs a half-marathon every month and trains five days a week"), # enriches m1 → UPDATE + _entity("n2", "User enjoys cooking Italian and Thai food"), # enriches m2 → UPDATE + _entity("n3", "User started playing chess online"), # brand new → ADD + ], + "expect": {"m1": "UPDATE", "m2": "UPDATE", "n3": "ADD"}, + }, + { + "label": "mixed_contradictions_and_new_facts", + "old": [ + _entity("m1", "User prefers working in the morning"), + _entity("m2", "User does not drink alcohol"), + _entity("m3", "User is an introvert"), + ], + "new": [ + _entity("n1", "User is a night owl who does their best work after midnight"), # contradicts m1 → DELETE + _entity("n2", "User does not drink alcohol"), # exact duplicate → NONE + _entity("n3", "User recently joined a public speaking club"), # brand new → ADD + ], + # m1: "morning person" vs "night owl" is a contradiction, but some LLMs + # may treat it as an UPDATE (preference changed) → DELETE or UPDATE are both valid. + # m2 is an exact duplicate → NONE. + # m3 ("introvert"): joining a public speaking club may or may not + # contradict introversion → NONE or DELETE. + # n3 is brand new → ADD. + "expect": {"m1": ["DELETE", "UPDATE"], "m2": "NONE", "m3": ["NONE", "DELETE"], "n3": "ADD"}, + }, + { + "label": "mixed_large_batch_all_four_events", + "old": [ + _entity("m1", "User uses Jira for project management"), + _entity("m2", "User prefers async communication"), + _entity("m3", "User writes documentation in Confluence"), + _entity("m4", "User attends daily standups"), + _entity("m5", "User works in a monorepo"), + ], + "new": [ + _entity("n1", "Team switched from Jira to Linear for project management"), # contradicts m1 → DELETE + _entity("n2", "User strongly prefers async communication over meetings"), # enriches m2 → UPDATE + _entity("n3", "User writes documentation in Confluence"), # exact duplicate → NONE + _entity("n4", "User started using Notion for personal notes"), # brand new → ADD + ], + "expect": {"m1": "DELETE", "m2": "UPDATE", "m3": "NONE", "n4": "ADD"}, + }, +] + + +# --------------------------------------------------------------------------- +# Parameterized test +# --------------------------------------------------------------------------- + + +@pytest.mark.llm +@pytest.mark.parametrize( + "scenario", + [pytest.param(scenario, id=scenario["label"]) for scenario in CONFLICT_SCENARIOS], +) +def test_conflict_resolution_scenarios(scenario: ConflictScenario) -> None: + """Parametrized test covering hundreds of conflict-resolution scenarios. + + Each case exercises a specific combination of old and new entities and + asserts that ``resolve_conflicts`` produces the expected event for every + entity ID listed in ``scenario["expect"]``. + + When ``expect`` maps an entity ID to a *list* of events, any one of those + events is considered a valid LLM answer (used for genuinely ambiguous + scenarios where multiple interpretations are semantically correct). + """ + result_by_id = {update.id: update for update in resolve_conflicts(scenario["old"], scenario["new"])} + for entity_id, expected_event in scenario["expect"].items(): + assert entity_id in result_by_id, f"Expected entity '{entity_id}' not found in result. Got IDs: {list(result_by_id.keys())}" + actual_event = result_by_id[entity_id].event + if isinstance(expected_event, list): + assert actual_event in expected_event, f"Entity '{entity_id}': expected one of {expected_event}, got '{actual_event}'" + else: + assert actual_event == expected_event, f"Entity '{entity_id}': expected event '{expected_event}', got '{actual_event}'" From 270034c04175640154952d8f000e5d748e192995 Mon Sep 17 00:00:00 2001 From: Punleuk Oum Date: Tue, 3 Mar 2026 09:36:53 -0800 Subject: [PATCH 5/6] don't run llm tests by default --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 119f5d1c..1a3b806c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ include = ["kaizen"] package = true [tool.pytest.ini_options] -addopts = "--ignore=explorations -m 'not phoenix'" +addopts = "--ignore=explorations -m 'not phoenix and not llm'" markers = [ "e2e", "unit", From 668ad67eb186b0b920a4db9950c5e9c54654ca67 Mon Sep 17 00:00:00 2001 From: Punleuk Oum Date: Tue, 3 Mar 2026 09:53:51 -0800 Subject: [PATCH 6/6] add test retries --- pyproject.toml | 1 + tests/llm/test_long_conflict_resolution.py | 1 + uv.lock | 14 ++++++++++++++ 3 files changed, 16 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1a3b806c..83ca6e3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ dev = [ "pre-commit", "pytest", "pytest-cov", + "pytest-retry", "python-dotenv>=1.2.1", "python-semantic-release", "ruff", diff --git a/tests/llm/test_long_conflict_resolution.py b/tests/llm/test_long_conflict_resolution.py index 11fef367..e0715a40 100644 --- a/tests/llm/test_long_conflict_resolution.py +++ b/tests/llm/test_long_conflict_resolution.py @@ -604,6 +604,7 @@ class ConflictScenario(TypedDict): @pytest.mark.llm +@pytest.mark.flaky(retries=3, delay=1) @pytest.mark.parametrize( "scenario", [pytest.param(scenario, id=scenario["label"]) for scenario in CONFLICT_SCENARIOS], diff --git a/uv.lock b/uv.lock index 2094f09a..4f7f1511 100644 --- a/uv.lock +++ b/uv.lock @@ -1602,6 +1602,7 @@ dev = [ { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "pytest-retry" }, { name = "python-dotenv" }, { name = "python-semantic-release" }, { name = "ruff" }, @@ -1638,6 +1639,7 @@ dev = [ { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "pytest-retry" }, { name = "python-dotenv", specifier = ">=1.2.1" }, { name = "python-semantic-release" }, { name = "ruff" }, @@ -3304,6 +3306,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, ] +[[package]] +name = "pytest-retry" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/5b/607b017994cca28de3a1ad22a3eee8418e5d428dcd8ec25b26b18e995a73/pytest_retry-1.7.0.tar.gz", hash = "sha256:f8d52339f01e949df47c11ba9ee8d5b362f5824dff580d3870ec9ae0057df80f", size = 19977, upload-time = "2025-01-19T01:56:13.115Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/ff/3266c8a73b9b93c4b14160a7e2b31d1e1088e28ed29f4c2d93ae34093bfd/pytest_retry-1.7.0-py3-none-any.whl", hash = "sha256:a2dac85b79a4e2375943f1429479c65beb6c69553e7dae6b8332be47a60954f4", size = 13775, upload-time = "2025-01-19T01:56:11.199Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"