From 769d4fc6fcc740264ffdb1273dc6a9116d117e81 Mon Sep 17 00:00:00 2001 From: Vatche Isahagian Date: Tue, 24 Feb 2026 01:00:24 -0500 Subject: [PATCH 1/2] feat: Add tip provenance tracking and metadata - Injects source_task_id and creation_mode to tip entities via save_trajectory and MCP create_entity. - Updates phoenix_sync to standardize source_task_id schema. - Adds comprehensive unit and E2E tests for metadata verification. - Updates main README and LOW_CODE_TRACING docs for tip provenance schema. --- README.md | 8 +++ docs/LOW_CODE_TRACING.md | 23 +++++++ kaizen/frontend/mcp/mcp_server.py | 8 ++- kaizen/sync/phoenix_sync.py | 3 +- tests/e2e/test_mcp.py | 17 ++++- tests/unit/test_mcp_server.py | 102 ++++++++++++++++++++++++++++++ tests/unit/test_phoenix_sync.py | 5 +- 7 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_mcp_server.py diff --git a/README.md b/README.md index cd8a9309..080519e5 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,14 @@ npx @modelcontextprotocol/inspector@latest http://127.0.0.1:8201/sse --cli --met - `create_entity(content: str, entity_type: str, metadata: str | None, enable_conflict_resolution: bool)`: Create a single entity in the namespace. - `delete_entity(entity_id: str)`: Delete a specific entity by its ID. +## Tip Provenance + +Kaizen automatically tracks the origin of every guideline it generates or stores. Every tip entity contains `metadata` identifying its source: +- `creation_mode`: Identifies how the tip was created (`auto-phoenix` via trace observability, `auto-mcp` via trajectory saving tools, or `manual`). +- `source_task_id`: The ID of the original trace or task that inspired the tip, providing full audibility. + +See the [Low-Code Tracing Guide](docs/LOW_CODE_TRACING.md#6-understanding-tip-provenance-metadata) for more details. + ## Documentation - [KAIZEN_LITE.md](KAIZEN_LITE.md) - Lightweight mode via Claude Code plugin (no infra required) diff --git a/docs/LOW_CODE_TRACING.md b/docs/LOW_CODE_TRACING.md index 5f3978bf..649f32df 100644 --- a/docs/LOW_CODE_TRACING.md +++ b/docs/LOW_CODE_TRACING.md @@ -212,6 +212,29 @@ KAIZEN_BACKEND=filesystem \ uv run python -m kaizen.frontend.cli.cli entities list kaizen --type guideline ``` +### 6. Understanding Tip Provenance (Metadata) + +When Kaizen generates tips from traced trajectories (or from explicit `save_trajectory` calls), it automatically injects provenance metadata into the resulting `guideline` entities. This helps you track exactly *where* a tip came from and *how* it was created. + +```json +{ + "type": "guideline", + "content": "Always verify the record exists before updating.", + "metadata": { + "creation_mode": "auto-phoenix", + "source_task_id": "0df020ed0bd2e...", + "source_span_id": "9218e1003f...", + "category": "optimization" + } +} +``` + +* **`creation_mode`**: Describes the origin of the tip. + * `"auto-phoenix"`: Auto-generated from observability traces via `kaizen sync phoenix`. + * `"auto-mcp"`: Auto-generated when an agent directly calls the Kaizen `save_trajectory` MCP tool. + * `"manual"`: Explicitly created by a human or agent (e.g., via the `create_entity` MCP tool). +* **`source_task_id`**: The originating trace ID (for Phoenix) or task ID (for MCP), linking the tip back to the specific execution that inspired it. + --- ## End-to-End Verification diff --git a/kaizen/frontend/mcp/mcp_server.py b/kaizen/frontend/mcp/mcp_server.py index 0701f561..038ae3c6 100644 --- a/kaizen/frontend/mcp/mcp_server.py +++ b/kaizen/frontend/mcp/mcp_server.py @@ -132,6 +132,8 @@ def save_trajectory(trajectory_data: str, task_id: str | None = None) -> list[Re "rationale": tip.rationale, "trigger": tip.trigger, "task_description": result.task_description, + "source_task_id": task_id, + "creation_mode": "auto-mcp", }, ) for tip in result.tips @@ -170,9 +172,13 @@ def create_entity(content: str, entity_type: str, metadata: str | None = None, e except json.JSONDecodeError as e: logger.exception(f"Invalid JSON in metadata parameter: {str(e)}") return json.dumps( - {"error": "Invalid metadata JSON", "message": f"Failed to parse metadata: {str(e)}", "invalid_metadata": metadata} + {"error": "Invalid JSON", "message": f"Failed to parse metadata: {str(e)}", "invalid_metadata": metadata} ) + # Inject creation mode for manually created guidelines/policies if not present + if entity_type in ("guideline", "policy"): + metadata_dict.setdefault("creation_mode", "manual") + # Create the entity using the Entity schema entity = Entity(type=entity_type, content=content, metadata=metadata_dict) diff --git a/kaizen/sync/phoenix_sync.py b/kaizen/sync/phoenix_sync.py index 966bfa3e..62c5f7f1 100644 --- a/kaizen/sync/phoenix_sync.py +++ b/kaizen/sync/phoenix_sync.py @@ -480,9 +480,10 @@ def _process_trajectory(self, trajectory: dict) -> int: "category": tip.category, "rationale": tip.rationale, "trigger": tip.trigger, - "source_trace_id": trajectory["trace_id"], + "source_task_id": trajectory["trace_id"], "source_span_id": trajectory["span_id"], "task_description": result.task_description, + "creation_mode": "auto-phoenix", }, ) for tip in result.tips diff --git a/tests/e2e/test_mcp.py b/tests/e2e/test_mcp.py index 9e88836b..0a8aa2f1 100644 --- a/tests/e2e/test_mcp.py +++ b/tests/e2e/test_mcp.py @@ -119,6 +119,21 @@ async def test_save_trajectory_and_retrieve_guidelines(mcp): guidelines = response.content[0].text assert "# Guidelines for: " in guidelines + # Verify tip provenance in Kaizen backend + from kaizen.frontend.client.kaizen_client import KaizenClient + from kaizen.config.kaizen import kaizen_config + client = KaizenClient() + entities = client.search_entities( + namespace_id=kaizen_config.namespace_id, + filters={"type": "guideline"}, + limit=10, + ) + assert len(entities) > 0 + for entity in entities: + metadata = entity.metadata or {} + assert metadata.get("source_task_id") == "123" + assert metadata.get("creation_mode") == "auto-mcp" + @pytest.mark.e2e async def test_create_entity_without_conflict_resolution(mcp): @@ -301,6 +316,6 @@ async def test_create_entity_with_invalid_json_metadata(mcp): # Should return an error assert "error" in result - assert result["error"] == "Invalid metadata JSON" + assert result["error"] == "Invalid JSON" assert "message" in result assert "invalid_metadata" in result diff --git a/tests/unit/test_mcp_server.py b/tests/unit/test_mcp_server.py new file mode 100644 index 00000000..46c9382b --- /dev/null +++ b/tests/unit/test_mcp_server.py @@ -0,0 +1,102 @@ +import json +import uuid +import pytest +from unittest.mock import patch, MagicMock + +from kaizen.frontend.mcp.mcp_server import save_trajectory, create_entity +from kaizen.schema.conflict_resolution import EntityUpdate + +@pytest.fixture +def mock_get_client(): + with patch("kaizen.frontend.mcp.mcp_server.get_client") as mock: + client_instance = mock.return_value + yield client_instance + +def test_save_trajectory_metadata_injection(mock_get_client): + # Mock tip generation to prevent actual LLM calls + with patch("kaizen.frontend.mcp.mcp_server.generate_tips") as mock_generate_tips: + mock_result = MagicMock() + mock_tip = MagicMock() + mock_tip.content = "Always write unit tests" + mock_tip.category = "testing" + mock_tip.rationale = "Helps catch bugs early" + mock_tip.trigger = "writing code" + mock_result.tips = [mock_tip] + mock_result.task_description = "Add feature" + mock_generate_tips.return_value = mock_result + + trajectory_data = json.dumps([{"role": "user", "content": "hi"}]) + task_id = str(uuid.uuid4()) + + save_trajectory.fn(trajectory_data=trajectory_data, task_id=task_id) + + # Ensure update_entities was called twice (once for trajectory, once for tips) + assert mock_get_client.update_entities.call_count == 2 + + # Second call is for tips + call_args = mock_get_client.update_entities.call_args_list[1][1] + entities = call_args["entities"] + + assert len(entities) == 1 + tip_entity = entities[0] + assert tip_entity.type == "guideline" + assert tip_entity.metadata["source_task_id"] == task_id + assert tip_entity.metadata["creation_mode"] == "auto-mcp" + +def test_create_entity_metadata_injection_manual_guideline(mock_get_client): + mock_update = EntityUpdate(id="123", type="guideline", content="docstrings", event="ADD", metadata={"creation_mode": "manual"}) + mock_get_client.update_entities.return_value = [mock_update] + + # Missing explicit metadata, should auto-inject "manual" + result_str = create_entity.fn( + content="Write clear docstrings", + entity_type="guideline" + ) + result = json.loads(result_str) + assert result["event"] == "ADD" + assert "id" in result + + call_args = mock_get_client.update_entities.call_args[1] + entities = call_args["entities"] + assert len(entities) == 1 + entity = entities[0] + + assert entity.type == "guideline" + assert entity.metadata["creation_mode"] == "manual" + +def test_create_entity_metadata_injection_manual_policy(mock_get_client): + mock_update = EntityUpdate(id="123", type="policy", content="PR reviews", event="ADD", metadata={"creation_mode": "manual"}) + mock_get_client.update_entities.return_value = [mock_update] + + result_str = create_entity.fn( + content="Require PR reviews", + entity_type="policy" + ) + result = json.loads(result_str) + assert result["event"] == "ADD" + + call_args = mock_get_client.update_entities.call_args[1] + entities = call_args["entities"] + entity = entities[0] + + assert entity.type == "policy" + assert entity.metadata["creation_mode"] == "manual" + +def test_create_entity_no_metadata_injection_for_other_types(mock_get_client): + mock_update = EntityUpdate(id="123", type="log", content="App started", event="ADD", metadata={}) + mock_get_client.update_entities.return_value = [mock_update] + + # A generic log entity shouldn't get creation_mode injected + result_str = create_entity.fn( + content="App started", + entity_type="log" + ) + result = json.loads(result_str) + assert result["event"] == "ADD" + + call_args = mock_get_client.update_entities.call_args[1] + entities = call_args["entities"] + entity = entities[0] + + assert entity.type == "log" + assert "creation_mode" not in (entity.metadata or {}) diff --git a/tests/unit/test_phoenix_sync.py b/tests/unit/test_phoenix_sync.py index 604ef568..a843577a 100644 --- a/tests/unit/test_phoenix_sync.py +++ b/tests/unit/test_phoenix_sync.py @@ -578,10 +578,13 @@ def test_sync_processes_valid_spans(self, mock_generate_tips, mock_urlopen, phoe assert result.tips_generated == 2 phoenix_sync.client.update_entities.assert_called() - # Verify task_description is persisted in tip entity metadata + # Verify provenance metadata is persisted in tip entities tip_update_call = phoenix_sync.client.update_entities.call_args_list[-1] tip_entities = tip_update_call.kwargs["entities"] assert all(e.metadata.get("task_description") == "Hello" for e in tip_entities) + assert all(e.metadata.get("source_task_id") == "t1" for e in tip_entities) + assert all(e.metadata.get("source_span_id") == "s1" for e in tip_entities) + assert all(e.metadata.get("creation_mode") == "auto-phoenix" for e in tip_entities) @patch("kaizen.sync.phoenix_sync.urllib.request.urlopen") @patch("kaizen.sync.phoenix_sync.generate_tips") From 1653652c9398a82a06ede0e672171223c9f8de10 Mon Sep 17 00:00:00 2001 From: Vatche Isahagian Date: Tue, 24 Feb 2026 09:10:01 -0500 Subject: [PATCH 2/2] style: format files with ruff --- kaizen/frontend/mcp/mcp_server.py | 4 +-- tests/e2e/test_mcp.py | 1 + tests/unit/test_mcp_server.py | 42 ++++++++++++++----------------- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/kaizen/frontend/mcp/mcp_server.py b/kaizen/frontend/mcp/mcp_server.py index 038ae3c6..eb217b59 100644 --- a/kaizen/frontend/mcp/mcp_server.py +++ b/kaizen/frontend/mcp/mcp_server.py @@ -171,9 +171,7 @@ def create_entity(content: str, entity_type: str, metadata: str | None = None, e metadata_dict = json.loads(metadata) except json.JSONDecodeError as e: logger.exception(f"Invalid JSON in metadata parameter: {str(e)}") - return json.dumps( - {"error": "Invalid JSON", "message": f"Failed to parse metadata: {str(e)}", "invalid_metadata": metadata} - ) + return json.dumps({"error": "Invalid JSON", "message": f"Failed to parse metadata: {str(e)}", "invalid_metadata": metadata}) # Inject creation mode for manually created guidelines/policies if not present if entity_type in ("guideline", "policy"): diff --git a/tests/e2e/test_mcp.py b/tests/e2e/test_mcp.py index 0a8aa2f1..8f7a5db0 100644 --- a/tests/e2e/test_mcp.py +++ b/tests/e2e/test_mcp.py @@ -122,6 +122,7 @@ async def test_save_trajectory_and_retrieve_guidelines(mcp): # Verify tip provenance in Kaizen backend from kaizen.frontend.client.kaizen_client import KaizenClient from kaizen.config.kaizen import kaizen_config + client = KaizenClient() entities = client.search_entities( namespace_id=kaizen_config.namespace_id, diff --git a/tests/unit/test_mcp_server.py b/tests/unit/test_mcp_server.py index 46c9382b..3d2245ec 100644 --- a/tests/unit/test_mcp_server.py +++ b/tests/unit/test_mcp_server.py @@ -6,12 +6,14 @@ from kaizen.frontend.mcp.mcp_server import save_trajectory, create_entity from kaizen.schema.conflict_resolution import EntityUpdate + @pytest.fixture def mock_get_client(): with patch("kaizen.frontend.mcp.mcp_server.get_client") as mock: client_instance = mock.return_value yield client_instance + def test_save_trajectory_metadata_injection(mock_get_client): # Mock tip generation to prevent actual LLM calls with patch("kaizen.frontend.mcp.mcp_server.generate_tips") as mock_generate_tips: @@ -32,71 +34,65 @@ def test_save_trajectory_metadata_injection(mock_get_client): # Ensure update_entities was called twice (once for trajectory, once for tips) assert mock_get_client.update_entities.call_count == 2 - + # Second call is for tips call_args = mock_get_client.update_entities.call_args_list[1][1] entities = call_args["entities"] - + assert len(entities) == 1 tip_entity = entities[0] assert tip_entity.type == "guideline" assert tip_entity.metadata["source_task_id"] == task_id assert tip_entity.metadata["creation_mode"] == "auto-mcp" + def test_create_entity_metadata_injection_manual_guideline(mock_get_client): mock_update = EntityUpdate(id="123", type="guideline", content="docstrings", event="ADD", metadata={"creation_mode": "manual"}) mock_get_client.update_entities.return_value = [mock_update] - + # Missing explicit metadata, should auto-inject "manual" - result_str = create_entity.fn( - content="Write clear docstrings", - entity_type="guideline" - ) + result_str = create_entity.fn(content="Write clear docstrings", entity_type="guideline") result = json.loads(result_str) assert result["event"] == "ADD" assert "id" in result - + call_args = mock_get_client.update_entities.call_args[1] entities = call_args["entities"] assert len(entities) == 1 entity = entities[0] - + assert entity.type == "guideline" assert entity.metadata["creation_mode"] == "manual" + def test_create_entity_metadata_injection_manual_policy(mock_get_client): mock_update = EntityUpdate(id="123", type="policy", content="PR reviews", event="ADD", metadata={"creation_mode": "manual"}) mock_get_client.update_entities.return_value = [mock_update] - - result_str = create_entity.fn( - content="Require PR reviews", - entity_type="policy" - ) + + result_str = create_entity.fn(content="Require PR reviews", entity_type="policy") result = json.loads(result_str) assert result["event"] == "ADD" - + call_args = mock_get_client.update_entities.call_args[1] entities = call_args["entities"] entity = entities[0] - + assert entity.type == "policy" assert entity.metadata["creation_mode"] == "manual" + def test_create_entity_no_metadata_injection_for_other_types(mock_get_client): mock_update = EntityUpdate(id="123", type="log", content="App started", event="ADD", metadata={}) mock_get_client.update_entities.return_value = [mock_update] - + # A generic log entity shouldn't get creation_mode injected - result_str = create_entity.fn( - content="App started", - entity_type="log" - ) + result_str = create_entity.fn(content="App started", entity_type="log") result = json.loads(result_str) assert result["event"] == "ADD" - + call_args = mock_get_client.update_entities.call_args[1] entities = call_args["entities"] entity = entities[0] - + assert entity.type == "log" assert "creation_mode" not in (entity.metadata or {})