diff --git a/README.md b/README.md index cd8a9309..080519e5 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,14 @@ npx @modelcontextprotocol/inspector@latest http://127.0.0.1:8201/sse --cli --met - `create_entity(content: str, entity_type: str, metadata: str | None, enable_conflict_resolution: bool)`: Create a single entity in the namespace. - `delete_entity(entity_id: str)`: Delete a specific entity by its ID. +## Tip Provenance + +Kaizen automatically tracks the origin of every guideline it generates or stores. Every tip entity contains `metadata` identifying its source: +- `creation_mode`: Identifies how the tip was created (`auto-phoenix` via trace observability, `auto-mcp` via trajectory saving tools, or `manual`). +- `source_task_id`: The ID of the original trace or task that inspired the tip, providing full audibility. + +See the [Low-Code Tracing Guide](docs/LOW_CODE_TRACING.md#6-understanding-tip-provenance-metadata) for more details. + ## Documentation - [KAIZEN_LITE.md](KAIZEN_LITE.md) - Lightweight mode via Claude Code plugin (no infra required) diff --git a/docs/LOW_CODE_TRACING.md b/docs/LOW_CODE_TRACING.md index 5f3978bf..649f32df 100644 --- a/docs/LOW_CODE_TRACING.md +++ b/docs/LOW_CODE_TRACING.md @@ -212,6 +212,29 @@ KAIZEN_BACKEND=filesystem \ uv run python -m kaizen.frontend.cli.cli entities list kaizen --type guideline ``` +### 6. Understanding Tip Provenance (Metadata) + +When Kaizen generates tips from traced trajectories (or from explicit `save_trajectory` calls), it automatically injects provenance metadata into the resulting `guideline` entities. This helps you track exactly *where* a tip came from and *how* it was created. + +```json +{ + "type": "guideline", + "content": "Always verify the record exists before updating.", + "metadata": { + "creation_mode": "auto-phoenix", + "source_task_id": "0df020ed0bd2e...", + "source_span_id": "9218e1003f...", + "category": "optimization" + } +} +``` + +* **`creation_mode`**: Describes the origin of the tip. + * `"auto-phoenix"`: Auto-generated from observability traces via `kaizen sync phoenix`. + * `"auto-mcp"`: Auto-generated when an agent directly calls the Kaizen `save_trajectory` MCP tool. + * `"manual"`: Explicitly created by a human or agent (e.g., via the `create_entity` MCP tool). +* **`source_task_id`**: The originating trace ID (for Phoenix) or task ID (for MCP), linking the tip back to the specific execution that inspired it. + --- ## End-to-End Verification diff --git a/kaizen/frontend/mcp/mcp_server.py b/kaizen/frontend/mcp/mcp_server.py index 0701f561..eb217b59 100644 --- a/kaizen/frontend/mcp/mcp_server.py +++ b/kaizen/frontend/mcp/mcp_server.py @@ -132,6 +132,8 @@ def save_trajectory(trajectory_data: str, task_id: str | None = None) -> list[Re "rationale": tip.rationale, "trigger": tip.trigger, "task_description": result.task_description, + "source_task_id": task_id, + "creation_mode": "auto-mcp", }, ) for tip in result.tips @@ -169,9 +171,11 @@ def create_entity(content: str, entity_type: str, metadata: str | None = None, e metadata_dict = json.loads(metadata) except json.JSONDecodeError as e: logger.exception(f"Invalid JSON in metadata parameter: {str(e)}") - return json.dumps( - {"error": "Invalid metadata JSON", "message": f"Failed to parse metadata: {str(e)}", "invalid_metadata": metadata} - ) + return json.dumps({"error": "Invalid JSON", "message": f"Failed to parse metadata: {str(e)}", "invalid_metadata": metadata}) + + # Inject creation mode for manually created guidelines/policies if not present + if entity_type in ("guideline", "policy"): + metadata_dict.setdefault("creation_mode", "manual") # Create the entity using the Entity schema entity = Entity(type=entity_type, content=content, metadata=metadata_dict) diff --git a/kaizen/sync/phoenix_sync.py b/kaizen/sync/phoenix_sync.py index 966bfa3e..62c5f7f1 100644 --- a/kaizen/sync/phoenix_sync.py +++ b/kaizen/sync/phoenix_sync.py @@ -480,9 +480,10 @@ def _process_trajectory(self, trajectory: dict) -> int: "category": tip.category, "rationale": tip.rationale, "trigger": tip.trigger, - "source_trace_id": trajectory["trace_id"], + "source_task_id": trajectory["trace_id"], "source_span_id": trajectory["span_id"], "task_description": result.task_description, + "creation_mode": "auto-phoenix", }, ) for tip in result.tips diff --git a/tests/e2e/test_mcp.py b/tests/e2e/test_mcp.py index 9e88836b..8f7a5db0 100644 --- a/tests/e2e/test_mcp.py +++ b/tests/e2e/test_mcp.py @@ -119,6 +119,22 @@ async def test_save_trajectory_and_retrieve_guidelines(mcp): guidelines = response.content[0].text assert "# Guidelines for: " in guidelines + # Verify tip provenance in Kaizen backend + from kaizen.frontend.client.kaizen_client import KaizenClient + from kaizen.config.kaizen import kaizen_config + + client = KaizenClient() + entities = client.search_entities( + namespace_id=kaizen_config.namespace_id, + filters={"type": "guideline"}, + limit=10, + ) + assert len(entities) > 0 + for entity in entities: + metadata = entity.metadata or {} + assert metadata.get("source_task_id") == "123" + assert metadata.get("creation_mode") == "auto-mcp" + @pytest.mark.e2e async def test_create_entity_without_conflict_resolution(mcp): @@ -301,6 +317,6 @@ async def test_create_entity_with_invalid_json_metadata(mcp): # Should return an error assert "error" in result - assert result["error"] == "Invalid metadata JSON" + assert result["error"] == "Invalid JSON" assert "message" in result assert "invalid_metadata" in result diff --git a/tests/unit/test_mcp_server.py b/tests/unit/test_mcp_server.py new file mode 100644 index 00000000..3d2245ec --- /dev/null +++ b/tests/unit/test_mcp_server.py @@ -0,0 +1,98 @@ +import json +import uuid +import pytest +from unittest.mock import patch, MagicMock + +from kaizen.frontend.mcp.mcp_server import save_trajectory, create_entity +from kaizen.schema.conflict_resolution import EntityUpdate + + +@pytest.fixture +def mock_get_client(): + with patch("kaizen.frontend.mcp.mcp_server.get_client") as mock: + client_instance = mock.return_value + yield client_instance + + +def test_save_trajectory_metadata_injection(mock_get_client): + # Mock tip generation to prevent actual LLM calls + with patch("kaizen.frontend.mcp.mcp_server.generate_tips") as mock_generate_tips: + mock_result = MagicMock() + mock_tip = MagicMock() + mock_tip.content = "Always write unit tests" + mock_tip.category = "testing" + mock_tip.rationale = "Helps catch bugs early" + mock_tip.trigger = "writing code" + mock_result.tips = [mock_tip] + mock_result.task_description = "Add feature" + mock_generate_tips.return_value = mock_result + + trajectory_data = json.dumps([{"role": "user", "content": "hi"}]) + task_id = str(uuid.uuid4()) + + save_trajectory.fn(trajectory_data=trajectory_data, task_id=task_id) + + # Ensure update_entities was called twice (once for trajectory, once for tips) + assert mock_get_client.update_entities.call_count == 2 + + # Second call is for tips + call_args = mock_get_client.update_entities.call_args_list[1][1] + entities = call_args["entities"] + + assert len(entities) == 1 + tip_entity = entities[0] + assert tip_entity.type == "guideline" + assert tip_entity.metadata["source_task_id"] == task_id + assert tip_entity.metadata["creation_mode"] == "auto-mcp" + + +def test_create_entity_metadata_injection_manual_guideline(mock_get_client): + mock_update = EntityUpdate(id="123", type="guideline", content="docstrings", event="ADD", metadata={"creation_mode": "manual"}) + mock_get_client.update_entities.return_value = [mock_update] + + # Missing explicit metadata, should auto-inject "manual" + result_str = create_entity.fn(content="Write clear docstrings", entity_type="guideline") + result = json.loads(result_str) + assert result["event"] == "ADD" + assert "id" in result + + call_args = mock_get_client.update_entities.call_args[1] + entities = call_args["entities"] + assert len(entities) == 1 + entity = entities[0] + + assert entity.type == "guideline" + assert entity.metadata["creation_mode"] == "manual" + + +def test_create_entity_metadata_injection_manual_policy(mock_get_client): + mock_update = EntityUpdate(id="123", type="policy", content="PR reviews", event="ADD", metadata={"creation_mode": "manual"}) + mock_get_client.update_entities.return_value = [mock_update] + + result_str = create_entity.fn(content="Require PR reviews", entity_type="policy") + result = json.loads(result_str) + assert result["event"] == "ADD" + + call_args = mock_get_client.update_entities.call_args[1] + entities = call_args["entities"] + entity = entities[0] + + assert entity.type == "policy" + assert entity.metadata["creation_mode"] == "manual" + + +def test_create_entity_no_metadata_injection_for_other_types(mock_get_client): + mock_update = EntityUpdate(id="123", type="log", content="App started", event="ADD", metadata={}) + mock_get_client.update_entities.return_value = [mock_update] + + # A generic log entity shouldn't get creation_mode injected + result_str = create_entity.fn(content="App started", entity_type="log") + result = json.loads(result_str) + assert result["event"] == "ADD" + + call_args = mock_get_client.update_entities.call_args[1] + entities = call_args["entities"] + entity = entities[0] + + assert entity.type == "log" + assert "creation_mode" not in (entity.metadata or {}) diff --git a/tests/unit/test_phoenix_sync.py b/tests/unit/test_phoenix_sync.py index 604ef568..a843577a 100644 --- a/tests/unit/test_phoenix_sync.py +++ b/tests/unit/test_phoenix_sync.py @@ -578,10 +578,13 @@ def test_sync_processes_valid_spans(self, mock_generate_tips, mock_urlopen, phoe assert result.tips_generated == 2 phoenix_sync.client.update_entities.assert_called() - # Verify task_description is persisted in tip entity metadata + # Verify provenance metadata is persisted in tip entities tip_update_call = phoenix_sync.client.update_entities.call_args_list[-1] tip_entities = tip_update_call.kwargs["entities"] assert all(e.metadata.get("task_description") == "Hello" for e in tip_entities) + assert all(e.metadata.get("source_task_id") == "t1" for e in tip_entities) + assert all(e.metadata.get("source_span_id") == "s1" for e in tip_entities) + assert all(e.metadata.get("creation_mode") == "auto-phoenix" for e in tip_entities) @patch("kaizen.sync.phoenix_sync.urllib.request.urlopen") @patch("kaizen.sync.phoenix_sync.generate_tips")