docs: add MemoryBench quantitative results to README

.github/workflows/ci.yml

@@ -64,11 +64,13 @@ jobs:
           python evals/runners/run_scholar_pipeline_smoke.py
           python evals/runners/run_track_pipeline_smoke.py
           python evals/runners/run_eventlog_replay_smoke.py
+          python evals/runners/run_retrieval_benchmark_smoke.py
 
       - name: Run memory range and acceptance testing
         env:
           PYTHONPATH: src
         run: |
           python evals/memory/test_deletion_compliance.py
           python evals/memory/test_retrieval_hit_rate.py
-
+          python evals/memory/test_scope_isolation.py
+          python evals/memory/test_injection_robustness.py

Makefile

@@ -0,0 +1,8 @@
+.PHONY: bench-roi
+
+bench-roi:
+	@if [ -z "$$OPENAI_API_KEY$$ANTHROPIC_API_KEY$$OPENROUTER_API_KEY$$NVIDIA_MINIMAX_API_KEY$$NVIDIA_GLM_API_KEY" ]; then \
+		echo "Missing API key: set OPENAI_API_KEY, ANTHROPIC_API_KEY, OPENROUTER_API_KEY, NVIDIA_MINIMAX_API_KEY, or NVIDIA_GLM_API_KEY"; \
+		exit 1; \
+	fi
+	PYTHONPATH=src python evals/memory/bench_roi.py --output evals/reports/memory_roi_benchmark.json

README.md

@@ -1,17 +1,27 @@
 <!-- markdownlint-disable MD001 MD041 -->
 
-<h1 align="center">PaperBot</h1>
+<h1 align="center">Oh, God! My idea comes true.</h1>
 
 <h3 align="center">
 AI-powered research workflow: paper discovery → LLM analysis → scholar tracking → Paper2Code → multi-agent studio
 </h3>
 
 <p align="center">
-  <a href="https://github.com/jerry609/PaperBot/actions"><img src="https://github.com/jerry609/PaperBot/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
-  <a href="https://github.com/jerry609/PaperBot/issues/232"><img src="https://img.shields.io/badge/roadmap-Q1%2FQ2%202026-blue" alt="Roadmap"></a>
-  <a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-green.svg" alt="License: MIT"></a>
-  <img src="https://img.shields.io/badge/python-3.10%2B-blue" alt="Python 3.10+">
-  <img src="https://img.shields.io/badge/Next.js-16-black" alt="Next.js 16">
+  <a href="https://github.com/jerry609/PaperBot/actions">
+    <img src="https://img.shields.io/github/actions/workflow/status/jerry609/PaperBot/ci.yml?branch=dev&label=CI&logo=github" alt="CI">
+  </a>
+  <a href="https://github.com/jerry609/PaperBot/issues/232">
+    <img src="https://img.shields.io/badge/roadmap-2026-blue?logo=roadmap" alt="Roadmap">
+  </a>
+  <img src="https://img.shields.io/badge/version-0.1.0-007ec6?logo=github" alt="Version 0.1.0">
+  <img src="https://img.shields.io/badge/license-MIT-green" alt="License">
+</p>
+
+<p align="center">
+  <img src="https://img.shields.io/badge/Python-3.10+-blue?logo=python&logoColor=white" alt="Python">
+  <img src="https://img.shields.io/badge/Next.js-16-black?logo=nextdotjs" alt="Next.js">
+  <img src="https://img.shields.io/badge/Platform-Win%20|%20Mac%20|%20Linux-lightgrey" alt="Platform">
+  <img src="https://img.shields.io/badge/Downloads-xx-lightgrey?style=social&logo=icloud" alt="Downloads">
 </p>
 
 <p align="center">
@@ -26,7 +36,7 @@ AI-powered research workflow: paper discovery → LLM analysis → scholar track
 
 ## About
 
-PaperBot is an end-to-end research assistant that automates the paper discovery → analysis → reproduction pipeline. It combines multi-source search, LLM-powered evaluation, scholar tracking, and code generation into a unified workflow with Web, CLI, and API interfaces.
+"Oh, God! My idea comes true." is an end-to-end research assistant that automates the paper discovery → analysis → reproduction pipeline. It combines multi-source search, LLM-powered evaluation, scholar tracking, and code generation into a unified workflow with Web, CLI, and API interfaces.
 
 **Backend** Python + FastAPI (SSE streaming) · **Frontend** Next.js + Ink CLI · **Sources** arXiv / Semantic Scholar / OpenAlex / HuggingFace Daily Papers / papers.cool
 
@@ -71,6 +81,7 @@ PaperBot is an end-to-end research assistant that automates the paper discovery
 - **Structured Cards** — LLM-extracted method / dataset / conclusion / limitations with DB caching
 - **Related Work** — Draft generation from saved papers with [AuthorYear] citation format
 - **Memory System** — Research memory with FTS5 + BM25 search, context engine for personalized recommendations
+- **MemoryBench Suite** — Retrieval / context / isolation / injection / performance / ROI / effectiveness benchmarks for the memory and Paper2Code stack
 
 ### Reproduction & Studio
 
@@ -84,6 +95,7 @@ PaperBot is an end-to-end research assistant that automates the paper discovery
 ### Install
 
 ```bash
+# Use python3 for macOS/Linux
 python -m venv .venv && source .venv/bin/activate
 pip install -e .
 ```
@@ -136,6 +148,7 @@ PAPERBOT_NOTIFY_EMAIL_TO=recipient@example.com
 alembic upgrade head
 
 # API server
+# Use python3 for macOS/Linux
 python -m uvicorn src.paperbot.api.main:app --reload --port 8000
 
 # Web dashboard (separate terminal)
@@ -184,6 +197,77 @@ python main.py review --title "..." --abstract "..."
 | **Usable** | Scholar Tracking · Deep Review · Paper2Code · Memory · Context Engine · Discovery · AgentSwarm · Harvest · Import/Sync |
 | **Planned** | [DB Modernization #231](https://github.com/jerry609/PaperBot/issues/231) · [Obsidian Integration #159](https://github.com/jerry609/PaperBot/issues/159) |
 
+## MemoryBench Evaluation
+
+> Aligned with [LongMemEval](https://arxiv.org/abs/2407.15460) (ICLR 2025), [LoCoMo](https://arxiv.org/abs/2402.09281) (ACL 2024), Mem0, Letta. Full methodology: [`evals/memory/README.md`](evals/memory/README.md) · Epic [#283](https://github.com/jerry609/PaperBot/issues/283)
+
+<details open>
+<summary><b>Retrieval Quality</b> — 40 queries, 45 memories, 2 users (FTS5 + BM25)</summary>
+
+| Metric | Target | Result | |
+|--------|--------|--------|-|
+| Recall@5 | ≥ 0.80 | **0.873** | :white_check_mark: |
+| MRR@10 | ≥ 0.65 | **0.731** | :white_check_mark: |
+| nDCG@10 | ≥ 0.70 | **0.747** | :white_check_mark: |
+| Hit@10 | — | 1.000 | |
+
+Breakdown by LoCoMo question type:
+
+| Type | Recall@5 | MRR@10 |
+|------|----------|--------|
+| single-hop (24) | 0.931 | 0.770 |
+| multi-hop (6) | 0.708 | 0.583 |
+| temporal (2) | 1.000 | 0.417 |
+| acronym (4) | 0.708 | 0.875 |
+
+</details>
+
+<details>
+<summary><b>Scope Isolation + CRUD</b> — zero-leak enforcement, Mem0 lifecycle</summary>
+
+| Check | Result |
+|-------|--------|
+| Cross-user leak rate | **0** (zero tolerance) |
+| Cross-scope leak rate | **0** (zero tolerance) |
+| CRUD Update (old content gone) | PASS |
+| CRUD Delete (soft-delete enforced) | PASS |
+| CRUD Dedup (exact duplicate skipped) | PASS |
+
+</details>
+
+<details>
+<summary><b>Context Extraction</b> — L0-L3 layer assembly, Letta alignment</summary>
+
+| Test | Result |
+|------|--------|
+| Layer completeness (L0 profile → L3 paper) | 8/8 PASS |
+| Graceful degradation (missing paper / empty user) | 3/3 PASS |
+| Context precision (query → relevant memories) | **100%** (3/3) |
+| Token budget guard (300 token cap) | **215 tokens** |
+| TrackRouter accuracy (query → correct track) | **100%** (5/5) |
+
+</details>
+
+<details>
+<summary><b>Injection Robustness</b> — offline pattern detection</summary>
+
+| Metric | Target | Result |
+|--------|--------|--------|
+| Pollution rate (missed malicious) | ≤ 2% | **0.0%** (6/6 caught) |
+| False positive rate (benign flagged) | — | **0.0%** (0/6 flagged) |
+
+Covers: instruction override, tag escape, special token injection, role hijack, Unicode bypass, privilege escalation.
+
+</details>
+
+```bash
+# Run full MemoryBench suite (~6s, fully offline, no API keys needed)
+PYTHONPATH=src pytest -q evals/memory/test_retrieval_bench.py \
+  evals/memory/test_scope_isolation.py \
+  evals/memory/test_context_extraction.py \
+  evals/memory/test_injection_robustness.py -s
+```
+
 ## Roadmap
 
 > **[Roadmap #232](https://github.com/jerry609/PaperBot/issues/232)** — Living roadmap organized by functional area, with checkbox tracking and Epic links.
@@ -197,6 +281,7 @@ Active Epics:
 | [#153](https://github.com/jerry609/PaperBot/issues/153) | Memory & Context | P0-P1 done |
 | [#154](https://github.com/jerry609/PaperBot/issues/154) | Agentic Research | Design done |
 | [#179](https://github.com/jerry609/PaperBot/issues/179) | Daily Push | Complete |
+| [#283](https://github.com/jerry609/PaperBot/issues/283) | MemoryBench | Complete |
 | [#159](https://github.com/jerry609/PaperBot/issues/159) | Obsidian CLI | Not started |
 
 ## Contributing
@@ -222,6 +307,13 @@ python -m black . && python -m isort .
 | [`docs/PLAN.md`](docs/PLAN.md) | Architecture assessment |
 | [`docs/PAPERSCOOL_WORKFLOW.md`](docs/PAPERSCOOL_WORKFLOW.md) | Topic Workflow guide |
 | [`docs/p2c/`](docs/p2c/) | Paper2Context design docs |
+| [`docs/benchmark/MEMORYBENCH_EPIC_283_COMPLETION.md`](docs/benchmark/MEMORYBENCH_EPIC_283_COMPLETION.md) | MemoryBench Epic completion report |
+| [`docs/benchmark/MEMORYBENCH_RUNTIME_REPORT_2026-03-07.md`](docs/benchmark/MEMORYBENCH_RUNTIME_REPORT_2026-03-07.md) | Live ROI + 1M memory runtime report |
+| [`docs/search_eval.md`](docs/search_eval.md) | Retrieval benchmark guide |
+| [`docs/context_engine_eval.md`](docs/context_engine_eval.md) | Context extraction benchmark guide |
+| [`docs/memory_performance_eval.md`](docs/memory_performance_eval.md) | Memory performance benchmark guide |
+| [`docs/p2c/P2C_ROI_BENCHMARK.md`](docs/p2c/P2C_ROI_BENCHMARK.md) | ROI benchmark guide |
+| [`docs/memory_effectiveness_eval.md`](docs/memory_effectiveness_eval.md) | Multi-session memory effectiveness benchmark guide |
 | [`docs/memory_system.md`](docs/memory_system.md) | Memory system design |
 | [`docs/anchor_system.md`](docs/anchor_system.md) | Anchor author system |
 | [`docs/AGENTIC_RESEARCH_EVOLUTION.md`](docs/AGENTIC_RESEARCH_EVOLUTION.md) | Agentic Research evolution plan |

alembic/versions/0019_memory_fts5.py

@@ -0,0 +1,89 @@
+"""memory_items FTS5 full-text index
+
+Revision ID: 0019_memory_fts5
+Revises: b94c1a2be26e
+Create Date: 2026-03-03
+
+Creates a SQLite FTS5 virtual table for memory_items.content and three triggers
+(after insert / after delete / after update) to keep it in sync.
+
+FTS5 provides BM25-ranked full-text search; falls back to LIKE on non-SQLite
+databases (Postgres) where FTS5 is not available.
+"""
+from __future__ import annotations
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0019_memory_fts5"
+down_revision = "b94c1a2be26e"
+branch_labels = None
+depends_on = None
+
+# --------------------------------------------------------------------------- #
+# SQLite-only helpers                                                          #
+# --------------------------------------------------------------------------- #
+
+_CREATE_FTS = """\
+CREATE VIRTUAL TABLE IF NOT EXISTS memory_items_fts
+USING fts5(content, tokenize='porter ascii');
+"""
+
+_POPULATE_FTS = """\
+INSERT INTO memory_items_fts(rowid, content)
+SELECT id, content FROM memory_items
+WHERE deleted_at IS NULL;
+"""
+
+_TRIGGER_INSERT = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_fts_ai
+AFTER INSERT ON memory_items BEGIN
+    INSERT INTO memory_items_fts(rowid, content) VALUES (new.id, new.content);
+END;
+"""
+
+_TRIGGER_DELETE = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_fts_ad
+AFTER DELETE ON memory_items BEGIN
+    DELETE FROM memory_items_fts WHERE rowid = old.id;
+END;
+"""
+
+_TRIGGER_UPDATE = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_fts_au
+AFTER UPDATE OF content ON memory_items BEGIN
+    DELETE FROM memory_items_fts WHERE rowid = old.id;
+    INSERT INTO memory_items_fts(rowid, content) VALUES (new.id, new.content);
+END;
+"""
+
+_DROP_TRIGGERS = """\
+DROP TRIGGER IF EXISTS memory_items_fts_ai;
+DROP TRIGGER IF EXISTS memory_items_fts_ad;
+DROP TRIGGER IF EXISTS memory_items_fts_au;
+"""
+
+_DROP_FTS = "DROP TABLE IF EXISTS memory_items_fts;"
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    dialect = bind.dialect.name
+    if dialect != "sqlite":
+        # FTS5 is SQLite-only; Postgres uses pg_trgm / tsvector instead.
+        return
+
+    bind.execute(sa.text(_CREATE_FTS))
+    bind.execute(sa.text(_POPULATE_FTS))
+    bind.execute(sa.text(_TRIGGER_INSERT))
+    bind.execute(sa.text(_TRIGGER_DELETE))
+    bind.execute(sa.text(_TRIGGER_UPDATE))
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    if bind.dialect.name != "sqlite":
+        return
+
+    bind.execute(sa.text(_DROP_TRIGGERS))
+    bind.execute(sa.text(_DROP_FTS))

alembic/versions/0020_memory_embedding.py

@@ -0,0 +1,116 @@
+"""memory_items embedding column + sqlite-vec virtual table
+
+Revision ID: 0020_memory_embedding
+Revises: 0019_memory_fts5
+Create Date: 2026-03-03
+
+Phase B of issue #161:
+- Adds `embedding BLOB` column to memory_items for storing float32 vectors.
+- Creates `vec_items` sqlite-vec virtual table (SQLite + sqlite-vec only).
+- Creates sync triggers to keep vec_items in step with memory_items.embedding.
+
+Graceful degradation: if sqlite-vec is not installed the column migration still
+runs; only the virtual table and triggers are skipped.
+"""
+from __future__ import annotations
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0020_memory_embedding"
+down_revision = "0019_memory_fts5"
+branch_labels = None
+depends_on = None
+
+_EMBEDDING_DIM = 1536  # text-embedding-3-small
+
+_ADD_COLUMN = "ALTER TABLE memory_items ADD COLUMN embedding BLOB"
+
+_CREATE_VEC = f"""\
+CREATE VIRTUAL TABLE IF NOT EXISTS vec_items
+USING vec0(embedding float[{_EMBEDDING_DIM}])
+"""
+
+_POPULATE_VEC = """\
+INSERT OR IGNORE INTO vec_items(rowid, embedding)
+SELECT id, embedding FROM memory_items
+WHERE embedding IS NOT NULL AND deleted_at IS NULL
+"""
+
+_TRIGGER_VEC_INSERT = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_vec_ai
+AFTER INSERT ON memory_items
+WHEN new.embedding IS NOT NULL
+BEGIN
+    INSERT OR REPLACE INTO vec_items(rowid, embedding) VALUES (new.id, new.embedding);
+END
+"""
+
+_TRIGGER_VEC_UPDATE = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_vec_au
+AFTER UPDATE OF embedding ON memory_items
+BEGIN
+    DELETE FROM vec_items WHERE rowid = old.id;
+    INSERT OR IGNORE INTO vec_items(rowid, embedding)
+    SELECT new.id, new.embedding WHERE new.embedding IS NOT NULL;
+END
+"""
+
+_TRIGGER_VEC_DELETE = """\
+CREATE TRIGGER IF NOT EXISTS memory_items_vec_ad
+AFTER DELETE ON memory_items BEGIN
+    DELETE FROM vec_items WHERE rowid = old.id;
+END
+"""
+
+_DROP_VEC_TRIGGERS = """\
+DROP TRIGGER IF EXISTS memory_items_vec_ai;
+DROP TRIGGER IF EXISTS memory_items_vec_au;
+DROP TRIGGER IF EXISTS memory_items_vec_ad;
+"""
+_DROP_VEC_TABLE = "DROP TABLE IF EXISTS vec_items"
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    dialect = bind.dialect.name
+
+    # 1. Add embedding column (all dialects — column is used for BLOB storage).
+    try:
+        bind.execute(sa.text(_ADD_COLUMN))
+    except Exception:
+        pass  # Column already exists.
+
+    if dialect != "sqlite":
+        return  # sqlite-vec is SQLite-only.
+
+    # 2. Load sqlite-vec extension (best-effort).
+    try:
+        import sqlite_vec  # type: ignore
+        raw_conn = bind.connection.dbapi_connection  # type: ignore
+        raw_conn.enable_load_extension(True)
+        sqlite_vec.load(raw_conn)
+        raw_conn.enable_load_extension(False)
+    except Exception:
+        return  # sqlite-vec not installed — skip virtual table creation.
+
+    # 3. Create vec_items virtual table + triggers.
+    bind.execute(sa.text(_CREATE_VEC))
+    bind.execute(sa.text(_POPULATE_VEC))
+    bind.execute(sa.text(_TRIGGER_VEC_INSERT))
+    bind.execute(sa.text(_TRIGGER_VEC_UPDATE))
+    bind.execute(sa.text(_TRIGGER_VEC_DELETE))
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+
+    # Drop vec infrastructure (SQLite only; ignore errors).
+    if bind.dialect.name == "sqlite":
+        try:
+            bind.execute(sa.text(_DROP_VEC_TRIGGERS))
+            bind.execute(sa.text(_DROP_VEC_TABLE))
+        except Exception:
+            pass
+
+    # NOTE: SQLite doesn't support DROP COLUMN; leave the embedding column in place.