improve citations and add object page

Author: lfnovo

app_home.py

@@ -1,3 +1,35 @@
 import streamlit as st
 
-st.switch_page("pages/2_📒_Notebooks.py")
+from open_notebook.domain.base import ObjectModel
+from open_notebook.exceptions import NotFoundError
+from pages.components import (
+    note_panel,
+    source_embedding_panel,
+    source_insight_panel,
+    source_panel,
+)
+from pages.stream_app.utils import setup_page
+
+setup_page("📒 Open Notebook", sidebar_state="collapsed")
+
+if "object_id" not in st.query_params:
+    st.switch_page("pages/2_📒_Notebooks.py")
+    st.stop()
+
+object_id = st.query_params["object_id"]
+try:
+    obj = ObjectModel.get(object_id)
+except NotFoundError:
+    st.switch_page("pages/2_📒_Notebooks.py")
+    st.stop()
+
+obj_type = object_id.split(":")[0]
+
+if obj_type == "note":
+    note_panel(object_id)
+elif obj_type == "source":
+    source_panel(object_id)
+elif obj_type == "source_insight":
+    source_insight_panel(object_id)
+elif obj_type == "source_embedding":
+    source_embedding_panel(object_id)

open_notebook/domain/base.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Any, ClassVar, Dict, List, Optional, Type, TypeVar
+from typing import Any, ClassVar, Dict, List, Optional, Type, TypeVar, cast
 
 from loguru import logger
 from pydantic import BaseModel, ValidationError, field_validator
@@ -29,15 +29,26 @@ class ObjectModel(BaseModel):
     @classmethod
     def get_all(cls: Type[T], order_by=None) -> List[T]:
         try:
+            # If called from a specific subclass, use its table_name
+            if cls.table_name:
+                target_class = cls
+                table_name = cls.table_name
+            else:
+                # This path is taken if called directly from ObjectModel
+                raise InvalidInputError(
+                    "get_all() must be called from a specific model class"
+                )
+
             if order_by:
                 order = f" ORDER BY {order_by}"
             else:
                 order = ""
-            result = repo_query(f"SELECT * FROM {cls.table_name} {order}")
+
+            result = repo_query(f"SELECT * FROM {table_name} {order}")
             objects = []
             for obj in result:
                 try:
-                    objects.append(cls(**obj))
+                    objects.append(target_class(**obj))
                 except Exception as e:
                     logger.critical(f"Error creating object: {str(e)}")
 
@@ -52,15 +63,44 @@ def get(cls: Type[T], id: str) -> T:
         if not id:
             raise InvalidInputError("ID cannot be empty")
         try:
+            # Get the table name from the ID (everything before the first colon)
+            table_name = id.split(":")[0] if ":" in id else id
+
+            # If we're calling from a specific subclass and IDs match, use that class
+            if cls.table_name and cls.table_name == table_name:
+                target_class: Type[T] = cls
+            else:
+                # Otherwise, find the appropriate subclass based on table_name
+                found_class = cls._get_class_by_table_name(table_name)
+                if not found_class:
+                    raise InvalidInputError(f"No class found for table {table_name}")
+                target_class = cast(Type[T], found_class)
+
             result = repo_query(f"SELECT * FROM {id}")
             if result:
-                return cls(**result[0])
+                return target_class(**result[0])
             else:
-                raise NotFoundError(f"{cls.table_name} with id {id} not found")
+                raise NotFoundError(f"{table_name} with id {id} not found")
         except Exception as e:
-            logger.error(f"Error fetching {cls.table_name} with id {id}: {str(e)}")
+            logger.error(f"Error fetching object with id {id}: {str(e)}")
             logger.exception(e)
-            raise NotFoundError(f"{cls.table_name} with id {id} not found")
+            raise NotFoundError(f"Object with id {id} not found - {str(e)}")
+
+    @classmethod
+    def _get_class_by_table_name(cls, table_name: str) -> Optional[Type["ObjectModel"]]:
+        """Find the appropriate subclass based on table_name."""
+
+        def get_all_subclasses(c: Type["ObjectModel"]) -> List[Type["ObjectModel"]]:
+            all_subclasses: List[Type["ObjectModel"]] = []
+            for subclass in c.__subclasses__():
+                all_subclasses.append(subclass)
+                all_subclasses.extend(get_all_subclasses(subclass))
+            return all_subclasses
+
+        for subclass in get_all_subclasses(ObjectModel):
+            if hasattr(subclass, "table_name") and subclass.table_name == table_name:
+                return subclass
+        return None
 
     def needs_embedding(self) -> bool:
         return False

open_notebook/domain/notebook.py

@@ -93,10 +93,42 @@ class Asset(BaseModel):
     url: Optional[str] = None
 
 
+class SourceEmbedding(ObjectModel):
+    table_name: ClassVar[str] = "source_embedding"
+    content: str
+
+    @property
+    def source(self) -> "Source":
+        try:
+            src = repo_query(f"""
+            select source.* from {self.id}                    fetch source
+
+            """)
+            return Source(**src[0]["source"])
+        except Exception as e:
+            logger.error(f"Error fetching source for embedding {self.id}: {str(e)}")
+            logger.exception(e)
+            raise DatabaseOperationError(e)
+
+
 class SourceInsight(ObjectModel):
+    table_name: ClassVar[str] = "source_insight"
     insight_type: str
     content: str
 
+    @property
+    def source(self) -> "Source":
+        try:
+            src = repo_query(f"""
+            select source.* from {self.id}                    fetch source
+
+            """)
+            return Source(**src[0]["source"])
+        except Exception as e:
+            logger.error(f"Error fetching source for insight {self.id}: {str(e)}")
+            logger.exception(e)
+            raise DatabaseOperationError(e)
+
 
 class Source(ObjectModel):
     table_name: ClassVar[str] = "source"
@@ -112,7 +144,7 @@ def get_context(
             return dict(
                 id=self.id,
                 title=self.title,
-                insights=self.insights,
+                insights=[insight.model_dump() for insight in self.insights],
                 full_text=self.full_text,
             )
         else:

pages/3_🔍_Ask_and_Search.py

@@ -3,7 +3,7 @@
 from open_notebook.domain.models import Model
 from open_notebook.domain.notebook import text_search, vector_search
 from open_notebook.graphs.rag import graph as rag_graph
-from pages.stream_app.utils import setup_page
+from pages.stream_app.utils import convert_source_references, setup_page
 
 setup_page("🔍 Search")
 
@@ -40,7 +40,7 @@ def results_card(item):
                 messages=messages
             ),  # config=dict(configurable=dict(model_id=model.id))
         )
-        st.markdown(rag_results["messages"][-1].content)
+        st.markdown(convert_source_references(rag_results["messages"][-1].content))
         with st.expander("Details (for debugging)"):
             st.json(rag_results)
 

pages/components/__init__.py

@@ -0,0 +1,11 @@
+from pages.components.note_panel import note_panel
+from pages.components.source_embedding_panel import source_embedding_panel
+from pages.components.source_insight import source_insight_panel
+from pages.components.source_panel import source_panel
+
+__all__ = [
+    "note_panel",
+    "source_embedding_panel",
+    "source_insight_panel",
+    "source_panel",
+]

pages/components/note_panel.py

@@ -0,0 +1,30 @@
+import streamlit as st
+from loguru import logger
+from streamlit_monaco import st_monaco  # type: ignore
+
+from open_notebook.domain.notebook import Note
+
+
+def note_panel(note_id, notebook_id=None):
+    note: Note = Note.get(note_id)
+    if not note:
+        raise ValueError(f"Note not fonud {note_id}")
+    t_preview, t_edit = st.tabs(["Preview", "Edit"])
+    with t_preview:
+        st.subheader(note.title)
+        st.markdown(note.content)
+    with t_edit:
+        note.title = st.text_input("Title", value=note.title)
+        note.content = st_monaco(
+            value=note.content, height="600px", language="markdown"
+        )
+        if st.button("Save", key=f"pn_edit_note_{note.id or 'new'}"):
+            logger.debug("Editing note")
+            note.save()
+            if not note.id and notebook_id:
+                note.add_to_notebook(notebook_id)
+            st.rerun()
+    if st.button("Delete", type="primary", key=f"delete_note_{note.id or 'new'}"):
+        logger.debug("Deleting note")
+        note.delete()
+        st.rerun()

pages/components/source_embedding_panel.py

@@ -0,0 +1,17 @@
+import streamlit as st
+
+from open_notebook.domain.notebook import SourceEmbedding
+
+
+def source_embedding_panel(source_embedding_id):
+    si: SourceEmbedding = SourceEmbedding.get(source_embedding_id)
+    if not si:
+        raise ValueError(f"Embedding not found {source_embedding_id}")
+    with st.container(border=True):
+        url = f"Navigator?object_id={si.source.id}"
+        st.markdown("**Original Source**")
+        st.markdown(f"{si.source.title} [link](%s)" % url)
+    st.markdown(si.content)
+    if st.button("Delete", type="primary", key=f"delete_embedding_{si.id or 'new'}"):
+        si.delete()
+        st.rerun()

pages/components/source_insight.py

@@ -0,0 +1,18 @@
+import streamlit as st
+
+from open_notebook.domain.notebook import SourceInsight
+
+
+def source_insight_panel(source, notebook_id=None):
+    si: SourceInsight = SourceInsight.get(source)
+    if not si:
+        raise ValueError(f"insight not found {source}")
+    st.subheader(si.insight_type)
+    with st.container(border=True):
+        url = f"Navigator?object_id={si.source.id}"
+        st.markdown("**Original Source**")
+        st.markdown(f"{si.source.title} [link](%s)" % url)
+    st.markdown(si.content)
+    if st.button("Delete", type="primary", key=f"delete_insight_{si.id or 'new'}"):
+        si.delete()
+        st.rerun()

pages/components/source_panel.py

@@ -0,0 +1,84 @@
+import streamlit as st
+import streamlit_scrollable_textbox as stx  # type: ignore
+import yaml
+from humanize import naturaltime
+
+from open_notebook.domain.notebook import Source
+from open_notebook.utils import surreal_clean
+from pages.stream_app.utils import run_patterns
+
+
+def source_panel(source_id: str, modal=False):
+    source: Source = Source.get(source_id)
+    if not source:
+        raise ValueError(f"Source not found: {source_id}")
+
+    current_title = source.title if source.title else "No Title"
+    source.title = st.text_input("Title", value=current_title)
+    if source.title != current_title:
+        st.toast("Saved new Title")
+        source.save()
+
+    process_tab, source_tab = st.tabs(["Process", "Source"])
+    with process_tab:
+        c1, c2 = st.columns([3, 1])
+        with c1:
+            title = st.empty()
+            if source.title:
+                title.subheader(source.title)
+            if source.asset and source.asset.url:
+                from_src = f"from URL: {source.asset.url}"
+            elif source.asset and source.asset.file_path:
+                from_src = f"from file: {source.asset.file_path}"
+            else:
+                from_src = "from text"
+            st.caption(f"Created {naturaltime(source.created)}, {from_src}")
+            for insight in source.insights:
+                with st.expander(f"**{insight.insight_type}**"):
+                    st.markdown(insight.content)
+                    if st.button(
+                        "Delete", type="primary", key=f"delete_insight_{insight.id}"
+                    ):
+                        insight.delete()
+                        st.rerun(scope="fragment" if modal else "app")
+
+        with c2:
+            with open("transformations.yaml", "r") as file:
+                transformations = yaml.safe_load(file)
+                for transformation in transformations["source_insights"]:
+                    if st.button(
+                        transformation["name"], help=transformation["description"]
+                    ):
+                        result = run_patterns(
+                            source.full_text, transformation["patterns"]
+                        )
+                        source.add_insight(
+                            transformation["insight_type"], surreal_clean(result)
+                        )
+                        st.rerun(scope="fragment" if modal else "app")
+
+            if st.button(
+                "Embed vectors",
+                icon="🦾",
+                disabled=source.embedded_chunks > 0,
+                help="This will generate your embedding vectors on the database for powerful search capabilities",
+            ):
+                source.vectorize()
+                st.success("Embedding complete")
+
+            chk_delete = st.checkbox(
+                "🗑️ Delete source", key=f"delete_source_{source.id}", value=False
+            )
+            if chk_delete:
+                st.warning(
+                    "Source will be deleted with all its insights and embeddings"
+                )
+                if st.button(
+                    "Delete", type="primary", key=f"bt_delete_source_{source.id}"
+                ):
+                    source.delete()
+                    st.rerun()
+
+    with source_tab:
+        st.subheader("Content")
+        stx.scrollableTextbox(source.full_text, height=300)