From e1e2bef87d6b0d79a2fd850a60333609ac949a6b Mon Sep 17 00:00:00 2001 From: swati354 Date: Wed, 12 Feb 2025 15:42:17 +0530 Subject: [PATCH 01/12] fix citations add comments --- backend/danswer/chat/chat_utils.py | 53 +++++++++++++++---------- backend/danswer/llm/answering/answer.py | 44 +++++++++++++++++++- 2 files changed, 76 insertions(+), 21 deletions(-) diff --git a/backend/danswer/chat/chat_utils.py b/backend/danswer/chat/chat_utils.py index f4b0b2e02c5..c02e49efd44 100644 --- a/backend/danswer/chat/chat_utils.py +++ b/backend/danswer/chat/chat_utils.py @@ -114,27 +114,29 @@ def combine_message_chain( return "\n\n".join(message_strs) -def reorganize_citations( - answer: str, citations: list[CitationInfo] -) -> tuple[str, list[CitationInfo]]: - """For a complete, citation-aware response, we want to reorganize the citations so that +def reorganize_citations(answer: str, citations: list) -> tuple[str, list]: + """ + For a complete, citation-aware response, we want to reorganize the citations so that they are in the order of the documents that were used in the response. This just looks nicer / avoids - confusion ("Why is there [7] when only 2 documents are cited?").""" + confusion ("Why is there [7] when only 2 documents are cited?"). - # Regular expression to find all instances of [[x]](LINK) - pattern = r"\[\[(.*?)\]\]\((.*?)\)" + Now also handles citations in the format [number] in addition to [[number]](LINK). + """ + + pattern = r"\[\[(\d+)\]\]\((.*?)\)|\[(\d+)\]" all_citation_matches = re.findall(pattern, answer) new_citation_info: dict[int, CitationInfo] = {} for citation_match in all_citation_matches: try: - citation_num = int(citation_match[0]) + citation_str = citation_match[0] if citation_match[0] else citation_match[2] + citation_num = int(citation_str) if citation_num in new_citation_info: continue matching_citation = next( - iter([c for c in citations if c.citation_num == int(citation_num)]), + (c for c in citations if c.citation_num == citation_num), None, ) if matching_citation is None: @@ -146,19 +148,30 @@ def reorganize_citations( ) except Exception: pass - # Function to replace citations with their new number def slack_link_format(match: re.Match) -> str: - link_text = match.group(1) - try: - citation_num = int(link_text) - if citation_num in new_citation_info: - link_text = new_citation_info[citation_num].citation_num - except Exception: - pass - - link_url = match.group(2) - return f"[[{link_text}]]({link_url})" + # Case 1: Linked citation ([[number]](LINK)) + if match.group(1): + link_text = match.group(1) + try: + citation_num = int(link_text) + if citation_num in new_citation_info: + link_text = new_citation_info[citation_num].citation_num + except Exception: + pass + link_url = match.group(2) + return f"[[{link_text}]]({link_url})" + # Case 2: Non-linked citation ([number]) + elif match.group(3): + try: + citation_num = int(match.group(3)) + if citation_num in new_citation_info: + citation_num = new_citation_info[citation_num].citation_num + except Exception: + pass + return f"[{citation_num}]" + else: + return match.group(0) # Substitute all matches in the input text new_answer = re.sub(pattern, slack_link_format, answer) diff --git a/backend/danswer/llm/answering/answer.py b/backend/danswer/llm/answering/answer.py index 6a250d02d2a..694764a6c0a 100644 --- a/backend/danswer/llm/answering/answer.py +++ b/backend/danswer/llm/answering/answer.py @@ -1,4 +1,5 @@ from collections.abc import Iterator +import re from typing import cast from uuid import uuid4 @@ -382,6 +383,47 @@ def _raw_output_for_non_explicit_tool_calling_llms( prompt = prompt_builder.build() yield from message_generator_to_string_generator(self.llm.stream(prompt=prompt)) + def _fix_document_references(self, answer: str) -> str: + """ + Searches the input string for DOCUMENT references in any of these forms: + - DOCUMENT (link) + - [DOCUMENT ] (link) + - DOCUMENT + - [DOCUMENT ] + + and converts them to the proper citation format: + - If a link is provided, returns a linked citation: [[number]](link) + - Otherwise, returns a non-linked citation: [number] + + However, if an adjacent citation (linked or non-linked) for the same number already follows + immediately (ignoring whitespace), the DOCUMENT reference is not converted (i.e. it is removed) + to avoid duplicate citations. + """ + pattern = r"\[?DOCUMENT\s+(\d+)\]?(?:\s*\((.*?)\))?" + + def replacer(match: re.Match) -> str: + try: + num = int(match.group(1)) + except Exception: + return match.group(0) + + if match.group(2) and match.group(2).strip(): + citation = f"[[{num}]]({match.group(2).strip()})" + else: + citation = f"[{num}]" + + post_text = answer[match.end():] + adj_pattern = ( + r"^\s*(\[\[\s*" + re.escape(str(num)) + r"\s*\]\]\([^)]+\)|\[\s*" + re.escape(str(num)) + r"\s*\])" + ) + if re.match(adj_pattern, post_text): + # If an adjacent citation for the same number exists, return an empty string (skip replacement). + return "" + else: + return citation + + return re.sub(pattern, replacer, answer) + @property def processed_streamed_output(self) -> AnswerStream: if self._processed_stream is not None: @@ -465,7 +507,7 @@ def llm_answer(self) -> str: if isinstance(packet, DanswerAnswerPiece) and packet.answer_piece: answer += packet.answer_piece - return answer + return self._fix_document_references(answer) @property def citations(self) -> list[CitationInfo]: From 0ec9627ae16318be0b402558812efc7d377f5cd8 Mon Sep 17 00:00:00 2001 From: swati354 Date: Fri, 24 Jan 2025 14:04:08 +0530 Subject: [PATCH 02/12] Add darwin assistant Add changes to reply only to darwinAssistant Add changes for custom_llm.py --- ...af3dc44_create_table_user_slack_persona.py | 33 +++++++ .../slack/handlers/handle_buttons.py | 68 +++++++++++++++ .../slack/handlers/handle_message.py | 85 ++++++++++++++++++- backend/danswer/danswerbot/slack/listener.py | 7 ++ backend/danswer/danswerbot/slack/models.py | 1 + backend/danswer/db/models.py | 10 +++ backend/danswer/db/persona.py | 12 +++ backend/danswer/db/users.py | 30 +++++++ backend/danswer/llm/custom_llm.py | 2 +- 9 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 backend/alembic/versions/792d1af3dc44_create_table_user_slack_persona.py diff --git a/backend/alembic/versions/792d1af3dc44_create_table_user_slack_persona.py b/backend/alembic/versions/792d1af3dc44_create_table_user_slack_persona.py new file mode 100644 index 00000000000..458cca9d0df --- /dev/null +++ b/backend/alembic/versions/792d1af3dc44_create_table_user_slack_persona.py @@ -0,0 +1,33 @@ +"""Create table user_slack_persona + +Revision ID: 792d1af3dc44 +Revises: 3a7802814195 +Create Date: 2025-01-24 04:26:02.844951 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "792d1af3dc44" +down_revision = "3a7802814195" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "user_slack_persona", + sa.Column("sender_id", sa.String(), nullable=False), + sa.Column("persona_id", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["persona_id"], + ["persona.id"], + ), + sa.PrimaryKeyConstraint("sender_id"), + ) + + +def downgrade() -> None: + op.drop_table("user_slack_persona") \ No newline at end of file diff --git a/backend/danswer/danswerbot/slack/handlers/handle_buttons.py b/backend/danswer/danswerbot/slack/handlers/handle_buttons.py index 3a0209b076f..41f015c4d1c 100644 --- a/backend/danswer/danswerbot/slack/handlers/handle_buttons.py +++ b/backend/danswer/danswerbot/slack/handlers/handle_buttons.py @@ -32,10 +32,16 @@ from danswer.db.engine import get_sqlalchemy_engine from danswer.db.feedback import create_chat_message_feedback from danswer.db.feedback import create_doc_retrieval_feedback +from danswer.db.persona import fetch_persona_by_id +from danswer.db.users import fetch_user_slack_persona +from danswer.db.users import add_user_slack_persona +from danswer.db.users import add_slack_persona_for_user from danswer.document_index.document_index_utils import get_both_index_names from danswer.document_index.factory import get_default_document_index from danswer.utils.logger import setup_logger +from sqlalchemy.orm.exc import NoResultFound + logger_base = setup_logger() @@ -293,3 +299,65 @@ def handle_followup_resolved_button( thread_ts=thread_ts, unfurl=False, ) + + +def handle_persona_selection(req: SocketModeRequest, client: SocketModeClient) -> None: + action = cast(dict[str, Any], req.payload.get("actions", [])[0]) + user_id = req.payload["user"]["id"] + channel_id = req.payload["container"]["channel_id"] + persona_id = action.get( + "value" + ) + message_ts_to_respond_to = req.payload.get("container", {}).get("thread_ts") + + with Session(get_sqlalchemy_engine()) as db_session: + try: + persona = fetch_persona_by_id(db_session=db_session, persona_id=persona_id) + + if persona is None: + respond_in_thread( + client=client.web_client, + channel=channel_id, + text=f"Persona not found.", + thread_ts=message_ts_to_respond_to, + ) + return + + user_slack_persona = fetch_user_slack_persona( + db_session=db_session, sender_id=user_id + ) + if user_slack_persona: + add_slack_persona_for_user( + db_session=db_session, + persona=persona, + user_slack_persona=user_slack_persona, + ) + response_text = f"Persona '{persona.name}' has been set!\n" + respond_in_thread( + client=client.web_client, + channel=channel_id, + text=response_text, + thread_ts=message_ts_to_respond_to, + ) + return + + else: + add_user_slack_persona( + db_session=db_session, sender_id=user_id, persona=persona + ) + respond_in_thread( + client=client.web_client, + channel=channel_id, + text=f"'{persona.name}' has been successfully set as the current persona.", + thread_ts=message_ts_to_respond_to, + ) + return + + except NoResultFound: + respond_in_thread( + client=client.web_client, + channel=channel_id, + text="Error in fetching persona", + thread_ts=message_ts_to_respond_to, + ) + return diff --git a/backend/danswer/danswerbot/slack/handlers/handle_message.py b/backend/danswer/danswerbot/slack/handlers/handle_message.py index 85ddfe70a6c..9457af3248d 100644 --- a/backend/danswer/danswerbot/slack/handlers/handle_message.py +++ b/backend/danswer/danswerbot/slack/handlers/handle_message.py @@ -47,6 +47,9 @@ from danswer.db.models import SlackBotConfig from danswer.db.models import SlackBotResponseType from danswer.db.persona import fetch_persona_by_id +from danswer.db.persona import get_persona_with_docset_and_prompts +from danswer.db.persona import get_personas +from danswer.db.users import fetch_user_slack_persona from danswer.llm.answering.prompts.citations_prompt import ( compute_max_document_tokens_for_persona, ) @@ -177,6 +180,7 @@ def handle_message( channel_config: SlackBotConfig | None, client: WebClient, feedback_reminder_id: str | None, + channel_name: str | None, num_retries: int = DANSWER_BOT_NUM_RETRIES, answer_generation_timeout: int = DANSWER_BOT_ANSWER_GENERATION_TIMEOUT, should_respond_with_error_msgs: bool = DANSWER_BOT_DISPLAY_ERROR_MSGS, @@ -206,9 +210,88 @@ def handle_message( bypass_filters = message_info.bypass_filters is_bot_msg = message_info.is_bot_msg is_bot_dm = message_info.is_bot_dm + persona_name = None + + if channel_name is None: + with Session(get_sqlalchemy_engine()) as db_session: + user_slack_persona = fetch_user_slack_persona( + db_session=db_session, sender_id=sender_id + ) + if user_slack_persona: + slack_persona_id = user_slack_persona.persona_id or None + persona = get_persona_with_docset_and_prompts( + persona_id=slack_persona_id, db_session=db_session + ) + persona_name = persona.name + else: + persona = None + else: + persona = channel_config.persona if channel_config else None + + if is_bot_msg: + command = message_info.command + if command == "/personas": + with Session(get_sqlalchemy_engine()) as db_session: + personas = get_personas( + user_id=None, db_session=db_session, include_default=False + ) + + if not personas: + respond_in_thread( + client=client, + channel=channel, + text="No personas are available.", + thread_ts=message_ts_to_respond_to, + ) + return + + buttons = [ + { + "type": "button", + "text": {"type": "plain_text", "text": persona.name}, + "value": str(persona.id), + "action_id": f"set_persona_{persona.id}", + } + for persona in personas + ] + + blocks = [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Here are the available personas. Click on one to set it:", + }, + }, + {"type": "actions", "elements": buttons}, + ] + + respond_in_thread( + client=client, + channel=channel, + blocks=blocks, + thread_ts=message_ts_to_respond_to, + ) + return + elif command == "/current_persona": + if persona_name is None: + respond_in_thread( + client=client, + channel=channel, + text="No persona is set. Please use the /personas command to set up a persona.", + thread_ts=message_ts_to_respond_to, + ) + return + else: + respond_in_thread( + client=client, + channel=channel, + text=f"Current persona : {persona_name}", + thread_ts=message_ts_to_respond_to, + ) + return document_set_names: list[str] | None = None - persona = channel_config.persona if channel_config else None prompt = None if persona: document_set_names = [ diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py index f8dfb600211..5a74669fbbf 100644 --- a/backend/danswer/danswerbot/slack/listener.py +++ b/backend/danswer/danswerbot/slack/listener.py @@ -28,6 +28,7 @@ handle_followup_resolved_button, ) from danswer.danswerbot.slack.handlers.handle_buttons import handle_slack_feedback +from danswer.danswerbot.slack.handlers.handle_buttons import handle_persona_selection from danswer.danswerbot.slack.handlers.handle_message import handle_message from danswer.danswerbot.slack.handlers.handle_message import ( remove_scheduled_feedback_reminder, @@ -277,6 +278,7 @@ def build_request_details( channel = req.payload["channel_id"] msg = req.payload["text"] sender = req.payload["user_id"] + command = req.payload["command"] single_msg = ThreadMessage(message=msg, sender=None, role=MessageType.USER) @@ -288,6 +290,7 @@ def build_request_details( bypass_filters=True, is_bot_msg=True, is_bot_dm=False, + command=command ) raise RuntimeError("Programming fault, this should never happen.") @@ -356,6 +359,7 @@ def process_message( channel_config=slack_bot_config, client=client.web_client, feedback_reminder_id=feedback_reminder_id, + channel_name = channel_name ) if failed: @@ -391,6 +395,9 @@ def action_routing(req: SocketModeRequest, client: SocketModeClient) -> None: return handle_followup_resolved_button(req, client, immediate=True) elif action["action_id"] == FOLLOWUP_BUTTON_RESOLVED_ACTION_ID: return handle_followup_resolved_button(req, client, immediate=False) + elif action["action_id"].startswith("set_persona_"): + # Persona selection + return handle_persona_selection(req, client) def view_routing(req: SocketModeRequest, client: SocketModeClient) -> None: diff --git a/backend/danswer/danswerbot/slack/models.py b/backend/danswer/danswerbot/slack/models.py index 57a92a29753..375b92d364c 100644 --- a/backend/danswer/danswerbot/slack/models.py +++ b/backend/danswer/danswerbot/slack/models.py @@ -11,3 +11,4 @@ class SlackMessageInfo(BaseModel): bypass_filters: bool # User has tagged @DanswerBot is_bot_msg: bool # User is using /DanswerBot is_bot_dm: bool # User is direct messaging to DanswerBot + command: str | None # Slash command used by user diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py index 909236a978b..cf1fdaa7ed9 100644 --- a/backend/danswer/db/models.py +++ b/backend/danswer/db/models.py @@ -1108,6 +1108,16 @@ class SlackBotConfig(Base): persona: Mapped[Persona | None] = relationship("Persona") +class UserSlackPersona(Base): + __tablename__ = "user_slack_persona" + + sender_id: Mapped[str] = mapped_column(primary_key=True) + persona_id: Mapped[int | None] = mapped_column( + ForeignKey("persona.id"), nullable=True + ) + persona: Mapped[Persona | None] = relationship("Persona", foreign_keys=[persona_id]) + + class TaskQueueState(Base): # Currently refers to Celery Tasks __tablename__ = "task_queue_jobs" diff --git a/backend/danswer/db/persona.py b/backend/danswer/db/persona.py index 4726cf42637..08662dc5873 100644 --- a/backend/danswer/db/persona.py +++ b/backend/danswer/db/persona.py @@ -10,6 +10,7 @@ from sqlalchemy import select from sqlalchemy import update from sqlalchemy.orm import Session +from sqlalchemy.orm import joinedload from danswer.auth.schemas import UserRole from danswer.db.constants import SLACK_BOT_PERSONA_PREFIX @@ -621,3 +622,14 @@ def delete_persona_by_name( db_session.execute(stmt) db_session.commit() + + +def get_persona_with_docset_and_prompts( + persona_id: int, db_session: Session +) -> Persona | None: + persona = db_session.scalar( + select(Persona) + .options(joinedload(Persona.document_sets), joinedload(Persona.prompts)) + .filter_by(id=persona_id, deleted=False) + ) + return persona diff --git a/backend/danswer/db/users.py b/backend/danswer/db/users.py index f8a3938027f..34e3ada7d4a 100644 --- a/backend/danswer/db/users.py +++ b/backend/danswer/db/users.py @@ -1,9 +1,12 @@ from collections.abc import Sequence +from sqlalchemy import select from sqlalchemy.orm import Session from sqlalchemy.schema import Column +from danswer.db.models import Persona from danswer.db.models import User +from danswer.db.models import UserSlackPersona def list_users(db_session: Session, q: str = "") -> Sequence[User]: @@ -19,3 +22,30 @@ def get_user_by_email(email: str, db_session: Session) -> User | None: user = db_session.query(User).filter(User.email == email).first() # type: ignore return user + + +def fetch_user_slack_persona( + db_session: Session, sender_id: str +) -> UserSlackPersona | None: + return db_session.scalar( + select(UserSlackPersona).where(UserSlackPersona.sender_id == sender_id) + ) + + +def add_user_slack_persona( + db_session: Session, sender_id: str, persona: Persona +) -> None: + user_persona = UserSlackPersona( + sender_id=sender_id, persona_id=persona.id, persona=persona + ) + db_session.add(user_persona) + db_session.commit() + + +def add_slack_persona_for_user( + db_session: Session, persona: Persona, user_slack_persona: UserSlackPersona +) -> None: + user_slack_persona.persona_id = persona.id + user_slack_persona.persona = persona + + db_session.commit() diff --git a/backend/danswer/llm/custom_llm.py b/backend/danswer/llm/custom_llm.py index 0b9924b2bed..daeeb71d534 100644 --- a/backend/danswer/llm/custom_llm.py +++ b/backend/danswer/llm/custom_llm.py @@ -76,7 +76,7 @@ def __init__( identity_url: str | None = GEN_AI_IDENTITY_ENDPOINT, client_id: str | None = GEN_AI_CLIENT_ID, client_secret: str | None = GEN_AI_CLIENT_SECRET, - max_output_tokens: int = GEN_AI_MAX_OUTPUT_TOKENS, + max_output_tokens: int = int(GEN_AI_MAX_OUTPUT_TOKENS), api_version: str | None = GEN_AI_API_VERSION, ): From df76982d47b0bac9cc246af85fe117db3e212d38 Mon Sep 17 00:00:00 2001 From: swati354 Date: Thu, 6 Feb 2025 12:10:10 +0530 Subject: [PATCH 03/12] Changing openai model from gpt-35-turbo to gpt-4o-mini --- backend/danswer/llm/custom_llm.py | 2 +- darwin-kubernetes/env-configmap.yaml | 2 +- deployment/docker_compose/docker-compose.local.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/danswer/llm/custom_llm.py b/backend/danswer/llm/custom_llm.py index daeeb71d534..c692173dc43 100644 --- a/backend/danswer/llm/custom_llm.py +++ b/backend/danswer/llm/custom_llm.py @@ -72,7 +72,7 @@ def __init__( # Not used here but you probably want a model server that isn't completely open api_key: str | None, timeout: int, - endpoint: str | None = 'https://alpha.uipath.com/llmgateway_/openai/deployments/gpt-35-turbo/chat/completions?api-version=2023-03-15-preview', + endpoint: str | None = 'https://alpha.uipath.com/llmgateway_/openai/deployments/gpt-4o-mini-2024-07-18/chat/completions?api-version=2024-06-01', identity_url: str | None = GEN_AI_IDENTITY_ENDPOINT, client_id: str | None = GEN_AI_CLIENT_ID, client_secret: str | None = GEN_AI_CLIENT_SECRET, diff --git a/darwin-kubernetes/env-configmap.yaml b/darwin-kubernetes/env-configmap.yaml index e796c046e94..d5dd301ffed 100644 --- a/darwin-kubernetes/env-configmap.yaml +++ b/darwin-kubernetes/env-configmap.yaml @@ -15,7 +15,7 @@ data: EMAIL_FROM: "" # 'your-email@company.com' SMTP_USER missing used instead # Gen AI Settings GEN_AI_MODEL_PROVIDER: "custom" - GEN_AI_API_ENDPOINT: "https://alpha.uipath.com/llmgateway_/openai/deployments/gpt-35-turbo/chat/completions?api-version=2023-03-15-preview" + GEN_AI_API_ENDPOINT: "https://alpha.uipath.com/llmgateway_/openai/deployments/gpt-4o-mini-2024-07-18/chat/completions?api-version=2024-06-01" GEN_AI_IDENTITY_ENDPOINT: "https://alpha.uipath.com/identity_/connect/token" GEN_AI_CLIENT_ID: "XXX" GEN_AI_CLIENT_SECRET: "XXX" diff --git a/deployment/docker_compose/docker-compose.local.yml b/deployment/docker_compose/docker-compose.local.yml index 7583b09b836..99179c0747f 100644 --- a/deployment/docker_compose/docker-compose.local.yml +++ b/deployment/docker_compose/docker-compose.local.yml @@ -39,7 +39,7 @@ services: - GEN_AI_MODEL_VERSION=${GEN_AI_MODEL_VERSION:-} - FAST_GEN_AI_MODEL_VERSION=${FAST_GEN_AI_MODEL_VERSION:-} - GEN_AI_API_KEY=${GEN_AI_API_KEY:-} - - GEN_AI_API_ENDPOINT=https://alpha.uipath.com/llmgateway_/openai/deployments/gpt-35-turbo/chat/completions?api-version=2023-03-15-preview + - GEN_AI_API_ENDPOINT=https://alpha.uipath.com/llmgateway_/openai/deployments/gpt-4o-mini-2024-07-18/chat/completions?api-version=2024-06-01 - GEN_AI_IDENTITY_ENDPOINT=https://alpha.uipath.com/identity_/connect/token - GEN_AI_CLIENT_ID=${GEN_AI_CLIENT_ID:-} - GEN_AI_CLIENT_SECRET=${GEN_AI_CLIENT_SECRET:-} @@ -128,7 +128,7 @@ services: - GEN_AI_MODEL_VERSION=${GEN_AI_MODEL_VERSION:-} - FAST_GEN_AI_MODEL_VERSION=${FAST_GEN_AI_MODEL_VERSION:-custom} - GEN_AI_API_KEY=${GEN_AI_API_KEY:-} - - GEN_AI_API_ENDPOINT=https://alpha.uipath.com/llmgateway_/openai/deployments/gpt-35-turbo/chat/completions?api-version=2023-03-15-preview + - GEN_AI_API_ENDPOINT=https://alpha.uipath.com/llmgateway_/openai/deployments/gpt-4o-mini-2024-07-18/chat/completions?api-version=2024-06-01 - GEN_AI_IDENTITY_ENDPOINT=https://alpha.uipath.com/identity_/connect/token - GEN_AI_CLIENT_ID=${GEN_AI_CLIENT_ID:-} - GEN_AI_CLIENT_SECRET=${GEN_AI_CLIENT_SECRET:-} From 7f9e37597e77d857250f9e48c9b508b09f8c9d93 Mon Sep 17 00:00:00 2001 From: swati354 Date: Thu, 6 Feb 2025 17:12:54 +0530 Subject: [PATCH 04/12] handle slack markdown --- backend/danswer/connectors/slack/utils.py | 6 ++++++ backend/danswer/danswerbot/slack/utils.py | 1 + 2 files changed, 7 insertions(+) diff --git a/backend/danswer/connectors/slack/utils.py b/backend/danswer/connectors/slack/utils.py index 21bae6571d8..7a37b65def4 100644 --- a/backend/danswer/connectors/slack/utils.py +++ b/backend/danswer/connectors/slack/utils.py @@ -278,3 +278,9 @@ def replace_special_catchall(message: str) -> str: def add_zero_width_whitespace_after_tag(message: str) -> str: """Add a 0 width whitespace after every @""" return message.replace("@", "@\u200B") + + @staticmethod + def handle_bold_syntax_for_slack(text: str) -> str: + """ Replace instances of '**' with a single '*'""" + corrected_text = text.replace('**', '*') + return corrected_text diff --git a/backend/danswer/danswerbot/slack/utils.py b/backend/danswer/danswerbot/slack/utils.py index 3132aa6f24a..481f49c6b59 100644 --- a/backend/danswer/danswerbot/slack/utils.py +++ b/backend/danswer/danswerbot/slack/utils.py @@ -275,6 +275,7 @@ def remove_slack_text_interactions(slack_str: str) -> str: slack_str = SlackTextCleaner.replace_links(slack_str) slack_str = SlackTextCleaner.replace_special_catchall(slack_str) slack_str = SlackTextCleaner.add_zero_width_whitespace_after_tag(slack_str) + slack_str = SlackTextCleaner.handle_bold_syntax_for_slack(slack_str) return slack_str From 2f415723f9bf9d102c74ad478dec04a5717ca2de Mon Sep 17 00:00:00 2001 From: swati354 Date: Mon, 10 Feb 2025 18:08:00 +0530 Subject: [PATCH 05/12] opt-out of darwin --- backend/danswer/danswerbot/slack/listener.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py index 5a74669fbbf..4738c458866 100644 --- a/backend/danswer/danswerbot/slack/listener.py +++ b/backend/danswer/danswerbot/slack/listener.py @@ -1,3 +1,4 @@ +import re import time from threading import Event from typing import Any @@ -94,6 +95,10 @@ def prefilter_requests(req: SocketModeRequest, client: SocketModeClient) -> bool if not msg: channel_specific_logger.error("Cannot respond to empty message - skipping") return False + + if re.search(r"!darwin", msg, re.IGNORECASE): + channel_specific_logger.info("Ignoring message containing '!darwin'") + return False if ( req.payload.setdefault("event", {}).get("user", "") From e89989bd71a5c41e046ce9aaae2bbf8eed5edf51 Mon Sep 17 00:00:00 2001 From: swati354 Date: Thu, 13 Feb 2025 10:41:22 +0530 Subject: [PATCH 06/12] Do not respond without any citations --- .../danswer/danswerbot/slack/handlers/handle_message.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/danswer/danswerbot/slack/handlers/handle_message.py b/backend/danswer/danswerbot/slack/handlers/handle_message.py index 9457af3248d..69646debde2 100644 --- a/backend/danswer/danswerbot/slack/handlers/handle_message.py +++ b/backend/danswer/danswerbot/slack/handlers/handle_message.py @@ -654,6 +654,15 @@ def _get_answer(new_message_request: DirectQARequest) -> OneShotQAResponse | Non if matching_doc: cited_docs.append((citation.citation_num, matching_doc)) + if not cited_docs: + respond_in_thread( + client=client, + channel=channel, + text="Unable to generate an answer as no relevant documents are available.", + thread_ts=message_ts_to_respond_to, + ) + return False + cited_docs.sort() citations_block = build_sources_blocks(cited_documents=cited_docs) elif priority_ordered_docs: From 9b00663b10d3a38ee9ff73e8304ee3f03a9fdf42 Mon Sep 17 00:00:00 2001 From: swati354 Date: Thu, 13 Feb 2025 11:45:41 +0530 Subject: [PATCH 07/12] Handle question marks and channel/here tags --- .../danswerbot/slack/handlers/handle_message.py | 16 ++++++++++++++-- backend/danswer/danswerbot/slack/listener.py | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/backend/danswer/danswerbot/slack/handlers/handle_message.py b/backend/danswer/danswerbot/slack/handlers/handle_message.py index 69646debde2..7b69e102591 100644 --- a/backend/danswer/danswerbot/slack/handlers/handle_message.py +++ b/backend/danswer/danswerbot/slack/handlers/handle_message.py @@ -2,6 +2,7 @@ import functools import logging from collections.abc import Callable +import re from typing import Any from typing import cast from typing import Optional @@ -175,6 +176,17 @@ def remove_scheduled_feedback_reminder( ) +def contains_questionmark_outside_links(message: str) -> bool: + """ + Checks if the message contains a question mark outside of URLs. + """ + url_pattern = r"]+>|https?://\S+" + + message_without_links = re.sub(url_pattern, "", message) + + return "?" in message_without_links + + def handle_message( message_info: SlackMessageInfo, channel_config: SlackBotConfig | None, @@ -332,7 +344,7 @@ def handle_message( if ( "questionmark_prefilter" in channel_conf["answer_filters"] - and "?" not in messages[-1].message + and not contains_questionmark_outside_links(messages[-1].message) ): logger.info( "Skipping message since it does not contain a question mark" @@ -658,7 +670,7 @@ def _get_answer(new_message_request: DirectQARequest) -> OneShotQAResponse | Non respond_in_thread( client=client, channel=channel, - text="Unable to generate an answer as no relevant documents are available.", + text="Could not generate an answer due to a lack of relevant documents. Please try refining your search query with more context.", thread_ts=message_ts_to_respond_to, ) return False diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py index 4738c458866..35633f17f6d 100644 --- a/backend/danswer/danswerbot/slack/listener.py +++ b/backend/danswer/danswerbot/slack/listener.py @@ -203,6 +203,22 @@ def prefilter_requests(req: SocketModeRequest, client: SocketModeClient) -> bool "Cannot respond to DanswerBot command without sender to respond to." ) return False + + #Do not respond to messages if the channel is tagged + payload = req.payload + event = payload.get("event", {}) + blocks = event.get("blocks", []) + for block in blocks: + if block.get("type") == "rich_text": + for element in block.get("elements", []): + if element.get("type") == "rich_text_section": + for sub_element in element.get("elements", []): + if ( + sub_element.get("type") == "broadcast" + and sub_element.get("range") in {"channel", "here"} + ): + logger.info("Broadcast message detected; skipping reply.") + return False logger.debug(f"Handling Slack request with Payload: '{req.payload}'") return True From 9501828fa245dd193856178cb43efeee1c1436b1 Mon Sep 17 00:00:00 2001 From: swati354 Date: Fri, 14 Feb 2025 12:34:25 +0530 Subject: [PATCH 08/12] Modifying system prompts --- backend/danswer/prompts/chat_prompts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/danswer/prompts/chat_prompts.py b/backend/danswer/prompts/chat_prompts.py index bf2a9d8cdc7..2395674290d 100644 --- a/backend/danswer/prompts/chat_prompts.py +++ b/backend/danswer/prompts/chat_prompts.py @@ -2,7 +2,7 @@ from danswer.prompts.constants import QUESTION_PAT REQUIRE_CITATION_STATEMENT = """ -Cite relevant statements INLINE using the format [1], [2], [3], etc to reference the document number, \ +CRUCIAL: Cite relevant statements INLINE using the format [1], [2], [3], etc to reference the document number, \ DO NOT provide a reference section at the end and DO NOT provide any links following the citations. """.rstrip() @@ -11,7 +11,7 @@ """.rstrip() CITATION_REMINDER = """ -Remember to provide inline citations in the format [1], [2], [3], etc. +MANDATORY: For any information sourced from documents in your response, include inline citations formatted as [1], [2], [3], etc. """ ADDITIONAL_INFO = "\n\nAdditional Information:\n\t- {datetime_info}." From 52f2ed8aa7556eb2bb7c5d969e0cc230388c9799 Mon Sep 17 00:00:00 2001 From: swati354 Date: Fri, 14 Feb 2025 12:36:33 +0530 Subject: [PATCH 09/12] Remove invalid-ans logic and add retry mechanism --- .../slack/handlers/handle_message.py | 31 +++--- .../one_shot_answer/answer_question.py | 103 ++++++++++-------- 2 files changed, 73 insertions(+), 61 deletions(-) diff --git a/backend/danswer/danswerbot/slack/handlers/handle_message.py b/backend/danswer/danswerbot/slack/handlers/handle_message.py index 7b69e102591..b33f63dc699 100644 --- a/backend/danswer/danswerbot/slack/handlers/handle_message.py +++ b/backend/danswer/danswerbot/slack/handlers/handle_message.py @@ -582,21 +582,22 @@ def _get_answer(new_message_request: DirectQARequest) -> OneShotQAResponse | Non except SlackApiError as e: logger.error(f"Failed to remove Reaction due to: {e}") - if answer.answer_valid is False: - logger.info( - "Answer was evaluated to be invalid, throwing it away without responding." - ) - update_emote_react( - emoji=DANSWER_FOLLOWUP_EMOJI, - channel=message_info.channel_to_respond, - message_ts=message_info.msg_to_respond, - remove=False, - client=client, - ) - - if answer.answer: - logger.debug(answer.answer) - return True + #Removing this as we are handling this with citations logic + # if answer.answer_valid is False: + # logger.info( + # "Answer was evaluated to be invalid, throwing it away without responding." + # ) + # update_emote_react( + # emoji=DANSWER_FOLLOWUP_EMOJI, + # channel=message_info.channel_to_respond, + # message_ts=message_info.msg_to_respond, + # remove=False, + # client=client, + # ) + + # if answer.answer: + # logger.debug(answer.answer) + # return True retrieval_info = answer.docs if not retrieval_info: diff --git a/backend/danswer/one_shot_answer/answer_question.py b/backend/danswer/one_shot_answer/answer_question.py index b315c3662c7..62a0439758d 100644 --- a/backend/danswer/one_shot_answer/answer_question.py +++ b/backend/danswer/one_shot_answer/answer_question.py @@ -310,49 +310,66 @@ def get_search_answer( rerank_metrics_callback: Callable[[RerankMetricsContainer], None] | None = None, ) -> OneShotQAResponse: """Collects the streamed one shot answer responses into a single object""" - qa_response = OneShotQAResponse() + max_attempts = 5 + attempt = 0 + qa_response = None - results = stream_answer_objects( - query_req=query_req, - user=user, - max_document_tokens=max_document_tokens, - max_history_tokens=max_history_tokens, - db_session=db_session, - bypass_acl=bypass_acl, - use_citations=use_citations, - danswerbot_flow=danswerbot_flow, - timeout=answer_generation_timeout, - retrieval_metrics_callback=retrieval_metrics_callback, - rerank_metrics_callback=rerank_metrics_callback, - ) + while attempt < max_attempts: + qa_response = OneShotQAResponse() - answer = "" - for packet in results: - if isinstance(packet, QueryRephrase): - qa_response.rephrase = packet.rephrased_query - if isinstance(packet, DanswerAnswerPiece) and packet.answer_piece: - answer += packet.answer_piece - elif isinstance(packet, QADocsResponse): - qa_response.docs = packet - elif isinstance(packet, LLMRelevanceFilterResponse): - qa_response.llm_chunks_indices = packet.relevant_chunk_indices - elif isinstance(packet, DanswerQuotes): - qa_response.quotes = packet - elif isinstance(packet, CitationInfo): - if qa_response.citations: - qa_response.citations.append(packet) - else: - qa_response.citations = [packet] - elif isinstance(packet, DanswerContexts): - qa_response.contexts = packet - elif isinstance(packet, StreamingError): - qa_response.error_msg = packet.error - elif isinstance(packet, ChatMessageDetail): - qa_response.chat_message_id = packet.message_id - - if answer: - qa_response.answer = answer + results = stream_answer_objects( + query_req=query_req, + user=user, + max_document_tokens=max_document_tokens, + max_history_tokens=max_history_tokens, + db_session=db_session, + bypass_acl=bypass_acl, + use_citations=use_citations, + danswerbot_flow=danswerbot_flow, + timeout=answer_generation_timeout, + retrieval_metrics_callback=retrieval_metrics_callback, + rerank_metrics_callback=rerank_metrics_callback, + ) + answer = "" + for packet in results: + if isinstance(packet, QueryRephrase): + qa_response.rephrase = packet.rephrased_query + if isinstance(packet, DanswerAnswerPiece) and packet.answer_piece: + answer += packet.answer_piece + elif isinstance(packet, QADocsResponse): + qa_response.docs = packet + elif isinstance(packet, LLMRelevanceFilterResponse): + qa_response.llm_chunks_indices = packet.relevant_chunk_indices + elif isinstance(packet, DanswerQuotes): + qa_response.quotes = packet + elif isinstance(packet, CitationInfo): + if qa_response.citations: + qa_response.citations.append(packet) + else: + qa_response.citations = [packet] + elif isinstance(packet, DanswerContexts): + qa_response.contexts = packet + elif isinstance(packet, StreamingError): + qa_response.error_msg = packet.error + elif isinstance(packet, ChatMessageDetail): + qa_response.chat_message_id = packet.message_id + + if answer: + qa_response.answer = answer + + if use_citations and qa_response.answer and qa_response.citations: + qa_response.answer, qa_response.citations = reorganize_citations( + qa_response.answer, qa_response.citations + ) + break # Citations found, break out of retry loop. + # If citations are not required, we can exit immediately. + elif not use_citations: + break + + logger.info(f"Citations not found, retrying... (attempt {attempt + 1}/{max_attempts})") + attempt += 1 + if enable_reflexion: # Because follow up messages are explicitly tagged, we don't need to verify the answer if len(query_req.messages) == 1: @@ -361,10 +378,4 @@ def get_search_answer( else: qa_response.answer_valid = True - if use_citations and qa_response.answer and qa_response.citations: - # Reorganize citation nums to be in the same order as the answer - qa_response.answer, qa_response.citations = reorganize_citations( - qa_response.answer, qa_response.citations - ) - return qa_response From fac773968731f37cc8c36b8ab5244c44ed7f42aa Mon Sep 17 00:00:00 2001 From: swati354 Date: Mon, 17 Feb 2025 08:35:49 +0530 Subject: [PATCH 10/12] Add sfkbarticles connector --- backend/danswer/configs/constants.py | 1 + backend/danswer/connectors/factory.py | 2 + .../connectors/sfkbarticles/__init__.py | 0 .../connectors/sfkbarticles/connector.py | 253 ++++++++++++++++ .../danswer/connectors/sfkbarticles/utils.py | 94 ++++++ backend/danswer/document_index/vespa/index.py | 8 +- backend/danswer/prompts/prompt_utils.py | 7 +- backend/danswer/utils/text_processing.py | 11 +- .../admin/connectors/sfkbarticles/page.tsx | 279 ++++++++++++++++++ web/src/lib/sources.ts | 5 + web/src/lib/types.ts | 13 + 11 files changed, 665 insertions(+), 8 deletions(-) create mode 100644 backend/danswer/connectors/sfkbarticles/__init__.py create mode 100644 backend/danswer/connectors/sfkbarticles/connector.py create mode 100644 backend/danswer/connectors/sfkbarticles/utils.py create mode 100644 web/src/app/admin/connectors/sfkbarticles/page.tsx diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py index b29d3558b84..db445958e61 100644 --- a/backend/danswer/configs/constants.py +++ b/backend/danswer/configs/constants.py @@ -95,6 +95,7 @@ class DocumentSource(str, Enum): SHAREPOINT = "sharepoint" TEAMS = "teams" SALESFORCE = "salesforce" + SFKBARTICLES = "sfkbarticles" DISCOURSE = "discourse" AXERO = "axero" CLICKUP = "clickup" diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py index 1a3d605d3a5..2f885bf2b7d 100644 --- a/backend/danswer/connectors/factory.py +++ b/backend/danswer/connectors/factory.py @@ -34,6 +34,7 @@ from danswer.connectors.productboard.connector import ProductboardConnector from danswer.connectors.requesttracker.connector import RequestTrackerConnector from danswer.connectors.salesforce.connector import SalesforceConnector +from danswer.connectors.sfkbarticles.connector import SfKbArticlesConnector from danswer.connectors.sharepoint.connector import SharepointConnector from danswer.connectors.slab.connector import SlabConnector from danswer.connectors.slack.connector import SlackPollConnector @@ -86,6 +87,7 @@ def identify_connector_class( DocumentSource.SHAREPOINT: SharepointConnector, DocumentSource.TEAMS: TeamsConnector, DocumentSource.SALESFORCE: SalesforceConnector, + DocumentSource.SFKBARTICLES: SfKbArticlesConnector, DocumentSource.DISCOURSE: DiscourseConnector, DocumentSource.AXERO: AxeroConnector, DocumentSource.CLICKUP: ClickupConnector, diff --git a/backend/danswer/connectors/sfkbarticles/__init__.py b/backend/danswer/connectors/sfkbarticles/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/backend/danswer/connectors/sfkbarticles/connector.py b/backend/danswer/connectors/sfkbarticles/connector.py new file mode 100644 index 00000000000..7b7be3a2553 --- /dev/null +++ b/backend/danswer/connectors/sfkbarticles/connector.py @@ -0,0 +1,253 @@ +import os +import requests + +from datetime import datetime +from datetime import timezone +from typing import Any, Tuple + +from danswer.configs.app_configs import INDEX_BATCH_SIZE +from danswer.configs.constants import DocumentSource +from danswer.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc +from danswer.connectors.interfaces import GenerateDocumentsOutput +from danswer.connectors.interfaces import LoadConnector +from danswer.connectors.interfaces import PollConnector +from danswer.connectors.interfaces import SecondsSinceUnixEpoch +from danswer.connectors.models import BasicExpertInfo +from danswer.connectors.models import Document +from danswer.connectors.models import Section +from danswer.connectors.salesforce.utils import extract_dict_text +from danswer.utils.logger import setup_logger + +ID_PREFIX = "SALESFORCE_" +AUTH_URL = "https://login.salesforce.com/services/oauth2/token" + +logger = setup_logger() + + +class SfKbArticlesConnector(LoadConnector, PollConnector): + def __init__( + self, + batch_size: int = INDEX_BATCH_SIZE, + requested_objects: list[str] = [], + ) -> None: + self.batch_size = batch_size + self.product_component_list = ( + [obj.strip() for obj in requested_objects[0].split(",")] + if requested_objects + else None + ) + + def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None: + self.client_id = credentials['sf_client_id'] + self.client_secret = credentials['sf_client_secret'] + self.username = credentials['sf_username'] + self.password = credentials['sf_password'] + + self.access_token, self.instance_url = self._get_access_token() + + self.headers = { + "Authorization": f"Bearer {self.access_token}", + "Content-Type": "application/json", + } + + def _get_access_token(self) -> Tuple[str, str]: + """ + Authenticates with Salesforce and retrieves the access token & instance URL. + """ + payload = { + "grant_type": "password", + "client_id": self.client_id, + "client_secret": self.client_secret, + "username": self.username, + "password": f"{self.password}", + } + response = requests.post(AUTH_URL, data=payload) + if response.status_code != 200: + logger.error(f"Authentication failed: {response.text}") + raise Exception("Failed to authenticate with Salesforce.") + + data = response.json() + logger.info("Successfully authenticated with Salesforce.") + return data["access_token"], data["instance_url"] + + def _convert_object_instance_to_document( + self, object_dict: dict[str, Any] + ) -> Document: + + salesforce_id = object_dict["Id"] + danswer_salesforce_id = f"{ID_PREFIX}{salesforce_id}" + extracted_link = f"{self.instance_url}/{salesforce_id}" + extracted_doc_updated_at = time_str_to_utc(object_dict["LastModifiedDate"]) + extracted_object_text = extract_dict_text(object_dict) + extracted_semantic_identifier = object_dict.get("Title", "Unknown Object") + extracted_primary_owners = [ + BasicExpertInfo( + display_name=self._get_name_from_id(object_dict["LastModifiedById"]) + ) + ] + + doc = Document( + id=danswer_salesforce_id, + sections=[Section(link=extracted_link, text=extracted_object_text)], + source=DocumentSource.SFKBARTICLES, + semantic_identifier=extracted_semantic_identifier, + doc_updated_at=extracted_doc_updated_at, + primary_owners=extracted_primary_owners, + metadata={}, + ) + return doc + + def _get_name_from_id(self, id: str) -> str: + """ + Fetches the name of a Salesforce user based on their ID. + """ + query = f"SELECT Name FROM User WHERE Id = '{id}'" + url = f"{self.instance_url}/services/data/v56.0/query" + params = {"q": query} + + response = requests.get(url, headers=self.headers, params=params) + if response.status_code != 200: + logger.error(f"Failed to fetch name for ID {id}: {response.text}") + return "Unknown" + + data = response.json() + records = data.get("records", []) + if not records: + logger.warning(f"No name found for ID {id}") + return "Unknown" + + return records[0].get("Name", "Unknown") + + def build_salesforce_query(self, + parent_objects: list[str], + start: datetime | None = None, + end: datetime | None = None) -> str: + """ + Builds the Salesforce query dynamically. + If product_component_list is empty, it fetches all Product_Component__c values. + Otherwise, it filters using the provided product_component_list. + """ + if not parent_objects: + product_filter = "" # No filter, fetch all + else: + product_components_str = ", ".join( + [ + f"'{component.strip()}'" + for component in parent_objects + if component.strip() + ] + ) + product_filter = f"AND Product_Component__c IN ({product_components_str})" + + query = f""" + SELECT + Id, + Title, + Summary, + Product_Component__c, + Product_Component_Version__c, + Question_Problem__c, + Resolution__c, + Sub_Component__c, + IsVisibleInPkb, + IsVisibleInCsp, + IsVisibleInPrm, + ArticleCreatedDate, + ArticleNumber, + CreatedDate, + ArticleTotalViewCount, + FirstPublishedDate, + IsDeleted, + IsLatestVersion, + Language, + LastModifiedDate, + LastModifiedById, + LastPublishedDate, + Orchestrator_Version__c, + Studio_Version__c + FROM Knowledge__kav + WHERE Language='en_US' + AND PublishStatus = 'Online' + AND IsDeleted = FALSE + {product_filter} + """.strip() + + if start: + start_str = start.strftime("%Y-%m-%dT%H:%M:%S.000Z") + query += f" AND LastModifiedDate >= {start_str}" + if end: + end_str = end.strftime("%Y-%m-%dT%H:%M:%S.000Z") + query += f" AND LastModifiedDate <= {end_str}" + + return query + + def _fetch_from_salesforce( + self, + start: datetime | None = None, + end: datetime | None = None, + ) -> GenerateDocumentsOutput: + + query = self.build_salesforce_query(self.product_component_list, start, end) + doc_batch: list[Document] = [] + query_results: dict = {} + + url = f"{self.instance_url}/services/data/v56.0/query" + params = {"q": query} + query_result = requests.get( + url, headers=self.headers, params=params if "q" in params else None + ) + + while url: + query_result = requests.get( + url, headers=self.headers, params=params if "q" in params else None + ) + data = query_result.json() + + if isinstance(data, list): + error_message = "; ".join(error.get("message", "Unknown error") for error in data) + raise Exception(f"Salesforce API error: {error_message}") + + if "records" in data: + for record_dict in data["records"]: + query_results.setdefault(record_dict["Id"], {}).update(record_dict) + + url = data.get("nextRecordsUrl", None) + if url: + url = f"{self.instance_url}{url}" + + for combined_object_dict in query_results.values(): + doc_batch.append( + self._convert_object_instance_to_document(combined_object_dict) + ) + if len(doc_batch) > self.batch_size: + yield doc_batch + doc_batch = [] + + yield doc_batch + + def load_from_state(self) -> GenerateDocumentsOutput: + return self._fetch_from_salesforce() + + def poll_source( + self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch + ) -> GenerateDocumentsOutput: + start_datetime = datetime.fromtimestamp(start, tz=timezone.utc) + end_datetime = datetime.fromtimestamp(end, tz=timezone.utc) + return self._fetch_from_salesforce(start=start_datetime, end=end_datetime) + + +if __name__ == "__main__": + connector = SfKbArticlesConnector( + requested_objects=os.environ["REQUESTED_OBJECTS"].split(",") + ) + + connector.load_credentials( + { + "sf_client_id": os.environ["SF_CLIENT_ID"], + "sf_client_secret": os.environ["SF_CLIENT_SECRET"], + "sf_username": os.environ["SF_USERNAME"], + "sf_password": os.environ["SF_PASSWORD"], + } + ) + document_batches = connector.load_from_state() + print(next(document_batches)) diff --git a/backend/danswer/connectors/sfkbarticles/utils.py b/backend/danswer/connectors/sfkbarticles/utils.py new file mode 100644 index 00000000000..d89b61039b3 --- /dev/null +++ b/backend/danswer/connectors/sfkbarticles/utils.py @@ -0,0 +1,94 @@ +import re +from bs4 import BeautifulSoup +from typing import Union +from danswer.utils.logger import setup_logger + +SF_JSON_FILTER = r"Id$|Date$|stamp$|url$" + + +def _clean_salesforce_dict(data: Union[dict, list]) -> Union[dict, list]: + if isinstance(data, dict): + if "records" in data.keys(): + data = data["records"] + if isinstance(data, dict): + if "attributes" in data.keys(): + if isinstance(data["attributes"], dict): + data.update(data.pop("attributes")) + + if isinstance(data, dict): + filtered_dict = {} + for key, value in data.items(): + if not re.search(SF_JSON_FILTER, key, re.IGNORECASE): + if "__c" in key: # remove the custom object indicator for display + key = key[:-3] + if isinstance(value, (dict, list)): + filtered_value = _clean_salesforce_dict(value) + if filtered_value: + filtered_dict[key] = filtered_value + elif value is not None: + filtered_dict[key] = value + return filtered_dict + elif isinstance(data, list): + filtered_list = [] + for item in data: + if isinstance(item, (dict, list)): + filtered_item = _clean_salesforce_dict(item) + if filtered_item: + filtered_list.append(filtered_item) + elif item is not None: + filtered_list.append(filtered_item) + return filtered_list + else: + return data + + +def _json_to_natural_language(data: Union[dict, list], indent: int = 0) -> str: + result = [] + indent_str = " " * indent + + if isinstance(data, dict): + for key, value in data.items(): + if isinstance(value, (dict, list)): + result.append(f"{indent_str}{key}:") + result.append(_json_to_natural_language(value, indent + 2)) + else: + result.append(f"{indent_str}{key}: {value}") + elif isinstance(data, list): + for item in data: + result.append(_json_to_natural_language(item, indent)) + else: + result.append(f"{indent_str}{data}") + + return "\n".join(result) + + +def extract_dict_text(raw_dict: dict) -> str: + processed_dict = _clean_salesforce_dict(raw_dict) + + if 'Resolution' in processed_dict: + processed_dict['Resolution'] = clean_html(processed_dict['Resolution']) + + natural_language_dict = _json_to_natural_language(processed_dict) + return natural_language_dict + + +def clean_html(html_content: str) -> str: + """ + Cleans the HTML content by removing tags and returning the plain text, + while preserving the links. + :param html_content: HTML content as string + :return: Cleaned text with preserved links + """ + soup = BeautifulSoup(html_content, "lxml") + + # Replace tags with their text and the href as a link in brackets + for a_tag in soup.find_all('a', href=True): + a_tag.insert_before(f"[{a_tag.get_text()}]({a_tag['href']})") + a_tag.decompose() + + cleaned_text = soup.get_text(separator="\n", strip=True) + + # Replace non-breaking spaces (\xa0) with regular spaces + cleaned_text = cleaned_text.replace('\xa0', ' ') + + return cleaned_text diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py index a7892e40f45..ef93c29d1e6 100644 --- a/backend/danswer/document_index/vespa/index.py +++ b/backend/danswer/document_index/vespa/index.py @@ -704,12 +704,12 @@ def _query_vespa(query_params: Mapping[str, str | int | float]) -> list[Inferenc params["hits"] = 50 filtered_hits_all = query_vespa_helper(params) - #Only Web Records + #Only Web Records and Salesforce KB articles params["hits"] = 10 - params["yql"] = params["yql"] + ' and source_type contains "web"' - filtered_hits_web = query_vespa_helper(params) + params["yql"] = params["yql"] + ' and (source_type contains "web" or source_type contains "sfkabarticles")' + filtered_hits_web_sf = query_vespa_helper(params) - filtered_hits_final = filtered_hits_web + filtered_hits_all + filtered_hits_final = filtered_hits_web_sf + filtered_hits_all inference_chunks = [_vespa_hit_to_inference_chunk(hit) for hit in filtered_hits_final] #inplace sorting based on score diff --git a/backend/danswer/prompts/prompt_utils.py b/backend/danswer/prompts/prompt_utils.py index 6d7bddeec95..117de69cf9a 100644 --- a/backend/danswer/prompts/prompt_utils.py +++ b/backend/danswer/prompts/prompt_utils.py @@ -178,9 +178,10 @@ def drop_messages_history_overflow( final_msgs = [final_msg] # Start dropping from the history if necessary - ind_prev_msg_start = find_last_index( - token_counts, max_prompt_tokens=max_allowed_tokens - ) + # ind_prev_msg_start = find_last_index( + # token_counts, max_prompt_tokens=max_allowed_tokens + # ) + ind_prev_msg_start = 0 if system_msg and ind_prev_msg_start <= len(history_msgs): final_messages.append(system_msg) diff --git a/backend/danswer/utils/text_processing.py b/backend/danswer/utils/text_processing.py index b0fbcdfa1e9..6083ffded15 100644 --- a/backend/danswer/utils/text_processing.py +++ b/backend/danswer/utils/text_processing.py @@ -20,7 +20,16 @@ def decode_escapes(s: str) -> str: def decode_match(match: re.Match) -> str: - return codecs.decode(match.group(0), "unicode-escape") + matched_str = match.group(0) + + # Only double escape non-Unicode sequences + if matched_str.startswith("\\U") and not re.match(r"\\U[0-9a-fA-F]{8}", matched_str): + return matched_str + + try: + return codecs.decode(matched_str, "unicode-escape") + except UnicodeDecodeError: + return matched_str return ESCAPE_SEQUENCE_RE.sub(decode_match, s) diff --git a/web/src/app/admin/connectors/sfkbarticles/page.tsx b/web/src/app/admin/connectors/sfkbarticles/page.tsx new file mode 100644 index 00000000000..bd07892e537 --- /dev/null +++ b/web/src/app/admin/connectors/sfkbarticles/page.tsx @@ -0,0 +1,279 @@ +"use client"; + +import * as Yup from "yup"; +import { TrashIcon, SalesforceIcon } from "@/components/icons/icons"; // Make sure you have a Document360 icon +import { errorHandlingFetcher as fetcher } from "@/lib/fetcher"; +import useSWR, { useSWRConfig } from "swr"; +import { LoadingAnimation } from "@/components/Loading"; +import { HealthCheckBanner } from "@/components/health/healthcheck"; +import { + SfKbArticlesConfig, + SfKbArticlesCredentialJson, + ConnectorIndexingStatus, + Credential, +} from "@/lib/types"; // Modify or create these types as required +import { adminDeleteCredential, linkCredential } from "@/lib/credential"; +import { CredentialForm } from "@/components/admin/connectors/CredentialForm"; +import { + TextFormField, + TextArrayFieldBuilder, +} from "@/components/admin/connectors/Field"; +import { ConnectorsTable } from "@/components/admin/connectors/table/ConnectorsTable"; +import { ConnectorForm } from "@/components/admin/connectors/ConnectorForm"; +import { usePublicCredentials } from "@/lib/hooks"; +import { AdminPageTitle } from "@/components/admin/Title"; +import { Card, Text, Title } from "@tremor/react"; + +const MainSection = () => { + const { mutate } = useSWRConfig(); + const { + data: connectorIndexingStatuses, + isLoading: isConnectorIndexingStatusesLoading, + error: isConnectorIndexingStatusesError, + } = useSWR[]>( + "/api/manage/admin/connector/indexing-status", + fetcher + ); + + const { + data: credentialsData, + isLoading: isCredentialsLoading, + error: isCredentialsError, + refreshCredentials, + } = usePublicCredentials(); + + if ( + (!connectorIndexingStatuses && isConnectorIndexingStatusesLoading) || + (!credentialsData && isCredentialsLoading) + ) { + return ; + } + + if (isConnectorIndexingStatusesError || !connectorIndexingStatuses) { + return
Failed to load connectors
; + } + + if (isCredentialsError || !credentialsData) { + return
Failed to load credentials
; + } + + const SalesforceConnectorIndexingStatuses: ConnectorIndexingStatus< + SfKbArticlesConfig, + SfKbArticlesCredentialJson + >[] = connectorIndexingStatuses.filter( + (connectorIndexingStatus) => + connectorIndexingStatus.connector.source === "salesforce" + ); + + const SfKbArticlesCredential: Credential | undefined = + credentialsData.find( + (credential) => credential.credential_json?.sf_username + ); + + return ( + <> + + The Salesforce Knowledge Base Articles connector allows you to index and search through your + Salesforce Knowledge Base. Once setup, all indicated Salesforce data will + be queryable within Darwin. + + + + Step 1: Provide Salesforce credentials + + {SfKbArticlesCredential ? ( + <> +
+ Existing SalesForce Username: + + {SfKbArticlesCredential.credential_json.sf_username} + + +
+ + ) : ( + <> + + As a first step, please provide the Salesforce account's + client_id, client_secret, username and password. + + + + formBody={ + <> + + + + + + } + validationSchema={Yup.object().shape({ + sf_client_id: Yup.string().required( + "Please enter your Salesforce Client Id" + ), + sf_client_secret: Yup.string().required( + "Please enter your Salesforce Client Secret" + ), + sf_username: Yup.string().required( + "Please enter your Salesforce username" + ), + sf_password: Yup.string().required( + "Please enter your Salesforce password" + ), + })} + initialValues={{ + sf_client_id: "", + sf_client_secret: "", + sf_username: "", + sf_password: "", + }} + onSubmit={(isSuccess) => { + if (isSuccess) { + refreshCredentials(); + } + }} + /> + + + )} + + + Step 2: Manage Salesforce KB Articles Connector + + + {SalesforceConnectorIndexingStatuses.length > 0 && ( + <> + + The latest state of your Salesforce objects are fetched every 10 + minutes. + +
+ + connectorIndexingStatuses={SalesforceConnectorIndexingStatuses} + liveCredential={SfKbArticlesCredential} + getCredential={(credential) => + credential.credential_json.sf_client_secret + } + onUpdate={() => + mutate("/api/manage/admin/connector/indexing-status") + } + onCredentialLink={async (connectorId) => { + if (SfKbArticlesCredential) { + await linkCredential(connectorId, SfKbArticlesCredential.id); + mutate("/api/manage/admin/connector/indexing-status"); + } + }} + specialColumns={[ + { + header: "Connectors", + key: "connectors", + getValue: (ccPairStatus) => { + const connectorConfig = + ccPairStatus.connector.connector_specific_config; + return `${connectorConfig.requested_objects}`; + }, + }, + ]} + includeName + /> +
+ + )} + + {SfKbArticlesCredential ? ( + + + nameBuilder={(values) => + values.requested_objects && values.requested_objects.length > 0 + ? `SfKbArticles-${values.requested_objects.join("-")}` + : "SfKbArticles" + } + ccPairNameBuilder={(values) => + values.requested_objects && values.requested_objects.length > 0 + ? `SfKbArticles-${values.requested_objects.join("-")}` + : "SfKbArticles" + } + source="sfkbarticles" + inputType="poll" + // formBody={<>} + formBodyBuilder={TextArrayFieldBuilder({ + name: "requested_objects", + label: "Specify the Product Components", + subtext: ( + <> +
+ Specify the product components for which you want to fetch the Salesforce Knowledge Base articles. +
+
+ Example: Orchestrator, Activities, Studio, Robot, Automation Hub. +
+
+ By default, it will fetch articles for all the product components. +
+
+ Hint: Use the exact product component name for accurate results. + + ), + })} + validationSchema={Yup.object().shape({ + requested_objects: Yup.array() + .of( + Yup.string().required( + "Salesforce Product Component names must be strings" + ) + ) + .required(), + })} + initialValues={{ + requested_objects: [], + }} + credentialId={SfKbArticlesCredential.id} + refreshFreq={10 * 60} // 10 minutes + /> + + ) : ( + + Please provide all Salesforce info in Step 1 first! Once you're + done with that, you can then specify the product components for which + you want to fetch the Salesforce Knowledge Base articles. + + )} + + ); +}; + +export default function Page() { + return ( +
+
+ +
+ + } title="Salesforce KB Articles" /> + + +
+ ); +} diff --git a/web/src/lib/sources.ts b/web/src/lib/sources.ts index f141e42edb5..6c93d412a69 100644 --- a/web/src/lib/sources.ts +++ b/web/src/lib/sources.ts @@ -172,6 +172,11 @@ const SOURCE_METADATA_MAP: SourceMap = { displayName: "Salesforce", category: SourceCategory.AppConnection, }, + sfkbarticles: { + icon: SalesforceIcon, + displayName: "SfKbArticles", + category: SourceCategory.AppConnection, + }, sharepoint: { icon: SharepointIcon, displayName: "Sharepoint", diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index fe252afbf28..a0a5be116fa 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -144,6 +144,10 @@ export interface SalesforceConfig { requested_objects?: string[]; } +export interface SfKbArticlesConfig { + requested_objects?: string[]; +} + export interface SharepointConfig { sites?: string[]; } @@ -451,12 +455,21 @@ export interface OCICredentialJson { access_key_id: string; secret_access_key: string; } + export interface SalesforceCredentialJson { sf_username: string; sf_password: string; sf_security_token: string; } +export interface SfKbArticlesCredentialJson { + sf_client_id: string; + sf_client_secret: string; + sf_username: string; + sf_password: string; + sf_security_token: string; +} + export interface SharepointCredentialJson { sp_client_id: string; sp_client_secret: string; From 585ba22be456a3d1898d1f1be0912d72d44f39a7 Mon Sep 17 00:00:00 2001 From: swati354 Date: Mon, 17 Feb 2025 12:01:36 +0530 Subject: [PATCH 11/12] add sfkbartices to 'ValidSources' --- web/src/app/admin/connectors/sfkbarticles/page.tsx | 4 ++-- web/src/lib/types.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/src/app/admin/connectors/sfkbarticles/page.tsx b/web/src/app/admin/connectors/sfkbarticles/page.tsx index bd07892e537..9aedc114b6f 100644 --- a/web/src/app/admin/connectors/sfkbarticles/page.tsx +++ b/web/src/app/admin/connectors/sfkbarticles/page.tsx @@ -102,8 +102,8 @@ const MainSection = () => { ) : ( <> - As a first step, please provide the Salesforce account's - client_id, client_secret, username and password. + As a first step, please provide the Salesforce account's + client_id, client_secret, username and password. diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index a0a5be116fa..ea6f3c7defc 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -51,6 +51,7 @@ export type ValidSources = | "loopio" | "dropbox" | "salesforce" + | "sfkbarticles" | "sharepoint" | "teams" | "zendesk" @@ -467,7 +468,6 @@ export interface SfKbArticlesCredentialJson { sf_client_secret: string; sf_username: string; sf_password: string; - sf_security_token: string; } export interface SharepointCredentialJson { From 4d3e234c81aa547e0706d668e8fd8eaff445d5a9 Mon Sep 17 00:00:00 2001 From: swati354 Date: Wed, 19 Feb 2025 13:26:01 +0530 Subject: [PATCH 12/12] Update jira connector --- backend/danswer/configs/app_configs.py | 4 + .../connectors/danswer_jira/connector.py | 295 ++++++++---------- .../danswer/connectors/danswer_jira/utils.py | 71 +++++ web/src/app/admin/connectors/jira/page.tsx | 55 ++-- .../admin/connectors/ConnectorTitle.tsx | 4 +- web/src/lib/types.ts | 3 +- 6 files changed, 227 insertions(+), 205 deletions(-) diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py index f4e8ce14158..e447c189a47 100644 --- a/backend/danswer/configs/app_configs.py +++ b/backend/danswer/configs/app_configs.py @@ -198,6 +198,10 @@ for ignored_tag in os.environ.get("JIRA_CONNECTOR_LABELS_TO_SKIP", "").split(",") if ignored_tag ] +# Maximum size for Jira tickets in bytes (default: 100KB) +JIRA_CONNECTOR_MAX_TICKET_SIZE = int( + os.environ.get("JIRA_CONNECTOR_MAX_TICKET_SIZE", 100 * 1024) +) GONG_CONNECTOR_START_TIME = os.environ.get("GONG_CONNECTOR_START_TIME") diff --git a/backend/danswer/connectors/danswer_jira/connector.py b/backend/danswer/connectors/danswer_jira/connector.py index da525146f9f..2895370bf81 100644 --- a/backend/danswer/connectors/danswer_jira/connector.py +++ b/backend/danswer/connectors/danswer_jira/connector.py @@ -1,4 +1,5 @@ import os +from collections.abc import Iterable from datetime import datetime from datetime import timezone from typing import Any @@ -9,8 +10,13 @@ from danswer.configs.app_configs import INDEX_BATCH_SIZE from danswer.configs.app_configs import JIRA_CONNECTOR_LABELS_TO_SKIP +from danswer.configs.app_configs import JIRA_CONNECTOR_MAX_TICKET_SIZE from danswer.configs.constants import DocumentSource from danswer.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc +from danswer.connectors.danswer_jira.utils import best_effort_basic_expert_info +from danswer.connectors.danswer_jira.utils import best_effort_get_field_from_issue +from danswer.connectors.danswer_jira.utils import extract_text_from_content +from danswer.connectors.danswer_jira.utils import get_comment_strs from danswer.connectors.interfaces import GenerateDocumentsOutput from danswer.connectors.interfaces import LoadConnector from danswer.connectors.interfaces import PollConnector @@ -23,174 +29,130 @@ logger = setup_logger() -PROJECT_URL_PAT = "projects" -JIRA_API_VERSION = os.environ.get("JIRA_API_VERSION") or "2" - - -def extract_jira_project(url: str) -> tuple[str, str]: - parsed_url = urlparse(url) - jira_base = parsed_url.scheme + "://" + parsed_url.netloc - - # Split the path by '/' and find the position of 'projects' to get the project name - split_path = parsed_url.path.split("/") - if PROJECT_URL_PAT in split_path: - project_pos = split_path.index(PROJECT_URL_PAT) - if len(split_path) > project_pos + 1: - jira_project = split_path[project_pos + 1] - else: - raise ValueError("No project name found in the URL") - else: - raise ValueError("'projects' not found in the URL") - - return jira_base, jira_project +JIRA_API_VERSION = os.environ.get("JIRA_API_VERSION") or "2" +_JIRA_FULL_PAGE_SIZE = 50 -def extract_text_from_content(content: dict) -> str: - texts = [] - if "content" in content: - for block in content["content"]: - if "content" in block: - for item in block["content"]: - if item["type"] == "text": - texts.append(item["text"]) - return " ".join(texts) - - -def best_effort_get_field_from_issue(jira_issue: Issue, field: str) -> Any: - if hasattr(jira_issue.fields, field): - return getattr(jira_issue.fields, field) - - try: - return jira_issue.raw["fields"][field] - except Exception: - return None +def _paginate_jql_search( + jira_client: JIRA, + jql: str, + max_results: int, + fields: str | None = None, +) -> Iterable[Issue]: + start = 0 + while True: + logger.debug( + f"Fetching Jira issues with JQL: {jql}, " + f"starting at {start}, max results: {max_results}" + ) + issues = jira_client.search_issues( + jql_str=jql, + startAt=start, + maxResults=max_results, + fields=fields, + ) -def _get_comment_strs( - jira: Issue, comment_email_blacklist: tuple[str, ...] = () -) -> list[str]: - comment_strs = [] - for comment in jira.fields.comment.comments: - try: - if hasattr(comment, "body"): - body_text = extract_text_from_content(comment.raw["body"]) - elif hasattr(comment, "raw"): - body = comment.raw.get("body", "No body content available") - body_text = ( - extract_text_from_content(body) if isinstance(body, dict) else body - ) + for issue in issues: + if isinstance(issue, Issue): + yield issue else: - body_text = "No body attribute found" - - if ( - hasattr(comment, "author") - and comment.author.emailAddress in comment_email_blacklist - ): - continue # Skip adding comment if author's email is in blacklist + raise Exception(f"Found Jira object not of type Issue: {issue}") - comment_strs.append(body_text) - except Exception as e: - logger.error(f"Failed to process comment due to an error: {e}") - continue + if len(issues) < max_results: + break - return comment_strs + start += max_results def fetch_jira_issues_batch( - jql: str, - start_index: int, jira_client: JIRA, - batch_size: int = INDEX_BATCH_SIZE, + jql: str, + batch_size: int, comment_email_blacklist: tuple[str, ...] = (), labels_to_skip: set[str] | None = None, -) -> tuple[list[Document], int]: - doc_batch = [] - - batch = jira_client.search_issues( - jql, - startAt=start_index, - maxResults=batch_size, - ) +) -> Iterable[Document]: + for issue in _paginate_jql_search( + jira_client=jira_client, + jql=jql, + max_results=batch_size, + ): + if labels_to_skip: + if any(label in issue.fields.labels for label in labels_to_skip): + logger.info( + f"Skipping {issue.key} because it has a label to skip. Found " + f"labels: {issue.fields.labels}. Labels to skip: {labels_to_skip}." + ) + continue - for jira in batch: - if type(jira) != Issue: - logger.warning(f"Found Jira object not of type Issue {jira}") - continue + description = ( + issue.fields.description + if JIRA_API_VERSION == "2" + else extract_text_from_content(issue.raw["fields"]["description"]) + ) + comments = get_comment_strs( + issue=issue, + comment_email_blacklist=comment_email_blacklist, + ) + ticket_content = f"{description}\n" + "\n".join( + [f"Comment: {comment}" for comment in comments if comment] + ) - if labels_to_skip and any( - label in jira.fields.labels for label in labels_to_skip - ): + # Check ticket size + if len(ticket_content.encode("utf-8")) > JIRA_CONNECTOR_MAX_TICKET_SIZE: logger.info( - f"Skipping {jira.key} because it has a label to skip. Found " - f"labels: {jira.fields.labels}. Labels to skip: {labels_to_skip}." + f"Skipping {issue.key} because it exceeds the maximum size of " + f"{JIRA_CONNECTOR_MAX_TICKET_SIZE} bytes." ) continue - comments = _get_comment_strs(jira, comment_email_blacklist) - semantic_rep = ( - f"{jira.fields.description}\n" - if jira.fields.description - else "" + "\n".join([f"Comment: {comment}" for comment in comments]) - ) - - page_url = f"{jira_client.client_info()}/browse/{jira.key}" + page_url = f"{jira_client.client_info()}/browse/{issue.key}" people = set() try: - people.add( - BasicExpertInfo( - display_name=jira.fields.creator.displayName, - email=jira.fields.creator.emailAddress, - ) - ) + creator = best_effort_get_field_from_issue(issue, "creator") + if basic_expert_info := best_effort_basic_expert_info(creator): + people.add(basic_expert_info) except Exception: # Author should exist but if not, doesn't matter pass try: - people.add( - BasicExpertInfo( - display_name=jira.fields.assignee.displayName, # type: ignore - email=jira.fields.assignee.emailAddress, # type: ignore - ) - ) + assignee = best_effort_get_field_from_issue(issue, "assignee") + if basic_expert_info := best_effort_basic_expert_info(assignee): + people.add(basic_expert_info) except Exception: # Author should exist but if not, doesn't matter pass metadata_dict = {} - priority = best_effort_get_field_from_issue(jira, "priority") - if priority: + if priority := best_effort_get_field_from_issue(issue, "priority"): metadata_dict["priority"] = priority.name - status = best_effort_get_field_from_issue(jira, "status") - if status: + if status := best_effort_get_field_from_issue(issue, "status"): metadata_dict["status"] = status.name - resolution = best_effort_get_field_from_issue(jira, "resolution") - if resolution: + if resolution := best_effort_get_field_from_issue(issue, "resolution"): metadata_dict["resolution"] = resolution.name - labels = best_effort_get_field_from_issue(jira, "labels") - if labels: + if labels := best_effort_get_field_from_issue(issue, "labels"): metadata_dict["label"] = labels - doc_batch.append( - Document( - id=page_url, - sections=[Section(link=page_url, text=semantic_rep)], - source=DocumentSource.JIRA, - semantic_identifier=jira.fields.summary, - doc_updated_at=time_str_to_utc(jira.fields.updated), - primary_owners=list(people) or None, - # TODO add secondary_owners (commenters) if needed - metadata=metadata_dict, - ) + yield Document( + id=page_url, + sections=[Section(link=page_url, text=ticket_content)], + source=DocumentSource.JIRA, + semantic_identifier=f"{issue.key}: {issue.fields.summary}", + title=f"{issue.key} {issue.fields.summary}", + doc_updated_at=time_str_to_utc(issue.fields.updated), + primary_owners=list(people) or None, + # TODO add secondary_owners (commenters) if needed + metadata=metadata_dict, ) - return doc_batch, len(batch) class JiraConnector(LoadConnector, PollConnector): def __init__( self, - jira_project_url: str, + jira_base_url: str, + jira_filter: str, comment_email_blacklist: list[str] | None = None, batch_size: int = INDEX_BATCH_SIZE, # if a ticket has one of the labels specified in this list, we will just @@ -199,28 +161,36 @@ def __init__( labels_to_skip: list[str] = JIRA_CONNECTOR_LABELS_TO_SKIP, ) -> None: self.batch_size = batch_size - self.jira_base, self.jira_project = extract_jira_project(jira_project_url) - self.jira_client: JIRA | None = None + self.jira_base = jira_base_url + self._jira_client: JIRA | None = None self._comment_email_blacklist = comment_email_blacklist or [] self.labels_to_skip = set(labels_to_skip) + self.jira_filter = jira_filter + @property def comment_email_blacklist(self) -> tuple: return tuple(email.strip() for email in self._comment_email_blacklist) + + @property + def jira_client(self) -> JIRA: + if self._jira_client is None: + raise ConnectorMissingCredentialError("Jira") + return self._jira_client def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None: api_token = credentials["jira_api_token"] # if user provide an email we assume it's cloud if "jira_user_email" in credentials: email = credentials["jira_user_email"] - self.jira_client = JIRA( + self._jira_client = JIRA( basic_auth=(email, api_token), server=self.jira_base, options={"rest_api_version": JIRA_API_VERSION}, ) else: - self.jira_client = JIRA( + self._jira_client = JIRA( token_auth=api_token, server=self.jira_base, options={"rest_api_version": JIRA_API_VERSION}, @@ -228,26 +198,22 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None return None def load_from_state(self) -> GenerateDocumentsOutput: - if self.jira_client is None: - raise ConnectorMissingCredentialError("Jira") - - start_ind = 0 - while True: - doc_batch, fetched_batch_size = fetch_jira_issues_batch( - jql=f"project = {self.jira_project}", - start_index=start_ind, - jira_client=self.jira_client, - batch_size=self.batch_size, - comment_email_blacklist=self.comment_email_blacklist, - labels_to_skip=self.labels_to_skip, - ) - - if doc_batch: - yield doc_batch + jql = f"project = {self.quoted_jira_project}" + + document_batch = [] + for doc in fetch_jira_issues_batch( + jira_client=self.jira_client, + jql=jql, + batch_size=_JIRA_FULL_PAGE_SIZE, + comment_email_blacklist=self.comment_email_blacklist, + labels_to_skip=self.labels_to_skip, + ): + document_batch.append(doc) + if len(document_batch) >= self.batch_size: + yield document_batch + document_batch = [] - start_ind += fetched_batch_size - if fetched_batch_size < self.batch_size: - break + yield document_batch def poll_source( self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch @@ -263,35 +229,32 @@ def poll_source( ) jql = ( - f"project = {self.jira_project} AND " + f"{self.jira_filter} AND " f"updated >= '{start_date_str}' AND " f"updated <= '{end_date_str}'" ) - start_ind = 0 - while True: - doc_batch, fetched_batch_size = fetch_jira_issues_batch( - jql=jql, - start_index=start_ind, - jira_client=self.jira_client, - batch_size=self.batch_size, - comment_email_blacklist=self.comment_email_blacklist, - labels_to_skip=self.labels_to_skip, - ) - - if doc_batch: - yield doc_batch + document_batch = [] + for doc in fetch_jira_issues_batch( + jira_client=self.jira_client, + jql=jql, + batch_size=_JIRA_FULL_PAGE_SIZE, + comment_email_blacklist=self.comment_email_blacklist, + labels_to_skip=self.labels_to_skip, + ): + document_batch.append(doc) + if len(document_batch) >= self.batch_size: + yield document_batch + document_batch = [] - start_ind += fetched_batch_size - if fetched_batch_size < self.batch_size: - break + yield document_batch if __name__ == "__main__": import os connector = JiraConnector( - os.environ["JIRA_PROJECT_URL"], comment_email_blacklist=[] + os.environ["JIRA_FILTERS"], comment_email_blacklist=[] ) connector.load_credentials( { diff --git a/backend/danswer/connectors/danswer_jira/utils.py b/backend/danswer/connectors/danswer_jira/utils.py index 506f5eff75e..fac9a52dac3 100644 --- a/backend/danswer/connectors/danswer_jira/utils.py +++ b/backend/danswer/connectors/danswer_jira/utils.py @@ -1,4 +1,5 @@ """Module with custom fields processing functions""" +import os from typing import Any from typing import List @@ -7,10 +8,80 @@ from jira.resources import Issue from jira.resources import User +from danswer.connectors.models import BasicExpertInfo from danswer.utils.logger import setup_logger logger = setup_logger() +JIRA_API_VERSION = os.environ.get("JIRA_API_VERSION") or "2" + + +def best_effort_basic_expert_info(obj: Any) -> BasicExpertInfo | None: + display_name = None + email = None + if hasattr(obj, "display_name"): + display_name = obj.display_name + else: + display_name = obj.get("displayName") + + if hasattr(obj, "emailAddress"): + email = obj.emailAddress + else: + email = obj.get("emailAddress") + + if not email and not display_name: + return None + + return BasicExpertInfo(display_name=display_name, email=email) + + +def best_effort_get_field_from_issue(jira_issue: Issue, field: str) -> Any: + if hasattr(jira_issue.fields, field): + return getattr(jira_issue.fields, field) + + try: + return jira_issue.raw["fields"][field] + except Exception: + return None + + +def extract_text_from_content(content: dict) -> str: + texts = [] + if "content" in content: + for block in content["content"]: + if "content" in block: + for item in block["content"]: + if item["type"] == "text": + texts.append(item["text"]) + return " ".join(texts) + + +def get_comment_strs( + issue: Issue, comment_email_blacklist: tuple[str, ...] = () +) -> list[str]: + comment_strs = [] + for comment in issue.fields.comment.comments: + try: + body_text = ( + comment.body + if JIRA_API_VERSION == "2" + else extract_text_from_content(comment.raw["body"]) + ) + + if ( + hasattr(comment, "author") + and hasattr(comment.author, "emailAddress") + and comment.author.emailAddress in comment_email_blacklist + ): + continue # Skip adding comment if author's email is in blacklist + + comment_strs.append(body_text) + except Exception as e: + logger.error(f"Failed to process comment due to an error: {e}") + continue + + return comment_strs + class CustomFieldExtractor: @staticmethod diff --git a/web/src/app/admin/connectors/jira/page.tsx b/web/src/app/admin/connectors/jira/page.tsx index f960348e6da..c4276fc4845 100644 --- a/web/src/app/admin/connectors/jira/page.tsx +++ b/web/src/app/admin/connectors/jira/page.tsx @@ -26,17 +26,6 @@ import { usePublicCredentials } from "@/lib/hooks"; import { AdminPageTitle } from "@/components/admin/Title"; import { Card, Divider, Text, Title } from "@tremor/react"; -// Copied from the `extract_jira_project` function -const extractJiraProject = (url: string): string | null => { - const parsedUrl = new URL(url); - const splitPath = parsedUrl.pathname.split("/"); - const projectPos = splitPath.indexOf("projects"); - if (projectPos !== -1 && splitPath.length > projectPos + 1) { - const jiraProject = splitPath[projectPos + 1]; - return jiraProject; - } - return null; -}; const Main = () => { const { popup, setPopup } = usePopup(); @@ -224,14 +213,7 @@ const Main = () => { <> {" "} - Specify any link to a Jira page below and click "Index" to - Index. Based on the provided link, we will index the ENTIRE PROJECT, - not just the specified page. For example, entering{" "} - - https://danswer.atlassian.net/jira/software/projects/DAN/boards/1 - {" "} - and clicking the Index button will index the whole DAN Jira - project. + Please specify the filters you want to use for indexing the Jira Issues. {jiraConnectorIndexingStatuses.length > 0 && ( <> @@ -261,19 +243,12 @@ const Main = () => { }} specialColumns={[ { - header: "Url", - key: "url", + header: "Filters", + key: "filters", getValue: (ccPairStatus) => { const connectorConfig = ccPairStatus.connector.connector_specific_config; - return ( - - {connectorConfig.jira_project_url} - - ); + return connectorConfig.jira_filter; }, }, { @@ -301,10 +276,10 @@ const Main = () => {

Add a New Project

nameBuilder={(values) => - `JiraConnector-${values.jira_project_url}` + `JiraConnector-${values.jira_filter}` } ccPairNameBuilder={(values) => - extractJiraProject(values.jira_project_url) + `JIRA - ${values.jira_filter}` } credentialId={jiraCredential.id} source="jira" @@ -312,8 +287,12 @@ const Main = () => { formBody={ <> + } @@ -331,15 +310,19 @@ const Main = () => { ); }} validationSchema={Yup.object().shape({ - jira_project_url: Yup.string().required( - "Please enter any link to your jira project e.g. https://danswer.atlassian.net/jira/software/projects/DAN/boards/1" + jira_base_url: Yup.string().required( + "Please provide the base url e.g. https://danswer.atlassian.net" + ), + jira_filter: Yup.string().required( + "Please provide some filters." ), comment_email_blacklist: Yup.array() .of(Yup.string().required("Emails names must be strings")) .required(), })} initialValues={{ - jira_project_url: "", + jira_base_url: "", + jira_filter: "", comment_email_blacklist: [], }} refreshFreq={10 * 60} // 10 minutes diff --git a/web/src/components/admin/connectors/ConnectorTitle.tsx b/web/src/components/admin/connectors/ConnectorTitle.tsx index a0065025921..01f357c4118 100644 --- a/web/src/components/admin/connectors/ConnectorTitle.tsx +++ b/web/src/components/admin/connectors/ConnectorTitle.tsx @@ -54,8 +54,8 @@ export const ConnectorTitle = ({ } else if (connector.source === "jira") { const typedConnector = connector as Connector; additionalMetadata.set( - "Jira Project URL", - typedConnector.connector_specific_config.jira_project_url + "Jira Filters", + typedConnector.connector_specific_config.jira_filter ); } else if (connector.source === "google_drive") { const typedConnector = connector as Connector; diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index ea6f3c7defc..0344b002ccf 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -137,7 +137,8 @@ export interface ConfluenceConfig { } export interface JiraConfig { - jira_project_url: string; + jira_base_url: string; + jira_filter: string; comment_email_blacklist?: string[]; }