From 6efdb9d713c08ef02098699e4d8d1df08d5042c0 Mon Sep 17 00:00:00 2001 From: Yoan Salambashev Date: Fri, 19 Jan 2024 16:04:23 +0200 Subject: [PATCH 1/3] examples: add Confluence data retrieval example Data Job --- .../10_fetch_confluence_document.py | 40 +++++++++++++++++ .../20_fetch_confluence_space.py | 44 +++++++++++++++++++ .../requirements.txt | 7 +++ 3 files changed, 91 insertions(+) create mode 100644 examples/confluence-data-retrieval-example/10_fetch_confluence_document.py create mode 100644 examples/confluence-data-retrieval-example/20_fetch_confluence_space.py create mode 100644 examples/confluence-data-retrieval-example/requirements.txt diff --git a/examples/confluence-data-retrieval-example/10_fetch_confluence_document.py b/examples/confluence-data-retrieval-example/10_fetch_confluence_document.py new file mode 100644 index 0000000000..ba29eaa568 --- /dev/null +++ b/examples/confluence-data-retrieval-example/10_fetch_confluence_document.py @@ -0,0 +1,40 @@ +# Copyright 2021-2024 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 +import logging +import os + +from langchain_community.document_loaders import ConfluenceLoader +from vdk.api.job_input import IJobInput + +log = logging.getLogger(__name__) + + +def fetch_confluence_document(url, token, document_id): + try: + # For more info regarding the LangChain ConfluenceLoader: + # https://python.langchain.com/docs/integrations/document_loaders/confluence + loader = ConfluenceLoader(url=url, token=token) + documents = loader.load(page_ids=[document_id]) + return documents[0] if documents else None + except Exception as e: + log.error(f"Error fetching document with ID {document_id} from Confluence: {e}") + return None + + +def run(job_input: IJobInput): + log.info(f"Starting job step {__name__}") + + # the internal Confluence URL is https://confluence.eng.vmware.com/ + confluence_url = os.environ.get("VDK_CONFLUENCE_URL") + # create at: https://confluence.eng.vmware.com/plugins/personalaccesstokens/usertokens.action + token = os.environ.get("VDK_CONFLUENCE_TOKEN") + # for example: 1260314928 + doc_id = os.environ.get("VDK_CONFLUENCE_DOC_ID") + + doc = fetch_confluence_document(confluence_url, token, doc_id) + + if doc: + log.info(f"Document with ID {doc_id} fetched successfully.") + print(doc.page_content) + else: + log.error(f"Failed to fetch the document with ID {doc_id}.") diff --git a/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py b/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py new file mode 100644 index 0000000000..1c100ff989 --- /dev/null +++ b/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py @@ -0,0 +1,44 @@ +# Copyright 2021-2024 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 +import logging +import os + +from langchain_community.document_loaders import ConfluenceLoader +from vdk.api.job_input import IJobInput + +log = logging.getLogger(__name__) + + +def fetch_confluence_space(url, token, space_key): + try: + # For more info regarding the LangChain ConfluenceLoader: + # https://python.langchain.com/docs/integrations/document_loaders/confluence + loader = ConfluenceLoader(url=url, token=token) + documents = loader.load( + space_key=space_key, include_attachments=True, limit=20, max_pages=20 + ) + return documents + except Exception as e: + log.error(f"Error fetching documents from Confluence: {e}") + return None + + +def run(job_input: IJobInput): + log.info(f"Starting job step {__name__}") + + # the internal Confluence URL is https://confluence.eng.vmware.com/ + confluence_url = os.environ.get("VDK_CONFLUENCE_URL") + # create at: https://confluence.eng.vmware.com/plugins/personalaccesstokens/usertokens.action + token = os.environ.get("VDK_CONFLUENCE_TOKEN") + # for the Super Collider space, the key is SuperCollider + space_key = os.environ.get("VDK_CONFLUENCE_SPACE_KEY") + + docs = fetch_confluence_space(confluence_url, token, space_key) + + if docs: + log.info(f"{len(docs)} documents fetched successfully.") + log.info(f"Printing the first 50 chars of each doc") + for i, doc in enumerate(docs): + print(f"{i}: {doc.page_content[:50]}") + else: + log.error(f"Failed to fetch any documents from the space with key {space_key}.") diff --git a/examples/confluence-data-retrieval-example/requirements.txt b/examples/confluence-data-retrieval-example/requirements.txt new file mode 100644 index 0000000000..8ef381f7a6 --- /dev/null +++ b/examples/confluence-data-retrieval-example/requirements.txt @@ -0,0 +1,7 @@ +# Python jobs can specify extra library dependencies in requirements.txt file. +# See https://pip.readthedocs.io/en/stable/user_guide/#requirements-files +# The file is optional and can be deleted if no extra library dependencies are necessary. + +atlassian-python-api +langchain_community +lxml From b43ddfc6c1f53e176b00e64fd0c6885b48fe34ed Mon Sep 17 00:00:00 2001 From: Yoan Salambashev Date: Fri, 19 Jan 2024 16:20:16 +0200 Subject: [PATCH 2/3] examples: remove redundant comments Signed-off-by: Yoan Salambashev --- .../10_fetch_confluence_document.py | 3 --- .../20_fetch_confluence_space.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/examples/confluence-data-retrieval-example/10_fetch_confluence_document.py b/examples/confluence-data-retrieval-example/10_fetch_confluence_document.py index ba29eaa568..e2047141ca 100644 --- a/examples/confluence-data-retrieval-example/10_fetch_confluence_document.py +++ b/examples/confluence-data-retrieval-example/10_fetch_confluence_document.py @@ -24,11 +24,8 @@ def fetch_confluence_document(url, token, document_id): def run(job_input: IJobInput): log.info(f"Starting job step {__name__}") - # the internal Confluence URL is https://confluence.eng.vmware.com/ confluence_url = os.environ.get("VDK_CONFLUENCE_URL") - # create at: https://confluence.eng.vmware.com/plugins/personalaccesstokens/usertokens.action token = os.environ.get("VDK_CONFLUENCE_TOKEN") - # for example: 1260314928 doc_id = os.environ.get("VDK_CONFLUENCE_DOC_ID") doc = fetch_confluence_document(confluence_url, token, doc_id) diff --git a/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py b/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py index 1c100ff989..eb6622dacc 100644 --- a/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py +++ b/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py @@ -26,11 +26,8 @@ def fetch_confluence_space(url, token, space_key): def run(job_input: IJobInput): log.info(f"Starting job step {__name__}") - # the internal Confluence URL is https://confluence.eng.vmware.com/ confluence_url = os.environ.get("VDK_CONFLUENCE_URL") - # create at: https://confluence.eng.vmware.com/plugins/personalaccesstokens/usertokens.action token = os.environ.get("VDK_CONFLUENCE_TOKEN") - # for the Super Collider space, the key is SuperCollider space_key = os.environ.get("VDK_CONFLUENCE_SPACE_KEY") docs = fetch_confluence_space(confluence_url, token, space_key) From 635167c1050558fac4bdb8ae689aff42fa7e4d67 Mon Sep 17 00:00:00 2001 From: Yoan Salambashev Date: Mon, 22 Jan 2024 11:02:42 +0200 Subject: [PATCH 3/3] examples: fix log string Signed-off-by: Yoan Salambashev --- .../20_fetch_confluence_space.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py b/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py index eb6622dacc..cdd1a58e95 100644 --- a/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py +++ b/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py @@ -34,7 +34,7 @@ def run(job_input: IJobInput): if docs: log.info(f"{len(docs)} documents fetched successfully.") - log.info(f"Printing the first 50 chars of each doc") + log.info("Printing the first 50 chars of each doc") for i, doc in enumerate(docs): print(f"{i}: {doc.page_content[:50]}") else: