From 4e5459bd0be752ec4c52760f2967975c3a1f3cba Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 11 Mar 2025 20:14:58 +0000 Subject: [PATCH 01/11] feat: add deptry tests for dependency analysis Co-Authored-By: Aaron Steers --- .../workflows/python_dependency_analysis.yml | 38 +++++++++++++++++ pyproject.toml | 42 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 .github/workflows/python_dependency_analysis.yml diff --git a/.github/workflows/python_dependency_analysis.yml b/.github/workflows/python_dependency_analysis.yml new file mode 100644 index 000000000..27765e702 --- /dev/null +++ b/.github/workflows/python_dependency_analysis.yml @@ -0,0 +1,38 @@ +name: Dependency Analysis + +on: + push: + branches: + - main + paths: + - "airbyte_cdk/**" + - "poetry.lock" + - "pyproject.toml" + pull_request: + paths: + - "airbyte_cdk/**" + - "poetry.lock" + - "pyproject.toml" + +jobs: + dependency-analysis: + name: Dependency Analysis with Deptry + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "2.0.1" + - name: Install dependencies + run: poetry install --all-extras + + # Job-specific step(s): + - name: Run Deptry + run: | + poetry run deptry . diff --git a/pyproject.toml b/pyproject.toml index 7f8201692..1b467fe05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,6 +102,7 @@ types-requests = "^2.32.0.20241016" types-python-dateutil = "^2.9.0.20241003" types-pyyaml = "^6.0.12.20240917" types-cachetools = "^5.5.0.20240820" +deptry = "^0.23.0" [tool.poetry.extras] file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "python-snappy"] @@ -196,3 +197,44 @@ optional_poetry_groups = ["dev"] poetry_extras = ["file-based", "vector-db-based"] poe_tasks = ["check-ci"] mount_docker_socket = true + +[tool.deptry.per_rule_ignores] +# This is a mapping of rules and package names to be ignored for that rule. +DEP001 = [ + # Optional dependencies that might be imported conditionally + "dagger", # Used in bin scripts but not a core dependency + "source_declarative_manifest" # Used in tests but not a core dependency +] +DEP004 = [ + "pdoc", # Only used for generating documentation. Not a runtime dependency. + "pytest", # Test dependencies + "freezegun", # Test dependencies + "requests_mock", # Test dependencies + "_pytest", # Test dependencies + "pympler" # Test dependencies +] +DEP002 = [ + # Dependencies that are imported indirectly through other packages + "sqlalchemy", # Used through extras + # Optional dependencies in extras + "psutil", + "rapidfuzz", + "cohere", + "markdown", + "openai", + "pdf2image", + "pdfminer.six", + "pytesseract", + "python-calamine", + "python-snappy", + "tiktoken", + "unstructured.pytesseract", + "cryptography" +] +DEP003 = [ + # Transitive dependencies that are imported directly + "typing_extensions", # Common transitive dependency + "packaging", # Common transitive dependency + "pydantic_core", # Pydantic internals + "anyio" # Async dependency +] From 16b2e968de663a7f3a35360031613921bcc8acd2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 11 Mar 2025 20:30:46 +0000 Subject: [PATCH 02/11] refactor: improve deptry configuration with rule descriptions and links Co-Authored-By: Aaron Steers --- pyproject.toml | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1b467fe05..517b4e022 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,19 +200,17 @@ mount_docker_socket = true [tool.deptry.per_rule_ignores] # This is a mapping of rules and package names to be ignored for that rule. + +# DEP001: Unused direct dependencies - packages that are declared as dependencies but not imported +# https://deptry.com/rules/dep001/ DEP001 = [ # Optional dependencies that might be imported conditionally "dagger", # Used in bin scripts but not a core dependency "source_declarative_manifest" # Used in tests but not a core dependency ] -DEP004 = [ - "pdoc", # Only used for generating documentation. Not a runtime dependency. - "pytest", # Test dependencies - "freezegun", # Test dependencies - "requests_mock", # Test dependencies - "_pytest", # Test dependencies - "pympler" # Test dependencies -] + +# DEP002: Missing dependencies - packages that are imported but not declared as dependencies +# https://deptry.com/rules/dep002/ DEP002 = [ # Dependencies that are imported indirectly through other packages "sqlalchemy", # Used through extras @@ -231,10 +229,24 @@ DEP002 = [ "unstructured.pytesseract", "cryptography" ] + +# DEP003: Transitive dependencies - packages that are imported directly +# https://deptry.com/rules/dep003/ DEP003 = [ # Transitive dependencies that are imported directly - "typing_extensions", # Common transitive dependency - "packaging", # Common transitive dependency + "typing_extensions", # Common transitive dependency - "*" + "packaging", # Common transitive dependency - "*" "pydantic_core", # Pydantic internals - "anyio" # Async dependency + "anyio" # Async dependency - "*" +] + +# DEP004: Dev dependencies imported in main code - packages that are imported in non-test code but declared as dev dependencies +# https://deptry.com/rules/dep004/ +DEP004 = [ + "pdoc", # Only used for generating documentation. Not a runtime dependency. + "pytest", # Test dependencies + "freezegun", # Test dependencies + "requests_mock", # Test dependencies + "_pytest", # Test dependencies + "pympler" # Test dependencies ] From b829ad5f0d21b43ed6a592a62f3ca3381afacb37 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 11 Mar 2025 20:31:32 +0000 Subject: [PATCH 03/11] fix: add transitive dependencies as direct dependencies and re-lock poetry Co-Authored-By: Aaron Steers --- poetry.lock | 101 ++++++++++++++++++++++++++++++++++++++++++++++--- pyproject.toml | 8 ++-- 2 files changed, 99 insertions(+), 10 deletions(-) diff --git a/poetry.lock b/poetry.lock index 94f50c0a7..a01976469 100644 --- a/poetry.lock +++ b/poetry.lock @@ -172,7 +172,7 @@ files = [ name = "anyio" version = "4.8.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" -optional = true +optional = false python-versions = ">=3.9" groups = ["main"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" @@ -575,9 +575,9 @@ files = [ name = "click" version = "8.1.8" description = "Composable command line interface toolkit" -optional = true +optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["main", "dev"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, @@ -619,7 +619,7 @@ files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "(python_version <= \"3.11\" or python_version >= \"3.12\") and platform_system == \"Windows\"", dev = "(python_version <= \"3.11\" or python_version >= \"3.12\") and sys_platform == \"win32\""} +markers = {main = "(python_version <= \"3.11\" or python_version >= \"3.12\") and platform_system == \"Windows\"", dev = "(platform_system == \"Windows\" or sys_platform == \"win32\") and (python_version <= \"3.11\" or python_version >= \"3.12\")"} [[package]] name = "contourpy" @@ -972,6 +972,40 @@ files = [ marshmallow = ">=3.18.0,<4.0.0" typing-inspect = ">=0.4.0,<1" +[[package]] +name = "deptry" +version = "0.23.0" +description = "A command line utility to check for unused, missing and transitive dependencies in a Python project." +optional = false +python-versions = ">=3.9" +groups = ["dev"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "deptry-0.23.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:1f2a6817a37d76e8f6b667381b7caf6ea3e6d6c18b5be24d36c625f387c79852"}, + {file = "deptry-0.23.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:9601b64cc0aed42687fdd5c912d5f1e90d7f7333fb589b14e35bfdfebae866f3"}, + {file = "deptry-0.23.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6172b2205f6e84bcc9df25226693d4deb9576a6f746c2ace828f6d13401d357"}, + {file = "deptry-0.23.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1cfa4b3a46ee8a026eaa38e4b9ba43fe6036a07fe16bf0a663cb611b939f6af8"}, + {file = "deptry-0.23.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9d03cc99a61c348df92074a50e0a71b28f264f0edbf686084ca90e6fd44e3abe"}, + {file = "deptry-0.23.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9a46f78098f145100dc582a59af8548b26cdfa16cf0fbd85d2d44645e724cb6a"}, + {file = "deptry-0.23.0-cp39-abi3-win_amd64.whl", hash = "sha256:d53e803b280791d89a051b6183d9dc40411200e22a8ab7e6c32c6b169822a664"}, + {file = "deptry-0.23.0-cp39-abi3-win_arm64.whl", hash = "sha256:da7678624f4626d839c8c03675452cefc59d6cf57d25c84a9711dae514719279"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:40706dcbed54141f2d23afa70a272171c8c46531cd6f0f9c8ef482c906b3cee2"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:889541844092f18e7b48631852195f36c25c5afd4d7e074b19ba824b430add50"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aff9156228eb16cd81792f920c1623c00cb59091ae572600ba0eac587da33c0c"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:583154732cfd438a4a090b7d13d8b2016f1ac2732534f34fb689345768d8538b"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:736e7bc557aec6118b2a4d454f0d81f070782faeaa9d8d3c9a15985c9f265372"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5f7e4b1a5232ed6d352fca7173750610a169377d1951d3e9782947191942a765"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:04afae204654542406318fd3dd6f4a6697579597f37195437daf84a53ee0ebbf"}, + {file = "deptry-0.23.0.tar.gz", hash = "sha256:4915a3590ccf38ad7a9176aee376745aa9de121f50f8da8fb9ccec87fa93e676"}, +] + +[package.dependencies] +click = ">=8.0.0,<9" +colorama = {version = ">=0.4.6", markers = "sys_platform == \"win32\""} +packaging = ">=23.2" +requirements-parser = ">=0.11.0,<1" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} + [[package]] name = "dpath" version = "2.2.0" @@ -4344,6 +4378,23 @@ files = [ [package.dependencies] requests = ">=2.0.1,<3.0.0" +[[package]] +name = "requirements-parser" +version = "0.11.0" +description = "This is a small Python module for parsing Pip requirement files." +optional = false +python-versions = "<4.0,>=3.8" +groups = ["dev"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "requirements_parser-0.11.0-py3-none-any.whl", hash = "sha256:50379eb50311834386c2568263ae5225d7b9d0867fb55cf4ecc93959de2c2684"}, + {file = "requirements_parser-0.11.0.tar.gz", hash = "sha256:35f36dc969d14830bf459803da84f314dc3d17c802592e9e970f63d0359e5920"}, +] + +[package.dependencies] +packaging = ">=23.2" +types-setuptools = ">=69.1.0" + [[package]] name = "rich" version = "13.9.4" @@ -4565,6 +4616,28 @@ files = [ attributes-doc = "*" typing-extensions = "*" +[[package]] +name = "setuptools" +version = "76.0.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "setuptools-76.0.0-py3-none-any.whl", hash = "sha256:199466a166ff664970d0ee145839f5582cb9bca7a0a3a2e795b6a9cb2308e9c6"}, + {file = "setuptools-76.0.0.tar.gz", hash = "sha256:43b4ee60e10b0d0ee98ad11918e114c70701bc6051662a9a675a0496c1a158f4"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] +core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] + [[package]] name = "six" version = "1.17.0" @@ -4582,7 +4655,7 @@ files = [ name = "sniffio" version = "1.3.1" description = "Sniff out which async library your code is running under" -optional = true +optional = false python-versions = ">=3.7" groups = ["main"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" @@ -4952,6 +5025,22 @@ files = [ [package.dependencies] urllib3 = ">=2" +[[package]] +name = "types-setuptools" +version = "75.8.2.20250305" +description = "Typing stubs for setuptools" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "types_setuptools-75.8.2.20250305-py3-none-any.whl", hash = "sha256:ba80953fd1f5f49e552285c024f75b5223096a38a5138a54d18ddd3fa8f6a2d4"}, + {file = "types_setuptools-75.8.2.20250305.tar.gz", hash = "sha256:a987269b49488f21961a1d99aa8d281b611625883def6392a93855b31544e405"}, +] + +[package.dependencies] +setuptools = "*" + [[package]] name = "typing-extensions" version = "4.12.2" @@ -5418,4 +5507,4 @@ vector-db-based = ["cohere", "langchain", "openai", "tiktoken"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "dc50be30e74dea64afa594eb2267a0b782cb5cac11ae7f5c52b09269982b3ffd" +content-hash = "b7c97b1f33821f003aff39e73bd4aac99619287ffa57cfdc34d4c9795f87bd35" diff --git a/pyproject.toml b/pyproject.toml index 517b4e022..168977f55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ enable = true [tool.poetry.dependencies] python = ">=3.10,<3.13" airbyte-protocol-models-dataclasses = "^0.14" +anyio = "*" # Transitive dependency used directly in code backoff = "*" cachetools = "*" dpath = "^2.1.6" @@ -40,6 +41,7 @@ isodate = "~0.6.1" Jinja2 = "~3.1.2" jsonref = "~0.2" jsonschema = "~4.17.3" # 4.18 has some significant breaking changes: https://github.com/python-jsonschema/jsonschema/releases/tag/v4.18.0 +packaging = "*" # Transitive dependency used directly in code pandas = "2.2.2" psutil = "6.1.0" pydantic = "^2.7" @@ -50,6 +52,7 @@ PyYAML = "^6.0.1" rapidfuzz = "^3.10.1" requests = "*" requests_cache = "*" +typing-extensions = "*" # Transitive dependency used directly in code wcmatch = "10.0" # Extras depedencies avro = { version = ">=1.11.2,<1.13.0", optional = true } @@ -234,10 +237,7 @@ DEP002 = [ # https://deptry.com/rules/dep003/ DEP003 = [ # Transitive dependencies that are imported directly - "typing_extensions", # Common transitive dependency - "*" - "packaging", # Common transitive dependency - "*" - "pydantic_core", # Pydantic internals - "anyio" # Async dependency - "*" + "pydantic_core" # Pydantic internals ] # DEP004: Dev dependencies imported in main code - packages that are imported in non-test code but declared as dev dependencies From f0283185a77d7f1859fc98ebae0f8b92da9edf87 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 11 Mar 2025 21:28:54 +0000 Subject: [PATCH 04/11] docs: update deptry rule documentation links Co-Authored-By: Aaron Steers --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 168977f55..2025bb14e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -205,7 +205,7 @@ mount_docker_socket = true # This is a mapping of rules and package names to be ignored for that rule. # DEP001: Unused direct dependencies - packages that are declared as dependencies but not imported -# https://deptry.com/rules/dep001/ +# https://deptry.com/rules-violations/#unused-dependencies-dep001 DEP001 = [ # Optional dependencies that might be imported conditionally "dagger", # Used in bin scripts but not a core dependency @@ -213,7 +213,7 @@ DEP001 = [ ] # DEP002: Missing dependencies - packages that are imported but not declared as dependencies -# https://deptry.com/rules/dep002/ +# https://deptry.com/rules-violations/#missing-dependencies-dep002 DEP002 = [ # Dependencies that are imported indirectly through other packages "sqlalchemy", # Used through extras @@ -234,14 +234,14 @@ DEP002 = [ ] # DEP003: Transitive dependencies - packages that are imported directly -# https://deptry.com/rules/dep003/ +# https://deptry.com/rules-violations/#transitive-dependencies-dep003 DEP003 = [ # Transitive dependencies that are imported directly "pydantic_core" # Pydantic internals ] # DEP004: Dev dependencies imported in main code - packages that are imported in non-test code but declared as dev dependencies -# https://deptry.com/rules/dep004/ +# https://deptry.com/rules-violations/#misplaced-development-dependencies-dep004 DEP004 = [ "pdoc", # Only used for generating documentation. Not a runtime dependency. "pytest", # Test dependencies From 94e568d6e65b4a9d213045ec1b9d39fdcb5dcb72 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 11 Mar 2025 15:03:51 -0700 Subject: [PATCH 05/11] clean up deptry config and dependencies --- pyproject.toml | 57 ++++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2025bb14e..a644c59db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,13 +43,13 @@ jsonref = "~0.2" jsonschema = "~4.17.3" # 4.18 has some significant breaking changes: https://github.com/python-jsonschema/jsonschema/releases/tag/v4.18.0 packaging = "*" # Transitive dependency used directly in code pandas = "2.2.2" -psutil = "6.1.0" +psutil = "6.1.0" # TODO: Remove if unused pydantic = "^2.7" pyrate-limiter = "~3.1.0" python-dateutil = "^2.9.0" python-ulid = "^3.0.0" PyYAML = "^6.0.1" -rapidfuzz = "^3.10.1" +rapidfuzz = "^3.10.1" # TODO: Remove if unused requests = "*" requests_cache = "*" typing-extensions = "*" # Transitive dependency used directly in code @@ -60,14 +60,14 @@ cohere = { version = "4.21", optional = true } fastavro = { version = "~1.8.0", optional = true } langchain = { version = "0.1.16", optional = true } langchain_core = { version = "0.1.42", optional = true } -markdown = { version = "*", optional = true } -openai = { version = "0.27.9", extras = ["embeddings"], optional = true } +markdown = { version = "*", optional = true } # TODO: Remove if unused +openai = { version = "0.27.9", extras = ["embeddings"], optional = true } # Used indirectly by langchain library pdf2image = { version = "1.16.3", optional = true } -"pdfminer.six" = { version = "20221105", optional = true } +"pdfminer.six" = { version = "20221105", optional = true } # Used indirectly by unstructured library pyarrow = { version = "^19.0.0", optional = true } -pytesseract = { version = "0.3.10", optional = true } -python-calamine = { version = "0.2.3", optional = true } -python-snappy = { version = "0.7.3", optional = true } +pytesseract = { version = "0.3.10", optional = true } # Used indirectly by unstructured library +python-calamine = { version = "0.2.3", optional = true } # Used indirectly by unstructured library +python-snappy = { version = "0.7.3", optional = true } # Used indirectly by unstructured library tiktoken = { version = "0.8.0", optional = true } nltk = { version = "3.9.1", optional = true } # This will ensure that even when you run poetry install or pip install, the compatible version of numpy will always be chosen. @@ -76,7 +76,7 @@ numpy = "<2" unstructured = { version = "0.10.27", extras = ["docx", "pptx"], optional = true } "unstructured.pytesseract" = { version = ">=0.3.12", optional = true } pyjwt = "^2.8.0" -cryptography = ">=44.0.0,<45.0.0" +cryptography = ">=44.0.0,<45.0.0" # Constrained as transitive dependency due to a bug in newer versions pytz = "2024.2" orjson = "^3.10.7" serpyco-rs = "^1.10.2" @@ -201,24 +201,30 @@ poetry_extras = ["file-based", "vector-db-based"] poe_tasks = ["check-ci"] mount_docker_socket = true +[tool.deptry] +exclude = [ + "bin", + "docs", + "unit_tests", +] + [tool.deptry.per_rule_ignores] # This is a mapping of rules and package names to be ignored for that rule. -# DEP001: Unused direct dependencies - packages that are declared as dependencies but not imported -# https://deptry.com/rules-violations/#unused-dependencies-dep001 +# DEP001: Project should not contain missing dependencies. +# https://deptry.com/rules-violations/#missing-dependencies-dep001 DEP001 = [ - # Optional dependencies that might be imported conditionally - "dagger", # Used in bin scripts but not a core dependency - "source_declarative_manifest" # Used in tests but not a core dependency + # These are imported but not declared: + "dagger", # Used in `bin` scripts but not in main code + "source_declarative_manifest" # Imported only in dynamic import tests, not in main code ] -# DEP002: Missing dependencies - packages that are imported but not declared as dependencies +# DEP002: # https://deptry.com/rules-violations/#missing-dependencies-dep002 DEP002 = [ - # Dependencies that are imported indirectly through other packages - "sqlalchemy", # Used through extras - # Optional dependencies in extras - "psutil", + "cryptography", # Constrained as transitive dependency due to a bug in newer versions + + # TODO: Remove these dependencies if not needed: "rapidfuzz", "cohere", "markdown", @@ -230,23 +236,20 @@ DEP002 = [ "python-snappy", "tiktoken", "unstructured.pytesseract", - "cryptography" ] # DEP003: Transitive dependencies - packages that are imported directly # https://deptry.com/rules-violations/#transitive-dependencies-dep003 DEP003 = [ # Transitive dependencies that are imported directly - "pydantic_core" # Pydantic internals + "pydantic_core" # Pydantic internals, no need to define as separate dependency ] # DEP004: Dev dependencies imported in main code - packages that are imported in non-test code but declared as dev dependencies # https://deptry.com/rules-violations/#misplaced-development-dependencies-dep004 DEP004 = [ - "pdoc", # Only used for generating documentation. Not a runtime dependency. - "pytest", # Test dependencies - "freezegun", # Test dependencies - "requests_mock", # Test dependencies - "_pytest", # Test dependencies - "pympler" # Test dependencies + # The `airbyte_cdk.test.utils` module is main code. + # These should probably be declared in a `tests` extra (TODO): + "pytest", + "requests_mock", ] From e5fc4aa34db35a6479ebab466b317d9d0682bc29 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 11 Mar 2025 15:32:36 -0700 Subject: [PATCH 06/11] add missing DEP002 ignores --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a644c59db..b259029ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,6 +225,9 @@ DEP002 = [ "cryptography", # Constrained as transitive dependency due to a bug in newer versions # TODO: Remove these dependencies if not needed: + "anyio", + "avro", + "psutil", "rapidfuzz", "cohere", "markdown", From 25cd2c9a3b45a52b5fc6ddd3c7d2e67cc04faa49 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 11 Mar 2025 15:40:41 -0700 Subject: [PATCH 07/11] update pyproject.toml --- pyproject.toml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b259029ee..a218e6c51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ enable = true [tool.poetry.dependencies] python = ">=3.10,<3.13" airbyte-protocol-models-dataclasses = "^0.14" -anyio = "*" # Transitive dependency used directly in code backoff = "*" cachetools = "*" dpath = "^2.1.6" @@ -55,7 +54,7 @@ requests_cache = "*" typing-extensions = "*" # Transitive dependency used directly in code wcmatch = "10.0" # Extras depedencies -avro = { version = ">=1.11.2,<1.13.0", optional = true } +avro = { version = ">=1.11.2,<1.13.0", optional = true } # TODO: Remove if unused cohere = { version = "4.21", optional = true } fastavro = { version = "~1.8.0", optional = true } langchain = { version = "0.1.16", optional = true } @@ -219,14 +218,13 @@ DEP001 = [ "source_declarative_manifest" # Imported only in dynamic import tests, not in main code ] -# DEP002: +# DEP002: Project should not contain unused dependencies. # https://deptry.com/rules-violations/#missing-dependencies-dep002 DEP002 = [ "cryptography", # Constrained as transitive dependency due to a bug in newer versions # TODO: Remove these dependencies if not needed: - "anyio", - "avro", + "avro", # Only imported in `unit_tests` code "psutil", "rapidfuzz", "cohere", @@ -241,18 +239,18 @@ DEP002 = [ "unstructured.pytesseract", ] -# DEP003: Transitive dependencies - packages that are imported directly +# DEP003: Project should not use transitive dependencies. # https://deptry.com/rules-violations/#transitive-dependencies-dep003 DEP003 = [ # Transitive dependencies that are imported directly "pydantic_core" # Pydantic internals, no need to define as separate dependency ] -# DEP004: Dev dependencies imported in main code - packages that are imported in non-test code but declared as dev dependencies +# DEP004: Project should not use development dependencies in non-development code. # https://deptry.com/rules-violations/#misplaced-development-dependencies-dep004 DEP004 = [ # The `airbyte_cdk.test.utils` module is main code. - # These should probably be declared in a `tests` extra (TODO): + # TODO: These should probably be declared within a `tests` extra: "pytest", "requests_mock", ] From eda73a8480330918341096b819905fdd7ed97704 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 11 Mar 2025 15:42:14 -0700 Subject: [PATCH 08/11] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a218e6c51..5e2dc87a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ requests_cache = "*" typing-extensions = "*" # Transitive dependency used directly in code wcmatch = "10.0" # Extras depedencies -avro = { version = ">=1.11.2,<1.13.0", optional = true } # TODO: Remove if unused +avro = { version = ">=1.11.2,<1.13.0", optional = true } # TODO: Move into dev dependencies if only used in tests cohere = { version = "4.21", optional = true } fastavro = { version = "~1.8.0", optional = true } langchain = { version = "0.1.16", optional = true } From 3a21cd70494ab8d981991c8f855be73cb9d3dde8 Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Tue, 11 Mar 2025 22:44:39 +0000 Subject: [PATCH 09/11] Auto-commit Resolving dependencies... changes --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index a01976469..992f7f8f8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -172,7 +172,7 @@ files = [ name = "anyio" version = "4.8.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" @@ -4655,7 +4655,7 @@ files = [ name = "sniffio" version = "1.3.1" description = "Sniff out which async library your code is running under" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" @@ -5507,4 +5507,4 @@ vector-db-based = ["cohere", "langchain", "openai", "tiktoken"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "b7c97b1f33821f003aff39e73bd4aac99619287ffa57cfdc34d4c9795f87bd35" +content-hash = "c8731f26643e07136e524d5e0d6e0f5c2229cf63d43bf5644de9f1cf8e565197" From 416a8a54ff121fe1af78e4fe54e9f588e9084a95 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 11 Mar 2025 19:13:04 -0700 Subject: [PATCH 10/11] Update pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5e2dc87a9..f0275bc31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,8 +65,8 @@ pdf2image = { version = "1.16.3", optional = true } "pdfminer.six" = { version = "20221105", optional = true } # Used indirectly by unstructured library pyarrow = { version = "^19.0.0", optional = true } pytesseract = { version = "0.3.10", optional = true } # Used indirectly by unstructured library -python-calamine = { version = "0.2.3", optional = true } # Used indirectly by unstructured library -python-snappy = { version = "0.7.3", optional = true } # Used indirectly by unstructured library +python-calamine = { version = "0.2.3", optional = true } # TODO: Remove if unused +python-snappy = { version = "0.7.3", optional = true } # TODO: remove if unused tiktoken = { version = "0.8.0", optional = true } nltk = { version = "3.9.1", optional = true } # This will ensure that even when you run poetry install or pip install, the compatible version of numpy will always be chosen. From 8e42dc287bcf4bd3c8a2b6d24f163fc11759039e Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 11 Mar 2025 19:14:09 -0700 Subject: [PATCH 11/11] Update pyproject.toml --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f0275bc31..d236c0b9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -214,7 +214,6 @@ exclude = [ # https://deptry.com/rules-violations/#missing-dependencies-dep001 DEP001 = [ # These are imported but not declared: - "dagger", # Used in `bin` scripts but not in main code "source_declarative_manifest" # Imported only in dynamic import tests, not in main code ]