diff --git a/.github/workflows/python_dependency_analysis.yml b/.github/workflows/python_dependency_analysis.yml new file mode 100644 index 000000000..27765e702 --- /dev/null +++ b/.github/workflows/python_dependency_analysis.yml @@ -0,0 +1,38 @@ +name: Dependency Analysis + +on: + push: + branches: + - main + paths: + - "airbyte_cdk/**" + - "poetry.lock" + - "pyproject.toml" + pull_request: + paths: + - "airbyte_cdk/**" + - "poetry.lock" + - "pyproject.toml" + +jobs: + dependency-analysis: + name: Dependency Analysis with Deptry + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "2.0.1" + - name: Install dependencies + run: poetry install --all-extras + + # Job-specific step(s): + - name: Run Deptry + run: | + poetry run deptry . diff --git a/poetry.lock b/poetry.lock index 94f50c0a7..992f7f8f8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -575,9 +575,9 @@ files = [ name = "click" version = "8.1.8" description = "Composable command line interface toolkit" -optional = true +optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["main", "dev"] markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" files = [ {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, @@ -619,7 +619,7 @@ files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "(python_version <= \"3.11\" or python_version >= \"3.12\") and platform_system == \"Windows\"", dev = "(python_version <= \"3.11\" or python_version >= \"3.12\") and sys_platform == \"win32\""} +markers = {main = "(python_version <= \"3.11\" or python_version >= \"3.12\") and platform_system == \"Windows\"", dev = "(platform_system == \"Windows\" or sys_platform == \"win32\") and (python_version <= \"3.11\" or python_version >= \"3.12\")"} [[package]] name = "contourpy" @@ -972,6 +972,40 @@ files = [ marshmallow = ">=3.18.0,<4.0.0" typing-inspect = ">=0.4.0,<1" +[[package]] +name = "deptry" +version = "0.23.0" +description = "A command line utility to check for unused, missing and transitive dependencies in a Python project." +optional = false +python-versions = ">=3.9" +groups = ["dev"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "deptry-0.23.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:1f2a6817a37d76e8f6b667381b7caf6ea3e6d6c18b5be24d36c625f387c79852"}, + {file = "deptry-0.23.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:9601b64cc0aed42687fdd5c912d5f1e90d7f7333fb589b14e35bfdfebae866f3"}, + {file = "deptry-0.23.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6172b2205f6e84bcc9df25226693d4deb9576a6f746c2ace828f6d13401d357"}, + {file = "deptry-0.23.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1cfa4b3a46ee8a026eaa38e4b9ba43fe6036a07fe16bf0a663cb611b939f6af8"}, + {file = "deptry-0.23.0-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9d03cc99a61c348df92074a50e0a71b28f264f0edbf686084ca90e6fd44e3abe"}, + {file = "deptry-0.23.0-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9a46f78098f145100dc582a59af8548b26cdfa16cf0fbd85d2d44645e724cb6a"}, + {file = "deptry-0.23.0-cp39-abi3-win_amd64.whl", hash = "sha256:d53e803b280791d89a051b6183d9dc40411200e22a8ab7e6c32c6b169822a664"}, + {file = "deptry-0.23.0-cp39-abi3-win_arm64.whl", hash = "sha256:da7678624f4626d839c8c03675452cefc59d6cf57d25c84a9711dae514719279"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:40706dcbed54141f2d23afa70a272171c8c46531cd6f0f9c8ef482c906b3cee2"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:889541844092f18e7b48631852195f36c25c5afd4d7e074b19ba824b430add50"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aff9156228eb16cd81792f920c1623c00cb59091ae572600ba0eac587da33c0c"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:583154732cfd438a4a090b7d13d8b2016f1ac2732534f34fb689345768d8538b"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:736e7bc557aec6118b2a4d454f0d81f070782faeaa9d8d3c9a15985c9f265372"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5f7e4b1a5232ed6d352fca7173750610a169377d1951d3e9782947191942a765"}, + {file = "deptry-0.23.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:04afae204654542406318fd3dd6f4a6697579597f37195437daf84a53ee0ebbf"}, + {file = "deptry-0.23.0.tar.gz", hash = "sha256:4915a3590ccf38ad7a9176aee376745aa9de121f50f8da8fb9ccec87fa93e676"}, +] + +[package.dependencies] +click = ">=8.0.0,<9" +colorama = {version = ">=0.4.6", markers = "sys_platform == \"win32\""} +packaging = ">=23.2" +requirements-parser = ">=0.11.0,<1" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} + [[package]] name = "dpath" version = "2.2.0" @@ -4344,6 +4378,23 @@ files = [ [package.dependencies] requests = ">=2.0.1,<3.0.0" +[[package]] +name = "requirements-parser" +version = "0.11.0" +description = "This is a small Python module for parsing Pip requirement files." +optional = false +python-versions = "<4.0,>=3.8" +groups = ["dev"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "requirements_parser-0.11.0-py3-none-any.whl", hash = "sha256:50379eb50311834386c2568263ae5225d7b9d0867fb55cf4ecc93959de2c2684"}, + {file = "requirements_parser-0.11.0.tar.gz", hash = "sha256:35f36dc969d14830bf459803da84f314dc3d17c802592e9e970f63d0359e5920"}, +] + +[package.dependencies] +packaging = ">=23.2" +types-setuptools = ">=69.1.0" + [[package]] name = "rich" version = "13.9.4" @@ -4565,6 +4616,28 @@ files = [ attributes-doc = "*" typing-extensions = "*" +[[package]] +name = "setuptools" +version = "76.0.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "setuptools-76.0.0-py3-none-any.whl", hash = "sha256:199466a166ff664970d0ee145839f5582cb9bca7a0a3a2e795b6a9cb2308e9c6"}, + {file = "setuptools-76.0.0.tar.gz", hash = "sha256:43b4ee60e10b0d0ee98ad11918e114c70701bc6051662a9a675a0496c1a158f4"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] +core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] + [[package]] name = "six" version = "1.17.0" @@ -4952,6 +5025,22 @@ files = [ [package.dependencies] urllib3 = ">=2" +[[package]] +name = "types-setuptools" +version = "75.8.2.20250305" +description = "Typing stubs for setuptools" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "types_setuptools-75.8.2.20250305-py3-none-any.whl", hash = "sha256:ba80953fd1f5f49e552285c024f75b5223096a38a5138a54d18ddd3fa8f6a2d4"}, + {file = "types_setuptools-75.8.2.20250305.tar.gz", hash = "sha256:a987269b49488f21961a1d99aa8d281b611625883def6392a93855b31544e405"}, +] + +[package.dependencies] +setuptools = "*" + [[package]] name = "typing-extensions" version = "4.12.2" @@ -5418,4 +5507,4 @@ vector-db-based = ["cohere", "langchain", "openai", "tiktoken"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "dc50be30e74dea64afa594eb2267a0b782cb5cac11ae7f5c52b09269982b3ffd" +content-hash = "c8731f26643e07136e524d5e0d6e0f5c2229cf63d43bf5644de9f1cf8e565197" diff --git a/pyproject.toml b/pyproject.toml index 7f8201692..d236c0b9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,31 +40,33 @@ isodate = "~0.6.1" Jinja2 = "~3.1.2" jsonref = "~0.2" jsonschema = "~4.17.3" # 4.18 has some significant breaking changes: https://github.com/python-jsonschema/jsonschema/releases/tag/v4.18.0 +packaging = "*" # Transitive dependency used directly in code pandas = "2.2.2" -psutil = "6.1.0" +psutil = "6.1.0" # TODO: Remove if unused pydantic = "^2.7" pyrate-limiter = "~3.1.0" python-dateutil = "^2.9.0" python-ulid = "^3.0.0" PyYAML = "^6.0.1" -rapidfuzz = "^3.10.1" +rapidfuzz = "^3.10.1" # TODO: Remove if unused requests = "*" requests_cache = "*" +typing-extensions = "*" # Transitive dependency used directly in code wcmatch = "10.0" # Extras depedencies -avro = { version = ">=1.11.2,<1.13.0", optional = true } +avro = { version = ">=1.11.2,<1.13.0", optional = true } # TODO: Move into dev dependencies if only used in tests cohere = { version = "4.21", optional = true } fastavro = { version = "~1.8.0", optional = true } langchain = { version = "0.1.16", optional = true } langchain_core = { version = "0.1.42", optional = true } -markdown = { version = "*", optional = true } -openai = { version = "0.27.9", extras = ["embeddings"], optional = true } +markdown = { version = "*", optional = true } # TODO: Remove if unused +openai = { version = "0.27.9", extras = ["embeddings"], optional = true } # Used indirectly by langchain library pdf2image = { version = "1.16.3", optional = true } -"pdfminer.six" = { version = "20221105", optional = true } +"pdfminer.six" = { version = "20221105", optional = true } # Used indirectly by unstructured library pyarrow = { version = "^19.0.0", optional = true } -pytesseract = { version = "0.3.10", optional = true } -python-calamine = { version = "0.2.3", optional = true } -python-snappy = { version = "0.7.3", optional = true } +pytesseract = { version = "0.3.10", optional = true } # Used indirectly by unstructured library +python-calamine = { version = "0.2.3", optional = true } # TODO: Remove if unused +python-snappy = { version = "0.7.3", optional = true } # TODO: remove if unused tiktoken = { version = "0.8.0", optional = true } nltk = { version = "3.9.1", optional = true } # This will ensure that even when you run poetry install or pip install, the compatible version of numpy will always be chosen. @@ -73,7 +75,7 @@ numpy = "<2" unstructured = { version = "0.10.27", extras = ["docx", "pptx"], optional = true } "unstructured.pytesseract" = { version = ">=0.3.12", optional = true } pyjwt = "^2.8.0" -cryptography = ">=44.0.0,<45.0.0" +cryptography = ">=44.0.0,<45.0.0" # Constrained as transitive dependency due to a bug in newer versions pytz = "2024.2" orjson = "^3.10.7" serpyco-rs = "^1.10.2" @@ -102,6 +104,7 @@ types-requests = "^2.32.0.20241016" types-python-dateutil = "^2.9.0.20241003" types-pyyaml = "^6.0.12.20240917" types-cachetools = "^5.5.0.20240820" +deptry = "^0.23.0" [tool.poetry.extras] file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "python-snappy"] @@ -196,3 +199,57 @@ optional_poetry_groups = ["dev"] poetry_extras = ["file-based", "vector-db-based"] poe_tasks = ["check-ci"] mount_docker_socket = true + +[tool.deptry] +exclude = [ + "bin", + "docs", + "unit_tests", +] + +[tool.deptry.per_rule_ignores] +# This is a mapping of rules and package names to be ignored for that rule. + +# DEP001: Project should not contain missing dependencies. +# https://deptry.com/rules-violations/#missing-dependencies-dep001 +DEP001 = [ + # These are imported but not declared: + "source_declarative_manifest" # Imported only in dynamic import tests, not in main code +] + +# DEP002: Project should not contain unused dependencies. +# https://deptry.com/rules-violations/#missing-dependencies-dep002 +DEP002 = [ + "cryptography", # Constrained as transitive dependency due to a bug in newer versions + + # TODO: Remove these dependencies if not needed: + "avro", # Only imported in `unit_tests` code + "psutil", + "rapidfuzz", + "cohere", + "markdown", + "openai", + "pdf2image", + "pdfminer.six", + "pytesseract", + "python-calamine", + "python-snappy", + "tiktoken", + "unstructured.pytesseract", +] + +# DEP003: Project should not use transitive dependencies. +# https://deptry.com/rules-violations/#transitive-dependencies-dep003 +DEP003 = [ + # Transitive dependencies that are imported directly + "pydantic_core" # Pydantic internals, no need to define as separate dependency +] + +# DEP004: Project should not use development dependencies in non-development code. +# https://deptry.com/rules-violations/#misplaced-development-dependencies-dep004 +DEP004 = [ + # The `airbyte_cdk.test.utils` module is main code. + # TODO: These should probably be declared within a `tests` extra: + "pytest", + "requests_mock", +]