diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..a345d68 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,202 @@ +name: Publish to PyPI + +# Manually triggered release, mirroring awslabs/deequ's publish flow: the +# operator supplies the release version, the workflow sets it, builds, and +# publishes to PyPI, then tags the release and opens a follow-up PR that bumps +# the version in the repo. The PyPI token is never stored in GitHub — it is +# fetched at run time from AWS Secrets Manager via OIDC and passed to Poetry +# through an env var. + +on: + workflow_dispatch: + inputs: + release_version: + description: "Release version, e.g. 1.6.0 (must be a new version, not already on PyPI)" + required: true + dry_run: + description: "Dry run (build only, skip the PyPI upload, tag, and version-bump PR)" + type: boolean + default: true + +permissions: + id-token: write # assume the AWS role via OIDC + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 15 + # Real uploads run in the protected environment (required reviewer); a dry + # run resolves to '' (no environment, no approval, no credentials). + environment: ${{ !inputs.dry_run && 'pypi-release' || '' }} + outputs: + version: ${{ steps.version.outputs.version }} + sha: ${{ steps.sha.outputs.sha }} + steps: + - name: Require dispatch from master for a real release + # The pypi-release environment only permits deployments from master, and + # GitHub matches that against the dispatch ref (not the checkout). Fail + # early with a clear message instead of an opaque environment rejection. + if: ${{ !inputs.dry_run && github.ref != 'refs/heads/master' }} + run: | + echo "::error::Real releases must be dispatched from the master branch (got ${GITHUB_REF})." >&2 + exit 1 + + - name: Checkout master + # Always release from master regardless of the branch the dispatch UI + # selected, so the build/tag/bump reflect the mainline. + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: master + persist-credentials: false + + - name: Record release SHA + # Pin the exact commit built/published so the tag job tags THIS commit, + # even if master advances during the approval wait. + id: sha + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.9" + + - name: Install Poetry + run: pip install poetry==1.7.1 + + - name: Normalize and validate version + # Strip an accidental leading 'v', reject anything that isn't a plain + # release version, and expose the canonical value for every downstream + # step (build, tag, bump, URL) so tag/__version__/wheel cannot diverge. + id: version + run: | + V="${RAW_VERSION#v}" + if ! printf '%s' "$V" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+([abc]|rc|\.post|\.dev)?[0-9]*$'; then + echo "::error::Invalid release version '${RAW_VERSION}' (expected e.g. 1.6.0)" >&2 + exit 1 + fi + echo "version=$V" >> "$GITHUB_OUTPUT" + echo "Release version: $V" + env: + RAW_VERSION: ${{ inputs.release_version }} + + - name: Set release version + # Set the canonical version in both places pydeequ keeps it, then assert + # they agree with the wheel version poetry will build. + run: | + poetry version "${VERSION}" + sed -i "s/^__version__ = .*/__version__ = \"${VERSION}\"/" pydeequ/__init__.py + BUILT=$(poetry version --short) + if [ "$BUILT" != "${VERSION}" ]; then + echo "::error::pyproject version $BUILT != requested ${VERSION}" >&2 + exit 1 + fi + grep -q "__version__ = \"${VERSION}\"" pydeequ/__init__.py || { + echo "::error::__init__ __version__ was not updated to ${VERSION}" >&2; exit 1; } + env: + VERSION: ${{ steps.version.outputs.version }} + + - name: Build sdist + wheel + run: poetry build + + - name: Configure AWS credentials (OIDC) + if: ${{ !inputs.dry_run }} + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + role-to-assume: arn:aws:iam::059379938386:role/GitHubActionsPyDeequPublish + aws-region: us-east-1 + + - name: Publish to PyPI + if: ${{ !inputs.dry_run }} + # No --skip-existing: a version already on PyPI must fail loudly here so + # the tag/bump never run for an artifact this release did not upload. + # Token → POETRY_PYPI_TOKEN_PYPI env (read natively by poetry), never on + # argv, never in $GITHUB_OUTPUT, never written to a file. Masked first. + run: | + PYPI_TOKEN=$(aws secretsmanager get-secret-value --secret-id pypi_token --query SecretString --output text) + echo "::add-mask::$PYPI_TOKEN" + export POETRY_PYPI_TOKEN_PYPI="$PYPI_TOKEN" + poetry publish --no-interaction + + - name: Summary + run: | + if [ "${DRY_RUN}" = "true" ]; then + echo "### Dry run completed for ${VERSION} (not published)" >> "$GITHUB_STEP_SUMMARY" + else + echo "### Published ${VERSION} to PyPI" >> "$GITHUB_STEP_SUMMARY" + echo "View at: https://pypi.org/project/pydeequ/${VERSION}/" >> "$GITHUB_STEP_SUMMARY" + fi + env: + DRY_RUN: ${{ inputs.dry_run }} + VERSION: ${{ steps.version.outputs.version }} + + tag-and-bump: + # Runs ONLY after a successful real publish (needs: publish gates on + # success). Creates the release tag as a record of what is now on PyPI (tag + # after publish, never before), then opens a PR recording the version. + needs: publish + if: ${{ !inputs.dry_run }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: write + pull-requests: write + env: + VERSION: ${{ needs.publish.outputs.version }} + RELEASE_SHA: ${{ needs.publish.outputs.sha }} + steps: + - name: Checkout the released commit + # The exact SHA the publish job built/uploaded — not a fresh master tip, + # so a merge during the approval wait can't move the tag off the release. + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ needs.publish.outputs.sha }} + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.9" + + - name: Install Poetry + run: pip install poetry==1.7.1 + + - name: Tag the published release + # Tag the exact published commit, now that PyPI has the artifact. + # Idempotent: skip if the tag already exists remotely. + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + if git ls-remote --exit-code --tags origin "v${VERSION}" >/dev/null 2>&1; then + echo "Tag v${VERSION} already exists — skipping." + else + git tag -a "v${VERSION}" "${RELEASE_SHA}" -m "Release ${VERSION}" + git push origin "v${VERSION}" + fi + + - name: Bump version and open PR + # Idempotent: skip if the bump branch/PR already exist or master is + # already at this version (e.g. a re-run after a partial failure). + run: | + BRANCH_NAME="bump-version-${VERSION}" + if git ls-remote --exit-code --heads origin "${BRANCH_NAME}" >/dev/null 2>&1; then + echo "Branch ${BRANCH_NAME} already exists — skipping bump PR." + exit 0 + fi + git fetch origin master + git checkout -b "${BRANCH_NAME}" origin/master + poetry version "${VERSION}" + sed -i "s/^__version__ = .*/__version__ = \"${VERSION}\"/" pydeequ/__init__.py + if git diff --quiet -- pyproject.toml pydeequ/__init__.py; then + echo "master already at ${VERSION} — nothing to bump." + exit 0 + fi + git add pyproject.toml pydeequ/__init__.py + git commit -m "Update version to ${VERSION}" + git push origin "${BRANCH_NAME}" + gh pr create \ + --base master \ + --head "${BRANCH_NAME}" \ + --title "Update version to ${VERSION}" \ + --body "Automated version bump after publishing ${VERSION} to PyPI." + env: + GH_TOKEN: ${{ github.token }} diff --git a/README.md b/README.md index bde2d24..164b911 100644 --- a/README.md +++ b/README.md @@ -17,14 +17,6 @@ There are 4 main components of Deequ, and they are: ![](imgs/pydeequ_architecture.jpg) -## 🎉 Announcements 🎉 -- **NEW!!!** The 1.4.0 release of Python Deequ has been published to PYPI https://pypi.org/project/pydeequ/. This release adds support for Spark 3.5.0. -- The latest version of Deequ, 2.0.7, is made available With Python Deequ 1.3.0. -- 1.1.0 release of Python Deequ has been published to PYPI https://pypi.org/project/pydeequ/. This release brings many recent upgrades including support up to Spark 3.3.0! Any feedbacks are welcome through github issues. -- With PyDeequ v0.1.8+, we now officially support Spark3 ! Just make sure you have an environment variable `SPARK_VERSION` to specify your Spark version! -- We've release a blogpost on integrating PyDeequ onto AWS leveraging services such as AWS Glue, Athena, and SageMaker! Check it out: [Monitor data quality in your data lake using PyDeequ and AWS Glue](https://aws.amazon.com/blogs/big-data/monitor-data-quality-in-your-data-lake-using-pydeequ-and-aws-glue/). -- Check out the [PyDeequ Release Announcement Blogpost](https://aws.amazon.com/blogs/big-data/testing-data-quality-at-scale-with-pydeequ/) with a tutorial walkthrough the Amazon Reviews dataset! - ## Quickstart The following will quickstart you with some basic usage. For more in-depth examples, take a look in the [`tutorials/`](tutorials/) directory for executable Jupyter notebooks of each module. For documentation on supported interfaces, view the [`documentation`](https://pydeequ.readthedocs.io/). @@ -168,15 +160,6 @@ spark.sparkContext._gateway.shutdown_callback_server() spark.stop() ``` -## [Contributing](https://github.com/awslabs/python-deequ/blob/master/CONTRIBUTING.md) -Please refer to the [contributing doc](https://github.com/awslabs/python-deequ/blob/master/CONTRIBUTING.md) for how to contribute to PyDeequ. - -## [License](https://github.com/awslabs/python-deequ/blob/master/LICENSE) - -This library is licensed under the Apache 2.0 License. - -****** - ## Contributing Developer Setup 1. Setup [SDKMAN](#setup-sdkman) @@ -267,3 +250,15 @@ docker build . -t spark-3.3-docker-test docker run spark-3.3-docker-test ``` +## Contributing + +Please refer to the [contributing doc](https://github.com/awslabs/python-deequ/blob/master/CONTRIBUTING.md) for how to contribute to PyDeequ. + +## Security + +See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. + +## License + +This library is licensed under the Apache 2.0 License. +