Informatica provider: Add SQL auto-lineage and selective lineage control

dev/breeze/tests/test_selective_checks.py

@@ -2692,7 +2692,7 @@ def test_upgrade_to_newer_dependencies(
             {
                 "docs-list-as-string": "amazon apache.drill apache.druid apache.hive apache.iceberg "
                 "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks elasticsearch "
-                "exasol google jdbc microsoft.mssql mysql odbc openlineage "
+                "exasol google informatica jdbc microsoft.mssql mysql odbc openlineage "
                 "oracle pgvector postgres presto slack snowflake sqlite teradata trino vertica ydb",
             },
             id="Common SQL provider package python files changed",

docs/spelling_wordlist.txt

@@ -1386,6 +1386,7 @@ renderer
 renderers
 renewer
 reparse
+reparsed
 replicaSet
 repo
 repos

generated/provider_dependencies.json

@@ -1136,9 +1136,11 @@
     "deps": [
       "apache-airflow-providers-common-compat>=1.12.0",
       "apache-airflow-providers-http>=4.13.2",
-      "apache-airflow>=3.0.0"
+      "apache-airflow>=3.0.0",
+      "sqlglot>=30.0.0"
     ],
     "devel-deps": [
+      "sqlglot>=30.0.0",
       "uuid6>=2024.7.10"
     ],
     "plugins": [
@@ -1149,6 +1151,7 @@
     ],
     "cross-providers-deps": [
       "common.compat",
+      "common.sql",
       "http"
     ],
     "excluded-python-versions": [],

providers/common/ai/src/airflow/providers/common/ai/utils/dataquality/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

providers/informatica/README.rst

@@ -56,6 +56,7 @@ PIP package                                 Version required
 ``apache-airflow``                          ``>=3.0.0``
 ``apache-airflow-providers-common-compat``  ``>=1.12.0``
 ``apache-airflow-providers-http``           ``>=4.13.2``
+``sqlglot``                                 ``>=30.0.0``
 ==========================================  ==================
 
 Cross provider package dependencies

providers/informatica/dev/docker-compose.yml

@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+---
+
+# Usage:
+#   docker-compose up -d
+#
+# Services:
+#   postgres          : Source/target database for lineage test DAGs (port 55433)
+#   informatica_sim   : Informatica EDC simulator (port 8082)
+#
+# Airflow connection for the simulator (add via UI or env var):
+#   Conn ID   : informatica_edc_default
+#   Conn Type : HTTP
+#   Host      : informatica_sim (inside Breeze) or localhost (outside)
+#   Port      : 8082
+#   Schema    : http
+#
+# Airflow connection for Postgres (add via UI or env var):
+#   Conn ID   : postgres_lineage
+#   Conn Type : Postgres
+#   Host      : informatica-lineage-postgres (inside Breeze) or localhost (outside)
+#   Port      : 5433  (host-side mapping)
+#   Schema    : lineage_demo
+#   Login     : airflow
+#   Password  : airflow
+
+services:
+  postgres:
+    image: postgres:16
+    container_name: informatica-lineage-postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: lineage_demo
+      POSTGRES_USER: airflow
+      POSTGRES_PASSWORD: airflow
+      TZ: UTC
+    ports:
+      - "55433:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U airflow -d lineage_demo"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    volumes:
+      - ./init:/docker-entrypoint-initdb.d:ro
+      - informatica-lineage-postgres-data:/var/lib/postgresql/data
+    networks:
+      - default
+      - breeze
+
+  informatica_sim:
+    build:
+      context: ./informatica_simulator
+      dockerfile: Dockerfile
+    container_name: informatica-edc-sim
+    restart: unless-stopped
+    ports:
+      - "8082:8082"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -sf http://localhost:8082/access || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 6
+    networks:
+      - default
+      - breeze
+
+volumes:
+  informatica-lineage-postgres-data:
+
+networks:
+  breeze:
+    external: true
+    name: ${BREEZE_DOCKER_NETWORK:-breeze_default}

providers/informatica/dev/informatica_simulator/Dockerfile

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Get the image
+FROM python:3.10.10
+
+# Set the working directory
+WORKDIR /code
+
+# Copy the requirements package
+COPY ./requirements.txt /code/requirements.txt
+
+# Download the python package requirements in build process
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Copy the application
+COPY ./app /code/app
+
+# Expose port 8082
+EXPOSE 8082
+
+# Start the server when the container is launched
+CMD ["uvicorn", "app.main:app","--proxy-headers", "--host", "0.0.0.0", "--port", "8082"]