From a774d51ca383ef1058233827fc33bb638d75fec8 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sun, 19 Dec 2021 21:07:46 +0000 Subject: [PATCH 01/22] 20300 - initial files added --- .../airflow_providers_bug_report.yml | 1 + CONTRIBUTING.rst | 3 +- INSTALL | 2 +- airflow/providers/github/CHANGELOG.rst | 25 ++++++++ airflow/providers/github/__init__.py | 16 ++++++ .../providers/github/example_dags/__init__.py | 16 ++++++ .../github/example_dags/example_github.py | 57 +++++++++++++++++++ .../example_dags/example_github_query.py | 39 +++++++++++++ airflow/providers/github/hooks/__init__.py | 16 ++++++ .../providers/github/operators/__init__.py | 16 ++++++ airflow/providers/github/provider.yaml | 43 ++++++++++++++ 11 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 airflow/providers/github/CHANGELOG.rst create mode 100644 airflow/providers/github/__init__.py create mode 100644 airflow/providers/github/example_dags/__init__.py create mode 100644 airflow/providers/github/example_dags/example_github.py create mode 100644 airflow/providers/github/example_dags/example_github_query.py create mode 100644 airflow/providers/github/hooks/__init__.py create mode 100644 airflow/providers/github/operators/__init__.py create mode 100644 airflow/providers/github/provider.yaml diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index eb6a13046e914..91aa6545b805d 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -51,6 +51,7 @@ body: - exasol - facebook - ftp + - github - google - grpc - hashicorp diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 94dc3401b37d2..f0a35194d852a 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -609,7 +609,7 @@ apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, a apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol, -facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, +facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds, @@ -681,6 +681,7 @@ apache.hive amazon,microsoft.mssql,mysql,presto,samba,vertica apache.livy http dingding http discord http +github PyGithub google amazon,apache.beam,apache.cassandra,cncf.kubernetes,facebook,microsoft.azure,microsoft.mssql,mysql,oracle,postgres,presto,salesforce,sftp,ssh,trino hashicorp google microsoft.azure google,oracle,sftp diff --git a/INSTALL b/INSTALL index ce2d973d89fda..21ac12496b048 100644 --- a/INSTALL +++ b/INSTALL @@ -100,7 +100,7 @@ apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, a apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol, -facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, +facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds, diff --git a/airflow/providers/github/CHANGELOG.rst b/airflow/providers/github/CHANGELOG.rst new file mode 100644 index 0000000000000..1bafa2d67956f --- /dev/null +++ b/airflow/providers/github/CHANGELOG.rst @@ -0,0 +1,25 @@ + + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Changelog +--------- +1.0.0 +..... + +Initial version of the provider. diff --git a/airflow/providers/github/__init__.py b/airflow/providers/github/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/github/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/github/example_dags/__init__.py b/airflow/providers/github/example_dags/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/github/example_dags/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/github/example_dags/example_github.py b/airflow/providers/github/example_dags/example_github.py new file mode 100644 index 0000000000000..ee503c5fc7864 --- /dev/null +++ b/airflow/providers/github/example_dags/example_github.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datetime import datetime + +from airflow.decorators import task +from airflow.models.dag import DAG +from airflow.providers.github.hooks.github import GithubHook + + +@task(task_id="github_task") +def test_github_hook(): + bucket_name = 'test-influx' + github_hook = GithubHook() + client = github_hook.get_conn() + print(client) + print(f"Organization name {github_hook.org_name}") + + # Make sure enough permissions to create bucket. + github_hook.create_bucket(bucket_name, "Bucket to test github connection", github_hook.org_name) + github_hook.write(bucket_name, "test_point", "location", "Prague", "temperature", 25.3, True) + + tables = github_hook.query('from(bucket:"test-influx") |> range(start: -10m)') + + for table in tables: + print(table) + for record in table.records: + print(record.values) + + bucket_id = github_hook.find_bucket_id_by_name(bucket_name) + print(bucket_id) + # Delete bucket takes bucket id. + github_hook.delete_bucket(bucket_name) + + +with DAG( + dag_id='github_example_dag', + schedule_interval=None, + start_date=datetime(2021, 1, 1), + max_active_runs=1, + tags=['example'], +) as dag: + test_github_hook() diff --git a/airflow/providers/github/example_dags/example_github_query.py b/airflow/providers/github/example_dags/example_github_query.py new file mode 100644 index 0000000000000..e385309d4593a --- /dev/null +++ b/airflow/providers/github/example_dags/example_github_query.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datetime import datetime + +from airflow.models.dag import DAG +from airflow.providers.github.operators.github import GithubOperator + +dag = DAG( + 'example_github_operator', + start_date=datetime(2021, 1, 1), + tags=['example'], + catchup=False, +) + +# [START howto_operator_github] + +query_github_task = GithubOperator( + github_conn_id='github_conn_id', + task_id='query_github', + sql='from(bucket:"test-influx") |> range(start: -10m, stop: {{ds}})', + dag=dag, +) + +# [END howto_operator_github] diff --git a/airflow/providers/github/hooks/__init__.py b/airflow/providers/github/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/github/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/github/operators/__init__.py b/airflow/providers/github/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/github/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/github/provider.yaml b/airflow/providers/github/provider.yaml new file mode 100644 index 0000000000000..bb7d21cf4be09 --- /dev/null +++ b/airflow/providers/github/provider.yaml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-github +name: Github +description: | + `Github `__ +versions: + - 1.1.0 + - 1.0.0 +integrations: + - integration-name: Github + external-doc-url: https://www.github.com/ + tags: [software] + +hooks: + - integration-name: Github + python-modules: + - airflow.providers.github.hooks.github + +operators: + - integration-name: Github + python-modules: + - airflow.providers.github.operators.github + +connection-types: + - hook-class-name: airflow.providers.github.hooks.github.GithubHook + connection-type: github From a742f340d526cc19dc76344a92e81deef4ea61ef Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Fri, 24 Dec 2021 00:01:46 +0000 Subject: [PATCH 02/22] 20300 - test files added --- .../github/.latest-doc-only-change.txt | 1 + airflow/providers/github/hooks/github.py | 163 ++++++++++++++++++ .../providers/github/operators/influxdb.py | 55 ++++++ airflow/providers/github/provider.yaml | 1 - .../commits.rst | 39 +++++ .../connections/github.rst | 47 +++++ .../apache-airflow-providers-github/index.rst | 103 +++++++++++ .../installing-providers-from-sources.rst | 18 ++ .../operators/index.rst | 33 ++++ docs/apache-airflow/extra-packages-ref.rst | 2 + setup.py | 4 + tests/providers/github/__init__.py | 16 ++ tests/providers/github/hooks/__init__.py | 16 ++ tests/providers/github/hooks/test_influxdb.py | 88 ++++++++++ tests/providers/github/operators/__init__.py | 16 ++ .../github/operators/test_influxdb.py | 32 ++++ 16 files changed, 633 insertions(+), 1 deletion(-) create mode 100644 airflow/providers/github/.latest-doc-only-change.txt create mode 100644 airflow/providers/github/hooks/github.py create mode 100644 airflow/providers/github/operators/influxdb.py create mode 100644 docs/apache-airflow-providers-github/commits.rst create mode 100644 docs/apache-airflow-providers-github/connections/github.rst create mode 100644 docs/apache-airflow-providers-github/index.rst create mode 100644 docs/apache-airflow-providers-github/installing-providers-from-sources.rst create mode 100644 docs/apache-airflow-providers-github/operators/index.rst create mode 100644 tests/providers/github/__init__.py create mode 100644 tests/providers/github/hooks/__init__.py create mode 100644 tests/providers/github/hooks/test_influxdb.py create mode 100644 tests/providers/github/operators/__init__.py create mode 100644 tests/providers/github/operators/test_influxdb.py diff --git a/airflow/providers/github/.latest-doc-only-change.txt b/airflow/providers/github/.latest-doc-only-change.txt new file mode 100644 index 0000000000000..5312a67d64695 --- /dev/null +++ b/airflow/providers/github/.latest-doc-only-change.txt @@ -0,0 +1 @@ +87dc63b65daaf77c4c9f2f6611b72bcc78603d1e diff --git a/airflow/providers/github/hooks/github.py b/airflow/providers/github/hooks/github.py new file mode 100644 index 0000000000000..a33a2b1b8c1f3 --- /dev/null +++ b/airflow/providers/github/hooks/github.py @@ -0,0 +1,163 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""This module allows to connect to a InfluxDB database.""" + +from typing import Dict, List + +import pandas as pd +from influxdb_client import InfluxDBClient +from influxdb_client.client.flux_table import FluxTable +from influxdb_client.client.write.point import Point +from influxdb_client.client.write_api import SYNCHRONOUS + +from airflow.hooks.base import BaseHook +from airflow.models import Connection + + +class InfluxDBHook(BaseHook): + """ + Interact with InfluxDB. + + Performs a connection to InfluxDB and retrieves client. + + :param influxdb_conn_id: Reference to :ref:`Influxdb connection id `. + :type influxdb_conn_id: str + """ + + conn_name_attr = 'influxdb_conn_id' + default_conn_name = 'influxdb_default' + conn_type = 'influxdb' + hook_name = 'Influxdb' + + def __init__(self, conn_id: str = default_conn_name, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.influxdb_conn_id = conn_id + self.connection = kwargs.pop("connection", None) + self.client = None + self.extras: Dict = {} + self.uri = None + self.org_name = None + + def get_client(self, uri, token, org_name): + return InfluxDBClient(url=uri, token=token, org=org_name) + + def get_uri(self, conn: Connection): + """ + Function to add additional parameters to the URI + based on SSL or other InfluxDB host requirements + + """ + return '{scheme}://{host}:{port}'.format( + scheme='https' if conn.schema is None else f'{conn.schema}', + host=conn.host, + port='7687' if conn.port is None else f'{conn.port}', + ) + + def get_conn(self) -> InfluxDBClient: + """ + Function that initiates a new InfluxDB connection + with token and organization name + """ + self.connection = self.get_connection(self.influxdb_conn_id) + self.extras = self.connection.extra_dejson.copy() + + self.uri = self.get_uri(self.connection) + self.log.info('URI: %s', self.uri) + + if self.client is not None: + return self.client + + token = self.connection.extra_dejson.get('token') + self.org_name = self.connection.extra_dejson.get('org_name') + + self.log.info('URI: %s', self.uri) + self.log.info('Organization: %s', self.org_name) + + self.client = self.get_client(self.uri, token, self.org_name) + + return self.client + + def query(self, query) -> List[FluxTable]: + """ + Function to to run the query. + Note: The bucket name + should be included in the query + + :param query: InfluxDB query + :return: List + """ + client = self.get_conn() + + query_api = client.query_api() + return query_api.query(query) + + def query_to_df(self, query) -> pd.DataFrame: + """ + Function to run the query and + return a pandas dataframe + Note: The bucket name + should be included in the query + + :param query: InfluxDB query + :return: pd.DataFrame + """ + client = self.get_conn() + + query_api = client.query_api() + return query_api.query_data_frame(query) + + def write(self, bucket_name, point_name, tag_name, tag_value, field_name, field_value, synchronous=False): + """ + Writes a Point to the bucket specified. + Example: Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) + """ + # By defaults its Batching + if synchronous: + write_api = self.client.write_api(write_options=SYNCHRONOUS) + else: + write_api = self.client.write_api() + + p = Point(point_name).tag(tag_name, tag_value).field(field_name, field_value) + + write_api.write(bucket=bucket_name, record=p) + + def create_organization(self, name): + """Function to create a new organization""" + return self.client.organizations_api().create_organization(name=name) + + def delete_organization(self, org_id): + """Function to delete organization by organization id""" + return self.client.organizations_api().delete_organization(org_id=org_id) + + def create_bucket(self, bucket_name, description, org_id, retention_rules=None): + """Function to create a bucket for an organization""" + return self.client.buckets_api().create_bucket( + bucket_name=bucket_name, description=description, org_id=org_id, retention_rules=None + ) + + def find_bucket_id_by_name(self, bucket_name): + """Function to get bucket id by name.""" + bucket = self.client.buckets_api().find_bucket_by_name(bucket_name) + + return "" if bucket is None else bucket.id + + def delete_bucket(self, bucket_name): + """Function to delete bucket by bucket name.""" + bucket = self.find_bucket_id_by_name(bucket_name) + return self.client.buckets_api().delete_bucket(bucket) diff --git a/airflow/providers/github/operators/influxdb.py b/airflow/providers/github/operators/influxdb.py new file mode 100644 index 0000000000000..4b169d1ea7586 --- /dev/null +++ b/airflow/providers/github/operators/influxdb.py @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Dict + +from airflow.models import BaseOperator +from airflow.providers.influxdb.hooks.influxdb import InfluxDBHook + + +class InfluxDBOperator(BaseOperator): + """ + Executes sql code in a specific InfluxDB database + + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:InfluxDBOperator` + + :param sql: the sql code to be executed. Can receive a str representing a + sql statement + :type sql: str + :param influxdb_conn_id: Reference to :ref:`Influxdb connection id `. + :type influxdb_conn_id: str + """ + + template_fields = ['sql'] + + def __init__( + self, + *, + sql: str, + influxdb_conn_id: str = 'influxdb_default', + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.influxdb_conn_id = influxdb_conn_id + self.sql = sql + + def execute(self, context: Dict) -> None: + self.log.info('Executing: %s', self.sql) + self.hook = InfluxDBHook(conn_id=self.influxdb_conn_id) + self.hook.query(self.sql) diff --git a/airflow/providers/github/provider.yaml b/airflow/providers/github/provider.yaml index bb7d21cf4be09..3ddf4c21ce5c3 100644 --- a/airflow/providers/github/provider.yaml +++ b/airflow/providers/github/provider.yaml @@ -21,7 +21,6 @@ name: Github description: | `Github `__ versions: - - 1.1.0 - 1.0.0 integrations: - integration-name: Github diff --git a/docs/apache-airflow-providers-github/commits.rst b/docs/apache-airflow-providers-github/commits.rst new file mode 100644 index 0000000000000..eabdbf9713a51 --- /dev/null +++ b/docs/apache-airflow-providers-github/commits.rst @@ -0,0 +1,39 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Package apache-airflow-providers-github +------------------------------------------------------ + +`Github `__ + + +This is detailed commit list of changes for versions provider package: ``github``. +For high-level changelog, see :doc:`package information including changelog `. + + +1.0.0 +..... + +Latest change: 2021-09-29 + +================================================================================================= =========== ====================================================================== +Commit Committed Subject +================================================================================================= =========== ====================================================================== +`e84527509e `_ 2021-09-29 ``Updating the Github example DAG to use the TaskFlow API (#18596)`` +================================================================================================= =========== ====================================================================== diff --git a/docs/apache-airflow-providers-github/connections/github.rst b/docs/apache-airflow-providers-github/connections/github.rst new file mode 100644 index 0000000000000..95d39ae2f86fc --- /dev/null +++ b/docs/apache-airflow-providers-github/connections/github.rst @@ -0,0 +1,47 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/connection:github: + +Github Connection +==================== +The Github connection type provides connection to a Github database. + +Configuring the Connection +-------------------------- +Host (required) + The host to connect to. + +Extra (required) + Specify the extra parameters (as json dictionary) that can be used in Github + connection. + + The following extras are required: + + - token - Create token - https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token/ + + * ``token``: Create token using the influxdb cli or UI + + Example "extras" field: + + .. code-block:: JSON + + { + "token": "343434343423234234234343434", + "org_name": "Test" + } diff --git a/docs/apache-airflow-providers-github/index.rst b/docs/apache-airflow-providers-github/index.rst new file mode 100644 index 0000000000000..093ae911da4f0 --- /dev/null +++ b/docs/apache-airflow-providers-github/index.rst @@ -0,0 +1,103 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-influxdb`` +======================================= + +Content +------- + +.. toctree:: + :maxdepth: 1 + :caption: Guides + + Connection types + Operators + +.. toctree:: + :maxdepth: 1 + :caption: References + + Python API <_api/airflow/providers/influxdb/index> + +.. toctree:: + :maxdepth: 1 + :caption: Resources + + Example DAGs + +.. toctree:: + :maxdepth: 1 + :caption: Resources + + PyPI Repository + +.. toctree:: + :maxdepth: 1 + :caption: Resources + + PyPI Repository + Installing from sources + +.. toctree:: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +Package apache-airflow-providers-influxdb +------------------------------------------------------ + +`InfluxDB `__ + + +Release: 1.1.0 + +Provider package +---------------- + +This is a provider package for ``influxdb`` provider. All classes for this provider package +are in ``airflow.providers.influxdb`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow 2.1+ installation via +``pip install apache-airflow-providers-influxdb`` + +PIP requirements +---------------- + +=================== ================== +PIP package Version required +=================== ================== +``influxdb-client`` ``>=1.19.0`` +``pandas`` ``>=0.17.1, <2.0`` +=================== ================== + +.. include:: ../../airflow/providers/influxdb/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-github/installing-providers-from-sources.rst b/docs/apache-airflow-providers-github/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..1c90205d15b3a --- /dev/null +++ b/docs/apache-airflow-providers-github/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-github/operators/index.rst b/docs/apache-airflow-providers-github/operators/index.rst new file mode 100644 index 0000000000000..0787eb91cf821 --- /dev/null +++ b/docs/apache-airflow-providers-github/operators/index.rst @@ -0,0 +1,33 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + + +.. _howto/operator:GithubOperator: + +GithubOperator +================= + +Use the :class:`~airflow.providers.github.operators.GithubOperator` to execute +SQL commands in a `Github `__ database. + +An example of running the query using the operator: + +.. exampleinclude:: /../../airflow/providers/github/example_dags/example_github_query.py + :language: python + :start-after: [START howto_operator_github] + :end-before: [END howto_operator_github] diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index b761e2181cafe..20beb2bf37b16 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -222,6 +222,8 @@ Those are extras that add dependencies needed for integration with other softwar +---------------------+-----------------------------------------------------+-------------------------------------------+ | exasol | ``pip install 'apache-airflow[exasol]'`` | Exasol hooks and operators | +---------------------+-----------------------------------------------------+-------------------------------------------+ +| github | ``pip install 'apache-airflow[github]'`` | Github operators and hook | ++---------------------+-----------------------------------------------------+-------------------------------------------+ | influxdb | ``pip install 'apache-airflow[influxdb]'`` | Influxdb operators and hook | +---------------------+-----------------------------------------------------+-------------------------------------------+ | jenkins | ``pip install 'apache-airflow[jenkins]'`` | Jenkins hooks and operators | diff --git a/setup.py b/setup.py index f8caa19658b03..06f20f29e3455 100644 --- a/setup.py +++ b/setup.py @@ -288,6 +288,9 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version flask_appbuilder_authlib = [ 'authlib', ] +github = [ + 'PyGithub>=1.55', +] google = [ 'PyOpenSSL', # The Google Ads 14.0.1 breaks PIP and eager upgrade as it requires @@ -633,6 +636,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'exasol': exasol, 'facebook': facebook, 'ftp': [], + 'github': github, 'google': google, 'grpc': grpc, 'hashicorp': hashicorp, diff --git a/tests/providers/github/__init__.py b/tests/providers/github/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/github/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/github/hooks/__init__.py b/tests/providers/github/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/github/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/github/hooks/test_influxdb.py b/tests/providers/github/hooks/test_influxdb.py new file mode 100644 index 0000000000000..8b86f03cb0933 --- /dev/null +++ b/tests/providers/github/hooks/test_influxdb.py @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest +from unittest import mock + +from airflow.models import Connection +from airflow.providers.influxdb.hooks.influxdb import InfluxDBHook + + +class TestInfluxDbHook(unittest.TestCase): + def setUp(self): + super().setUp() + self.influxdb_hook = InfluxDBHook() + extra = {} + extra['token'] = '123456789' + extra['org_name'] = 'test' + + self.connection = Connection(schema='http', host='localhost', extra=extra) + + def test_get_conn(self): + self.influxdb_hook.get_connection = mock.Mock() + self.influxdb_hook.get_connection.return_value = self.connection + + self.influxdb_hook.get_client = mock.Mock() + self.influxdb_hook.get_conn() + + assert self.influxdb_hook.org_name == 'test' + assert self.influxdb_hook.uri == 'http://localhost:7687' + + assert self.influxdb_hook.get_connection.return_value.schema == 'http' + assert self.influxdb_hook.get_connection.return_value.host == 'localhost' + + assert self.influxdb_hook.get_client is not None + + def test_query(self): + self.influxdb_hook.get_conn = mock.Mock() + + influxdb_query = 'SELECT "duration" FROM "pyexample"' + self.influxdb_hook.query(influxdb_query) + + self.influxdb_hook.get_conn.assert_called() + + def test_query_to_df(self): + self.influxdb_hook.get_conn = mock.Mock() + + self.influxdb_hook.write = mock.Mock() + influxdb_query = 'SELECT "duration" FROM "pyexample"' + self.influxdb_hook.query_to_df(influxdb_query) + + self.influxdb_hook.get_conn.assert_called() + + def test_write(self): + self.influxdb_hook.get_connection = mock.Mock() + self.influxdb_hook.get_connection.return_value = self.connection + + self.influxdb_hook.get_conn = mock.Mock() + self.influxdb_hook.client = mock.Mock() + self.influxdb_hook.client.write_api = mock.Mock() + + self.influxdb_hook.write("test_bucket", "test_point", "location", "Prague", "temperature", 25.3, True) + + self.influxdb_hook.client.write_api.assert_called() + + def test_find_bucket_by_id(self): + self.influxdb_hook.get_connection = mock.Mock() + self.influxdb_hook.get_connection.return_value = self.connection + + self.influxdb_hook.client = mock.Mock() + self.influxdb_hook.client.buckets_api = mock.Mock() + + self.influxdb_hook.find_bucket_id_by_name("test") + + self.influxdb_hook.client.buckets_api.assert_called() diff --git a/tests/providers/github/operators/__init__.py b/tests/providers/github/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/github/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/github/operators/test_influxdb.py b/tests/providers/github/operators/test_influxdb.py new file mode 100644 index 0000000000000..f7b82424cfa7b --- /dev/null +++ b/tests/providers/github/operators/test_influxdb.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest +from unittest import mock + +from airflow.providers.influxdb.operators.influxdb import InfluxDBOperator + + +class TestInfluxDBOperator(unittest.TestCase): + @mock.patch('airflow.providers.influxdb.operators.influxdb.InfluxDBHook') + def test_influxdb_operator_test(self, mock_hook): + + sql = """from(bucket:"test") |> range(start: -10m)""" + op = InfluxDBOperator(task_id='basic_influxdb', sql=sql, influxdb_conn_id='influxdb_default') + op.execute(mock.MagicMock()) + mock_hook.assert_called_once_with(conn_id='influxdb_default') + mock_hook.return_value.query.assert_called_once_with(sql) From 1224ebf553527f5d0c827e82f5478bf7240c0357 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Fri, 24 Dec 2021 00:44:40 +0000 Subject: [PATCH 03/22] 20300 - ui behaviour added --- airflow/providers/github/hooks/github.py | 136 +++++------------- .../operators/{influxdb.py => github.py} | 18 +-- 2 files changed, 42 insertions(+), 112 deletions(-) rename airflow/providers/github/operators/{influxdb.py => github.py} (73%) diff --git a/airflow/providers/github/hooks/github.py b/airflow/providers/github/hooks/github.py index a33a2b1b8c1f3..c48dbe50f97d3 100644 --- a/airflow/providers/github/hooks/github.py +++ b/airflow/providers/github/hooks/github.py @@ -16,65 +16,49 @@ # specific language governing permissions and limitations # under the License. -"""This module allows to connect to a InfluxDB database.""" +"""This module allows to connect to a Github.""" -from typing import Dict, List - -import pandas as pd -from influxdb_client import InfluxDBClient -from influxdb_client.client.flux_table import FluxTable -from influxdb_client.client.write.point import Point -from influxdb_client.client.write_api import SYNCHRONOUS +from typing import Dict +from github import Github from airflow.hooks.base import BaseHook from airflow.models import Connection -class InfluxDBHook(BaseHook): +class GithubHook(BaseHook): """ - Interact with InfluxDB. + Interact with Github. - Performs a connection to InfluxDB and retrieves client. + Performs a connection to Github and retrieves client. - :param influxdb_conn_id: Reference to :ref:`Influxdb connection id `. - :type influxdb_conn_id: str + :param github_conn_id: Reference to :ref:`Github connection id `. + :type github_conn_id: str """ - conn_name_attr = 'influxdb_conn_id' - default_conn_name = 'influxdb_default' - conn_type = 'influxdb' - hook_name = 'Influxdb' + conn_name_attr = 'github_conn_id' + default_conn_name = 'github_default' + conn_type = 'github' + hook_name = 'Github' def __init__(self, conn_id: str = default_conn_name, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self.influxdb_conn_id = conn_id + self.github_conn_id = conn_id self.connection = kwargs.pop("connection", None) self.client = None self.extras: Dict = {} self.uri = None self.org_name = None - def get_client(self, uri, token, org_name): - return InfluxDBClient(url=uri, token=token, org=org_name) - - def get_uri(self, conn: Connection): - """ - Function to add additional parameters to the URI - based on SSL or other InfluxDB host requirements + def get_client(self, uri, token, base_url=DEFAULT_BASE_URL): + return Github(login_or_token=token, base_url) - """ - return '{scheme}://{host}:{port}'.format( - scheme='https' if conn.schema is None else f'{conn.schema}', - host=conn.host, - port='7687' if conn.port is None else f'{conn.port}', - ) - def get_conn(self) -> InfluxDBClient: + def get_conn(self) -> Github: """ - Function that initiates a new InfluxDB connection - with token and organization name + Function that initiates a new Github connection + with token and hostname name """ - self.connection = self.get_connection(self.influxdb_conn_id) + self.connection = self.get_connection(self.github_conn_id) self.extras = self.connection.extra_dejson.copy() self.uri = self.get_uri(self.connection) @@ -93,71 +77,17 @@ def get_conn(self) -> InfluxDBClient: return self.client - def query(self, query) -> List[FluxTable]: - """ - Function to to run the query. - Note: The bucket name - should be included in the query - - :param query: InfluxDB query - :return: List - """ - client = self.get_conn() - - query_api = client.query_api() - return query_api.query(query) - - def query_to_df(self, query) -> pd.DataFrame: - """ - Function to run the query and - return a pandas dataframe - Note: The bucket name - should be included in the query - - :param query: InfluxDB query - :return: pd.DataFrame - """ - client = self.get_conn() - - query_api = client.query_api() - return query_api.query_data_frame(query) - - def write(self, bucket_name, point_name, tag_name, tag_value, field_name, field_value, synchronous=False): - """ - Writes a Point to the bucket specified. - Example: Point("my_measurement").tag("location", "Prague").field("temperature", 25.3) - """ - # By defaults its Batching - if synchronous: - write_api = self.client.write_api(write_options=SYNCHRONOUS) - else: - write_api = self.client.write_api() - - p = Point(point_name).tag(tag_name, tag_value).field(field_name, field_value) - - write_api.write(bucket=bucket_name, record=p) - - def create_organization(self, name): - """Function to create a new organization""" - return self.client.organizations_api().create_organization(name=name) - - def delete_organization(self, org_id): - """Function to delete organization by organization id""" - return self.client.organizations_api().delete_organization(org_id=org_id) - - def create_bucket(self, bucket_name, description, org_id, retention_rules=None): - """Function to create a bucket for an organization""" - return self.client.buckets_api().create_bucket( - bucket_name=bucket_name, description=description, org_id=org_id, retention_rules=None - ) - - def find_bucket_id_by_name(self, bucket_name): - """Function to get bucket id by name.""" - bucket = self.client.buckets_api().find_bucket_by_name(bucket_name) - - return "" if bucket is None else bucket.id - - def delete_bucket(self, bucket_name): - """Function to delete bucket by bucket name.""" - bucket = self.find_bucket_id_by_name(bucket_name) - return self.client.buckets_api().delete_bucket(bucket) + @staticmethod + def get_ui_field_behaviour() -> Dict: + """Returns custom field behaviour""" + return { + "hidden_fields": ['schema', 'port', 'host', 'login'], + "relabeling": { + 'host': 'Github Url(Optional)', + 'password': 'Github Access Token', + }, + "placeholders": { + 'host': 'https://{hostname}/api/v3, Use for Github Enterprise ', + 'password': 'token credentials auth', + }, + } diff --git a/airflow/providers/github/operators/influxdb.py b/airflow/providers/github/operators/github.py similarity index 73% rename from airflow/providers/github/operators/influxdb.py rename to airflow/providers/github/operators/github.py index 4b169d1ea7586..cb650c6ed9610 100644 --- a/airflow/providers/github/operators/influxdb.py +++ b/airflow/providers/github/operators/github.py @@ -18,22 +18,22 @@ from typing import Dict from airflow.models import BaseOperator -from airflow.providers.influxdb.hooks.influxdb import InfluxDBHook +from airflow.providers.github.hooks.github import GithubHook -class InfluxDBOperator(BaseOperator): +class GithubOperator(BaseOperator): """ - Executes sql code in a specific InfluxDB database + Executes sql code in a specific Github database .. seealso:: For more information on how to use this operator, take a look at the guide: - :ref:`howto/operator:InfluxDBOperator` + :ref:`howto/operator:GithubOperator` :param sql: the sql code to be executed. Can receive a str representing a sql statement :type sql: str - :param influxdb_conn_id: Reference to :ref:`Influxdb connection id `. - :type influxdb_conn_id: str + :param github_conn_id: Reference to :ref:`Github connection id `. + :type github_conn_id: str """ template_fields = ['sql'] @@ -42,14 +42,14 @@ def __init__( self, *, sql: str, - influxdb_conn_id: str = 'influxdb_default', + github_conn_id: str = 'github_default', **kwargs, ) -> None: super().__init__(**kwargs) - self.influxdb_conn_id = influxdb_conn_id + self.github_conn_id = github_conn_id self.sql = sql def execute(self, context: Dict) -> None: self.log.info('Executing: %s', self.sql) - self.hook = InfluxDBHook(conn_id=self.influxdb_conn_id) + self.hook = GithubHook(conn_id=self.github_conn_id) self.hook.query(self.sql) From 73b7f916053873c537d17758279192dc72c16611 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Mon, 3 Jan 2022 16:25:09 +0000 Subject: [PATCH 04/22] 20300 - fixed depedency version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 06f20f29e3455..ae8b7e72b5ac2 100644 --- a/setup.py +++ b/setup.py @@ -289,7 +289,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version 'authlib', ] github = [ - 'PyGithub>=1.55', + 'pygithub', ] google = [ 'PyOpenSSL', From c4a305beccc6e6adeaba22b6cd18bdc546672e92 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Mon, 3 Jan 2022 17:38:37 +0000 Subject: [PATCH 05/22] 20300 - fixed the provider in UI --- .../github/.latest-doc-only-change.txt | 1 + airflow/providers/github/hooks/github.py | 42 +++++++------------ airflow/providers/github/operators/github.py | 13 ++---- 3 files changed, 21 insertions(+), 35 deletions(-) diff --git a/airflow/providers/github/.latest-doc-only-change.txt b/airflow/providers/github/.latest-doc-only-change.txt index 5312a67d64695..884bd61c6bef8 100644 --- a/airflow/providers/github/.latest-doc-only-change.txt +++ b/airflow/providers/github/.latest-doc-only-change.txt @@ -1 +1,2 @@ 87dc63b65daaf77c4c9f2f6611b72bcc78603d1e +# TODO: fix this diff --git a/airflow/providers/github/hooks/github.py b/airflow/providers/github/hooks/github.py index c48dbe50f97d3..b68f504ecf2be 100644 --- a/airflow/providers/github/hooks/github.py +++ b/airflow/providers/github/hooks/github.py @@ -20,9 +20,9 @@ from typing import Dict -from github import Github +from github import Github as GithubClient, PaginatedList + from airflow.hooks.base import BaseHook -from airflow.models import Connection class GithubHook(BaseHook): @@ -42,52 +42,42 @@ class GithubHook(BaseHook): def __init__(self, conn_id: str = default_conn_name, *args, **kwargs) -> None: super().__init__(*args, **kwargs) + self.connection = None self.github_conn_id = conn_id - self.connection = kwargs.pop("connection", None) self.client = None - self.extras: Dict = {} - self.uri = None - self.org_name = None - - def get_client(self, uri, token, base_url=DEFAULT_BASE_URL): - return Github(login_or_token=token, base_url) - - def get_conn(self) -> Github: + def get_conn(self) -> GithubClient: """ Function that initiates a new Github connection with token and hostname name """ - self.connection = self.get_connection(self.github_conn_id) - self.extras = self.connection.extra_dejson.copy() - - self.uri = self.get_uri(self.connection) - self.log.info('URI: %s', self.uri) - if self.client is not None: return self.client - token = self.connection.extra_dejson.get('token') - self.org_name = self.connection.extra_dejson.get('org_name') - - self.log.info('URI: %s', self.uri) - self.log.info('Organization: %s', self.org_name) + self.connection = self.get_connection(self.github_conn_id) + access_token = self.connection.password - self.client = self.get_client(self.uri, token, self.org_name) + # self.log.info('Organization: %s', self.org_name) + self.client = GithubClient(login_or_token=access_token) return self.client + def find_repos(self) -> PaginatedList: + """Function to get bucket id by name.""" + repos = self.client.get_repos() + return repos + @staticmethod def get_ui_field_behaviour() -> Dict: """Returns custom field behaviour""" return { - "hidden_fields": ['schema', 'port', 'host', 'login'], + "hidden_fields": ['schema', 'port', 'host', 'login', 'extra'], "relabeling": { - 'host': 'Github Url(Optional)', + # 'host': 'Github Enterprise Url', 'password': 'Github Access Token', }, "placeholders": { - 'host': 'https://{hostname}/api/v3, Use for Github Enterprise ', + # 'host': 'https://{hostname}/api/v3', 'password': 'token credentials auth', }, } diff --git a/airflow/providers/github/operators/github.py b/airflow/providers/github/operators/github.py index cb650c6ed9610..cf7903eb3fb56 100644 --- a/airflow/providers/github/operators/github.py +++ b/airflow/providers/github/operators/github.py @@ -23,7 +23,7 @@ class GithubOperator(BaseOperator): """ - Executes sql code in a specific Github database + Executes Github Operations .. seealso:: For more information on how to use this operator, take a look at the guide: @@ -36,20 +36,15 @@ class GithubOperator(BaseOperator): :type github_conn_id: str """ - template_fields = ['sql'] - def __init__( self, *, - sql: str, github_conn_id: str = 'github_default', **kwargs, ) -> None: super().__init__(**kwargs) self.github_conn_id = github_conn_id - self.sql = sql - def execute(self, context: Dict) -> None: - self.log.info('Executing: %s', self.sql) - self.hook = GithubHook(conn_id=self.github_conn_id) - self.hook.query(self.sql) + # def execute(self, context: Dict) -> None: + # self.hook = GithubHook(conn_id=self.github_conn_id) + # self.hook.query(self.sql) From fa2b56eec35cf2f0a8ba3a85fa35925064615991 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Fri, 14 Jan 2022 15:48:12 +0000 Subject: [PATCH 06/22] 20300 - files refactored to allow execution of parameterized methods --- airflow/providers/github/hooks/github.py | 9 +--- airflow/providers/github/operators/github.py | 50 +++++++++++++++----- airflow/providers/github/sensors/__init__.py | 16 +++++++ 3 files changed, 55 insertions(+), 20 deletions(-) create mode 100644 airflow/providers/github/sensors/__init__.py diff --git a/airflow/providers/github/hooks/github.py b/airflow/providers/github/hooks/github.py index b68f504ecf2be..6e3f5786fbede 100644 --- a/airflow/providers/github/hooks/github.py +++ b/airflow/providers/github/hooks/github.py @@ -40,10 +40,10 @@ class GithubHook(BaseHook): conn_type = 'github' hook_name = 'Github' - def __init__(self, conn_id: str = default_conn_name, *args, **kwargs) -> None: + def __init__(self, github_conn_id: str = default_conn_name, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.connection = None - self.github_conn_id = conn_id + self.github_conn_id = github_conn_id self.client = None def get_conn(self) -> GithubClient: @@ -62,11 +62,6 @@ def get_conn(self) -> GithubClient: self.client = GithubClient(login_or_token=access_token) return self.client - def find_repos(self) -> PaginatedList: - """Function to get bucket id by name.""" - repos = self.client.get_repos() - return repos - @staticmethod def get_ui_field_behaviour() -> Dict: """Returns custom field behaviour""" diff --git a/airflow/providers/github/operators/github.py b/airflow/providers/github/operators/github.py index cf7903eb3fb56..84f87062aa701 100644 --- a/airflow/providers/github/operators/github.py +++ b/airflow/providers/github/operators/github.py @@ -15,36 +15,60 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Dict +from typing import Dict, Optional, Callable +from github import GithubException + +from airflow import AirflowException from airflow.models import BaseOperator from airflow.providers.github.hooks.github import GithubHook class GithubOperator(BaseOperator): """ - Executes Github Operations - - .. seealso:: - For more information on how to use this operator, take a look at the guide: - :ref:`howto/operator:GithubOperator` + GithubOperator to interact and perform action on Github API. + This operator is designed to use Github Python SDK: https://github.com/PyGithub/PyGithub - :param sql: the sql code to be executed. Can receive a str representing a - sql statement - :type sql: str - :param github_conn_id: Reference to :ref:`Github connection id `. + :param github_conn_id: reference to a pre-defined Github Connection :type github_conn_id: str + :param github_method: method name from Github Python SDK to be called + :type github_method: str + :param github_method_args: required method parameters for the github_method. (templated) + :type github_method_args: dict + :param result_processor: function to further process the response from Github + :type result_processor: function """ + template_fields = ("github_method_args",) + def __init__( self, *, + github_method: str, github_conn_id: str = 'github_default', + github_method_args: Optional[dict] = None, + result_processor: Optional[Callable] = None, **kwargs, ) -> None: super().__init__(**kwargs) self.github_conn_id = github_conn_id + self.method_name = github_method + self.github_method_args = github_method_args + self.result_processor = result_processor + + def execute(self, context: Dict) -> Any: + try: + # Default method execution is on the top level Github client + hook = GithubHook(github_conn_id=self.github_conn_id) + resource = hook.client + + github_result = getattr(resource, self.method_name)(**self.github_method_args) + if self.result_processor: + return self.result_processor(context, github_result) + + return github_result - # def execute(self, context: Dict) -> None: - # self.hook = GithubHook(conn_id=self.github_conn_id) - # self.hook.query(self.sql) + except GithubException as github_error: + raise AirflowException(f"Failed to execute GithubOperator, error: {str(github_error)}") + except Exception as e: + raise AirflowException(f"Github operator error: {str(e)}") diff --git a/airflow/providers/github/sensors/__init__.py b/airflow/providers/github/sensors/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/github/sensors/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. From d78e150cfcd15a2b378ffa6f333d9171254e8ff6 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Fri, 14 Jan 2022 16:47:56 +0000 Subject: [PATCH 07/22] 20300 - tag sensor implemented --- airflow/providers/github/sensors/github.py | 153 ++++++++++++++++++ .../providers/github/sensors/github_old.py | 68 ++++++++ 2 files changed, 221 insertions(+) create mode 100644 airflow/providers/github/sensors/github.py create mode 100644 airflow/providers/github/sensors/github_old.py diff --git a/airflow/providers/github/sensors/github.py b/airflow/providers/github/sensors/github.py new file mode 100644 index 0000000000000..a6b28257f4389 --- /dev/null +++ b/airflow/providers/github/sensors/github.py @@ -0,0 +1,153 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Any, Callable, Dict, Optional + +from github import Repository, GithubException + +from airflow import AirflowException +from airflow.providers.github.operators.github import GithubOperator +from airflow.sensors.base import BaseSensorOperator +from airflow.utils.context import Context + + +class GithubSensor(BaseSensorOperator): + """ + Base GithubSensor which can monitor for any change. + + :param github_conn_id: reference to a pre-defined Github Connection + :type github_conn_id: str + :param method_name: method name from PyGithub to be executed + :type method_name: str + :param method_params: parameters for the method method_name + :type method_params: dict + :param result_processor: function that return boolean and act as a sensor response + :type result_processor: function + """ + + def __init__( + self, + *, + method_name: str, + github_conn_id: str = 'github_default', + method_params: Optional[dict] = None, + result_processor: Optional[Callable] = None, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.github_conn_id = github_conn_id + self.result_processor = None + if result_processor is not None: + self.result_processor = result_processor + self.method_name = method_name + self.method_params = method_params + self.github_operator = GithubOperator( + task_id=self.task_id, + github_conn_id=self.github_conn_id, + github_method=self.method_name, + github_method_args=self.method_params, + result_processor=self.result_processor, + ) + + def poke(self, context: Dict) -> Any: + return self.github_operator.execute(context=context) + + +class GithubRepositorySensor(GithubSensor): + """ + Base github sensor at Repository level. + + :param github_conn_id: reference to a pre-defined Github Connection + :type github_conn_id: str + :param repository_name: full qualified name of the repository to be monitored, ex. "apache/airflow" + :type repository_name: str + """ + + def __init__( + self, + *, + github_conn_id: str = 'github_default', + repository_name: Optional[str] = None, + result_processor: Optional[Callable] = None, + **kwargs, + ) -> None: + super().__init__(github_conn_id=github_conn_id, result_processor=result_processor, + method_name="get_repo", + method_params={'full_name_or_id': repository_name}, **kwargs) + + def poke(self, context: Context) -> bool: + """ + Function that the sensors defined while deriving this class should + override. + """ + raise AirflowException('Override me.') + + +class GithubTagSensor(GithubRepositorySensor): + """ + Monitors a github tag for its creation. + + :param github_conn_id: reference to a pre-defined Github Connection + :type github_conn_id: str + :param tag_name: name of the tag to be monitored + :type tag_name: str + :param repository_name: full qualified name of the repository to be monitored, ex. "apache/airflow" + :type repository_name: str + """ + + template_fields = ("tag_name",) + + def __init__( + self, + *, + github_conn_id: str = 'github_default', + tag_name: Optional[str] = None, + repository_name: Optional[str] = None, + **kwargs, + ) -> None: + self.repository_name = repository_name + self.tag_name = tag_name + super().__init__(github_conn_id=github_conn_id, repository_name=repository_name, + result_processor=self.tag_checker, + **kwargs) + + def poke(self, context: Dict) -> Any: + self.log.info('Poking for tag: %s in repository: %s', self.tag_name, self.repository_name) + return GithubSensor.poke(self, context=context) + + def tag_checker(self, repo: Repository) -> Optional[bool]: + """Checking existence of Tag in a Repository""" + result = None + try: + if repo is not None and self.tag_name is not None: + all_tags = [x.name for x in repo.get_tags()] + result = self.tag_name in all_tags + + except GithubException as github_error: + raise AirflowException(f"Failed to execute GithubSensor, error: {str(github_error)}") + except Exception as e: + raise AirflowException(f"Github operator error: {str(e)}") + + if result is True: + self.log.info( + "Tag %s exists in %s repository, Success.", self.tag_name, self.repository_name + ) + else: + self.log.info( + "Tag %s doesn't exists in %s repository yet.", self.tag_name, self.repository_name + ) + return result diff --git a/airflow/providers/github/sensors/github_old.py b/airflow/providers/github/sensors/github_old.py new file mode 100644 index 0000000000000..74b047d4f43f1 --- /dev/null +++ b/airflow/providers/github/sensors/github_old.py @@ -0,0 +1,68 @@ +# # +# # Licensed to the Apache Software Foundation (ASF) under one +# # or more contributor license agreements. See the NOTICE file +# # distributed with this work for additional information +# # regarding copyright ownership. The ASF licenses this file +# # to you under the Apache License, Version 2.0 (the +# # "License"); you may not use this file except in compliance +# # with the License. You may obtain a copy of the License at +# # +# # http://www.apache.org/licenses/LICENSE-2.0 +# # +# # Unless required by applicable law or agreed to in writing, +# # software distributed under the License is distributed on an +# # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# # KIND, either express or implied. See the License for the +# # specific language governing permissions and limitations +# # under the License. +# from typing import Any, Callable, Dict, Optional +# +# from github import Github as GithubClient +# +# from airflow.providers.github.hooks.github import GithubHook +# from airflow.sensors.base import BaseSensorOperator +# +# +# class GithubTagSensor (BaseSensorOperator): +# """ +# Monitors a creation of Github tag. +# +# :param github_conn_id: reference to a pre-defined Github Connection +# :type github_conn_id: str +# """ +# +# def __init__( +# self, +# *, +# repository: str, +# tag_name: str, +# github_conn_id: str = 'github_default', +# **kwargs, +# ) -> None: +# super().__init__(**kwargs) +# self.github_conn_id = github_conn_id +# self.repository = repository +# self.tag_name = tag_name +# self.hook = GithubHook(github_conn_id=github_conn_id, **kwargs) +# +# def poke(self, context: Dict[Any, Any]) -> bool: +# +# self.log.info('Poking for tag: %s in repository: %s', self.tag_name, self.repository) +# try: +# response = self.hook.run( +# self.endpoint, +# data=self.request_params, +# headers=self.headers, +# extra_options=self.extra_options, +# ) +# if self.response_check: +# kwargs = determine_kwargs(self.response_check, [response], context) +# return self.response_check(response, **kwargs) +# except AirflowException as exc: +# if str(exc).startswith("404"): +# return False +# +# raise exc +# +# return True +# From 25304ae919f344ca9cc233971b587ec099aa74f0 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Sat, 15 Jan 2022 14:19:17 +0000 Subject: [PATCH 08/22] 20300 - github tag sensor working in local env --- airflow/providers/github/hooks/github.py | 3 +-- airflow/providers/github/operators/github.py | 4 ++-- tests/providers/github/sensors/__init__.py | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 tests/providers/github/sensors/__init__.py diff --git a/airflow/providers/github/hooks/github.py b/airflow/providers/github/hooks/github.py index 6e3f5786fbede..bee2af5f18d21 100644 --- a/airflow/providers/github/hooks/github.py +++ b/airflow/providers/github/hooks/github.py @@ -45,6 +45,7 @@ def __init__(self, github_conn_id: str = default_conn_name, *args, **kwargs) -> self.connection = None self.github_conn_id = github_conn_id self.client = None + self.get_conn() def get_conn(self) -> GithubClient: """ @@ -57,8 +58,6 @@ def get_conn(self) -> GithubClient: self.connection = self.get_connection(self.github_conn_id) access_token = self.connection.password - # self.log.info('Organization: %s', self.org_name) - self.client = GithubClient(login_or_token=access_token) return self.client diff --git a/airflow/providers/github/operators/github.py b/airflow/providers/github/operators/github.py index 84f87062aa701..7864aac3552e2 100644 --- a/airflow/providers/github/operators/github.py +++ b/airflow/providers/github/operators/github.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Dict, Optional, Callable +from typing import Dict, Optional, Callable, Any from github import GithubException @@ -64,7 +64,7 @@ def execute(self, context: Dict) -> Any: github_result = getattr(resource, self.method_name)(**self.github_method_args) if self.result_processor: - return self.result_processor(context, github_result) + return self.result_processor(github_result) return github_result diff --git a/tests/providers/github/sensors/__init__.py b/tests/providers/github/sensors/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/github/sensors/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. From 85bf561d5c3da0a1435e8fe985bb1a187dc977bd Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Mon, 17 Jan 2022 21:50:16 +0000 Subject: [PATCH 09/22] 20300 - github tag sensor test working --- tests/providers/github/sensors/test_github.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/providers/github/sensors/test_github.py diff --git a/tests/providers/github/sensors/test_github.py b/tests/providers/github/sensors/test_github.py new file mode 100644 index 0000000000000..2e4a68a46f0d0 --- /dev/null +++ b/tests/providers/github/sensors/test_github.py @@ -0,0 +1,70 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import unittest +from unittest.mock import Mock, patch + +from github.GitTag import GitTag + +from airflow.models import Connection +from airflow.models.dag import DAG +from airflow.providers.github.sensors.github import GithubTagSensor +from airflow.utils import db, timezone + +DEFAULT_DATE = timezone.datetime(2017, 1, 1) +github_client_mock = Mock(name="github_client_for_test") + + +class TestGithubSensor(unittest.TestCase): + def setUp(self): + args = {'owner': 'airflow', 'start_date': DEFAULT_DATE} + dag = DAG('test_dag_id', default_args=args) + self.dag = dag + db.merge_conn( + Connection( + conn_id='github_default', + conn_type='github', + host='https://localhost/github/', + port=443, + extra='{"verify": "False", "project": "AIRFLOW"}', + ) + ) + + @patch("airflow.providers.github.hooks.github.GithubClient", autospec=True, return_value=github_client_mock) + def test_github_tag_created(self, github_mock): + class MyTag(object): + pass + tag = MyTag() + tag.name = "v1.0" + + github_mock.return_value.get_repo.return_value.get_tags.return_value = [tag] + + github_tag_sensor = GithubTagSensor( + task_id='search-ticket-test', + tag_name='v1.0', + repository_name="pateash/jetbrains_settings", + timeout=60, + poke_interval=10, + dag=self.dag, + ) + + github_tag_sensor.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) + + assert github_mock.return_value.get_repo.called + assert github_mock.return_value.get_repo.return_value.get_tags.called From 8b3361d77d0a9e3657ec52703c73f261651ef50f Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Mon, 24 Jan 2022 22:10:01 +0000 Subject: [PATCH 10/22] 20300 - github hook test added --- tests/providers/github/hooks/test_github.py | 48 ++++++++++ tests/providers/github/hooks/test_influxdb.py | 88 ------------------- .../github/operators/test_influxdb.py | 32 ------- 3 files changed, 48 insertions(+), 120 deletions(-) create mode 100644 tests/providers/github/hooks/test_github.py delete mode 100644 tests/providers/github/hooks/test_influxdb.py delete mode 100644 tests/providers/github/operators/test_influxdb.py diff --git a/tests/providers/github/hooks/test_github.py b/tests/providers/github/hooks/test_github.py new file mode 100644 index 0000000000000..f4ab2c6c0afb5 --- /dev/null +++ b/tests/providers/github/hooks/test_github.py @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import unittest +from unittest.mock import Mock, patch + +from airflow.models import Connection +from airflow.providers.github.hooks.github import GithubHook +from airflow.utils import db + +github_client_mock = Mock(name="github_client_for_test") + + +class TestGithubHook(unittest.TestCase): + def setUp(self): + db.merge_conn( + Connection( + conn_id='github_default', + conn_type='github', + host='https://localhost/github/', + port=443, + extra='{"verify": "False", "project": "AIRFLOW"}', + ) + ) + + @patch("airflow.providers.github.hooks.github.GithubClient", autospec=True, return_value=github_client_mock) + def test_github_client_connection(self, github_mock): + github_hook = GithubHook() + + assert github_mock.called + assert isinstance(github_hook.client, Mock) + assert github_hook.client.name == github_mock.return_value.name diff --git a/tests/providers/github/hooks/test_influxdb.py b/tests/providers/github/hooks/test_influxdb.py deleted file mode 100644 index 8b86f03cb0933..0000000000000 --- a/tests/providers/github/hooks/test_influxdb.py +++ /dev/null @@ -1,88 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -from unittest import mock - -from airflow.models import Connection -from airflow.providers.influxdb.hooks.influxdb import InfluxDBHook - - -class TestInfluxDbHook(unittest.TestCase): - def setUp(self): - super().setUp() - self.influxdb_hook = InfluxDBHook() - extra = {} - extra['token'] = '123456789' - extra['org_name'] = 'test' - - self.connection = Connection(schema='http', host='localhost', extra=extra) - - def test_get_conn(self): - self.influxdb_hook.get_connection = mock.Mock() - self.influxdb_hook.get_connection.return_value = self.connection - - self.influxdb_hook.get_client = mock.Mock() - self.influxdb_hook.get_conn() - - assert self.influxdb_hook.org_name == 'test' - assert self.influxdb_hook.uri == 'http://localhost:7687' - - assert self.influxdb_hook.get_connection.return_value.schema == 'http' - assert self.influxdb_hook.get_connection.return_value.host == 'localhost' - - assert self.influxdb_hook.get_client is not None - - def test_query(self): - self.influxdb_hook.get_conn = mock.Mock() - - influxdb_query = 'SELECT "duration" FROM "pyexample"' - self.influxdb_hook.query(influxdb_query) - - self.influxdb_hook.get_conn.assert_called() - - def test_query_to_df(self): - self.influxdb_hook.get_conn = mock.Mock() - - self.influxdb_hook.write = mock.Mock() - influxdb_query = 'SELECT "duration" FROM "pyexample"' - self.influxdb_hook.query_to_df(influxdb_query) - - self.influxdb_hook.get_conn.assert_called() - - def test_write(self): - self.influxdb_hook.get_connection = mock.Mock() - self.influxdb_hook.get_connection.return_value = self.connection - - self.influxdb_hook.get_conn = mock.Mock() - self.influxdb_hook.client = mock.Mock() - self.influxdb_hook.client.write_api = mock.Mock() - - self.influxdb_hook.write("test_bucket", "test_point", "location", "Prague", "temperature", 25.3, True) - - self.influxdb_hook.client.write_api.assert_called() - - def test_find_bucket_by_id(self): - self.influxdb_hook.get_connection = mock.Mock() - self.influxdb_hook.get_connection.return_value = self.connection - - self.influxdb_hook.client = mock.Mock() - self.influxdb_hook.client.buckets_api = mock.Mock() - - self.influxdb_hook.find_bucket_id_by_name("test") - - self.influxdb_hook.client.buckets_api.assert_called() diff --git a/tests/providers/github/operators/test_influxdb.py b/tests/providers/github/operators/test_influxdb.py deleted file mode 100644 index f7b82424cfa7b..0000000000000 --- a/tests/providers/github/operators/test_influxdb.py +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -from unittest import mock - -from airflow.providers.influxdb.operators.influxdb import InfluxDBOperator - - -class TestInfluxDBOperator(unittest.TestCase): - @mock.patch('airflow.providers.influxdb.operators.influxdb.InfluxDBHook') - def test_influxdb_operator_test(self, mock_hook): - - sql = """from(bucket:"test") |> range(start: -10m)""" - op = InfluxDBOperator(task_id='basic_influxdb', sql=sql, influxdb_conn_id='influxdb_default') - op.execute(mock.MagicMock()) - mock_hook.assert_called_once_with(conn_id='influxdb_default') - mock_hook.return_value.query.assert_called_once_with(sql) From d18eddcb373875af5fff55d9d6f9f5182c3687fe Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Mon, 24 Jan 2022 22:26:58 +0000 Subject: [PATCH 11/22] 20300 - github operator test added --- .../providers/github/operators/test_github.py | 67 +++++++++++++++++++ tests/providers/github/sensors/test_github.py | 4 +- 2 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 tests/providers/github/operators/test_github.py diff --git a/tests/providers/github/operators/test_github.py b/tests/providers/github/operators/test_github.py new file mode 100644 index 0000000000000..8c69e3949b423 --- /dev/null +++ b/tests/providers/github/operators/test_github.py @@ -0,0 +1,67 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import unittest +from unittest.mock import Mock, patch + +from airflow.models import Connection +from airflow.models.dag import DAG +from airflow.providers.github.operators.github import GithubOperator +from airflow.utils import db, timezone + +DEFAULT_DATE = timezone.datetime(2017, 1, 1) +github_client_mock = Mock(name="github_client_for_test") + +class TestGithubOperator(unittest.TestCase): + def setUp(self): + args = {'owner': 'airflow', 'start_date': DEFAULT_DATE} + dag = DAG('test_dag_id', default_args=args) + self.dag = dag + db.merge_conn( + Connection( + conn_id='github_default', + conn_type='github', + host='https://localhost/github/', + port=443, + extra='{"verify": "False", "project": "AIRFLOW"}', + ) + ) + + @patch("airflow.providers.github.hooks.github.GithubClient", autospec=True, return_value=github_client_mock) + def test_find_repos(self, github_mock): + class MockRepository(object): + pass + + repo = MockRepository() + repo.full_name = "apache/airflow" + + github_mock.return_value.get_repo.return_value = repo + + github_operator = GithubOperator( + task_id='githib-test', + github_method="get_repo", + github_method_args={'full_name_or_id': 'apache/airflow'}, + result_processor=lambda r: r.full_name, + dag=self.dag, + ) + + github_operator.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, ignore_ti_state=True) + + assert github_mock.called + assert github_mock.return_value.get_repo.called diff --git a/tests/providers/github/sensors/test_github.py b/tests/providers/github/sensors/test_github.py index 2e4a68a46f0d0..e58a30c9e639f 100644 --- a/tests/providers/github/sensors/test_github.py +++ b/tests/providers/github/sensors/test_github.py @@ -48,9 +48,9 @@ def setUp(self): @patch("airflow.providers.github.hooks.github.GithubClient", autospec=True, return_value=github_client_mock) def test_github_tag_created(self, github_mock): - class MyTag(object): + class MockTag(object): pass - tag = MyTag() + tag = MockTag() tag.name = "v1.0" github_mock.return_value.get_repo.return_value.get_tags.return_value = [tag] From bb3389331d3133e1bd0476c9d91bf318b6d64401 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Tue, 25 Jan 2022 15:56:56 +0000 Subject: [PATCH 12/22] 20300 - precommit changes --- CONTRIBUTING.rst | 1 - .../github/.latest-doc-only-change.txt | 2 - airflow/providers/github/hooks/github.py | 10 ++- airflow/providers/github/operators/github.py | 19 +++--- airflow/providers/github/sensors/github.py | 37 +++++----- .../providers/github/sensors/github_old.py | 68 ------------------- tests/providers/github/hooks/test_github.py | 4 +- .../providers/github/operators/test_github.py | 9 ++- tests/providers/github/sensors/test_github.py | 9 +-- 9 files changed, 48 insertions(+), 111 deletions(-) delete mode 100644 airflow/providers/github/.latest-doc-only-change.txt delete mode 100644 airflow/providers/github/sensors/github_old.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f0a35194d852a..142ac8d77b660 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -681,7 +681,6 @@ apache.hive amazon,microsoft.mssql,mysql,presto,samba,vertica apache.livy http dingding http discord http -github PyGithub google amazon,apache.beam,apache.cassandra,cncf.kubernetes,facebook,microsoft.azure,microsoft.mssql,mysql,oracle,postgres,presto,salesforce,sftp,ssh,trino hashicorp google microsoft.azure google,oracle,sftp diff --git a/airflow/providers/github/.latest-doc-only-change.txt b/airflow/providers/github/.latest-doc-only-change.txt deleted file mode 100644 index 884bd61c6bef8..0000000000000 --- a/airflow/providers/github/.latest-doc-only-change.txt +++ /dev/null @@ -1,2 +0,0 @@ -87dc63b65daaf77c4c9f2f6611b72bcc78603d1e -# TODO: fix this diff --git a/airflow/providers/github/hooks/github.py b/airflow/providers/github/hooks/github.py index bee2af5f18d21..2844158cb2c2e 100644 --- a/airflow/providers/github/hooks/github.py +++ b/airflow/providers/github/hooks/github.py @@ -18,9 +18,9 @@ """This module allows to connect to a Github.""" -from typing import Dict +from typing import Dict, Optional -from github import Github as GithubClient, PaginatedList +from github import Github as GithubClient from airflow.hooks.base import BaseHook @@ -42,9 +42,8 @@ class GithubHook(BaseHook): def __init__(self, github_conn_id: str = default_conn_name, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self.connection = None self.github_conn_id = github_conn_id - self.client = None + self.client: Optional[GithubClient] = None self.get_conn() def get_conn(self) -> GithubClient: @@ -55,8 +54,7 @@ def get_conn(self) -> GithubClient: if self.client is not None: return self.client - self.connection = self.get_connection(self.github_conn_id) - access_token = self.connection.password + access_token = self.get_connection(self.github_conn_id).password self.client = GithubClient(login_or_token=access_token) return self.client diff --git a/airflow/providers/github/operators/github.py b/airflow/providers/github/operators/github.py index 7864aac3552e2..56ddd299e5d89 100644 --- a/airflow/providers/github/operators/github.py +++ b/airflow/providers/github/operators/github.py @@ -15,27 +15,28 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Dict, Optional, Callable, Any +from typing import Any, Callable, Optional from github import GithubException from airflow import AirflowException from airflow.models import BaseOperator from airflow.providers.github.hooks.github import GithubHook +from airflow.utils.context import Context class GithubOperator(BaseOperator): """ - GithubOperator to interact and perform action on Github API. - This operator is designed to use Github Python SDK: https://github.com/PyGithub/PyGithub + GithubOperator to interact and perform action on GitHub API. + This operator is designed to use GitHub Python SDK: https://github.com/PyGithub/PyGithub - :param github_conn_id: reference to a pre-defined Github Connection + :param github_conn_id: reference to a pre-defined GitHub Connection :type github_conn_id: str - :param github_method: method name from Github Python SDK to be called + :param github_method: method name from GitHub Python SDK to be called :type github_method: str :param github_method_args: required method parameters for the github_method. (templated) :type github_method_args: dict - :param result_processor: function to further process the response from Github + :param result_processor: function to further process the response from GitHub :type result_processor: function """ @@ -56,9 +57,9 @@ def __init__( self.github_method_args = github_method_args self.result_processor = result_processor - def execute(self, context: Dict) -> Any: + def execute(self, context: Context) -> Any: try: - # Default method execution is on the top level Github client + # Default method execution is on the top level GitHub client hook = GithubHook(github_conn_id=self.github_conn_id) resource = hook.client @@ -71,4 +72,4 @@ def execute(self, context: Dict) -> Any: except GithubException as github_error: raise AirflowException(f"Failed to execute GithubOperator, error: {str(github_error)}") except Exception as e: - raise AirflowException(f"Github operator error: {str(e)}") + raise AirflowException(f'GitHub operator error: {str(e)}') diff --git a/airflow/providers/github/sensors/github.py b/airflow/providers/github/sensors/github.py index a6b28257f4389..942e4a00db3fe 100644 --- a/airflow/providers/github/sensors/github.py +++ b/airflow/providers/github/sensors/github.py @@ -15,9 +15,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Optional -from github import Repository, GithubException +from github import GithubException from airflow import AirflowException from airflow.providers.github.operators.github import GithubOperator @@ -63,7 +63,7 @@ def __init__( result_processor=self.result_processor, ) - def poke(self, context: Dict) -> Any: + def poke(self, context: Context) -> bool: return self.github_operator.execute(context=context) @@ -85,9 +85,13 @@ def __init__( result_processor: Optional[Callable] = None, **kwargs, ) -> None: - super().__init__(github_conn_id=github_conn_id, result_processor=result_processor, - method_name="get_repo", - method_params={'full_name_or_id': repository_name}, **kwargs) + super().__init__( + github_conn_id=github_conn_id, + result_processor=result_processor, + method_name="get_repo", + method_params={'full_name_or_id': repository_name}, + **kwargs, + ) def poke(self, context: Context) -> bool: """ @@ -121,15 +125,18 @@ def __init__( ) -> None: self.repository_name = repository_name self.tag_name = tag_name - super().__init__(github_conn_id=github_conn_id, repository_name=repository_name, - result_processor=self.tag_checker, - **kwargs) + super().__init__( + github_conn_id=github_conn_id, + repository_name=repository_name, + result_processor=self.tag_checker, + **kwargs, + ) - def poke(self, context: Dict) -> Any: + def poke(self, context: Context) -> bool: self.log.info('Poking for tag: %s in repository: %s', self.tag_name, self.repository_name) return GithubSensor.poke(self, context=context) - def tag_checker(self, repo: Repository) -> Optional[bool]: + def tag_checker(self, repo: Any) -> Optional[bool]: """Checking existence of Tag in a Repository""" result = None try: @@ -143,11 +150,7 @@ def tag_checker(self, repo: Repository) -> Optional[bool]: raise AirflowException(f"Github operator error: {str(e)}") if result is True: - self.log.info( - "Tag %s exists in %s repository, Success.", self.tag_name, self.repository_name - ) + self.log.info("Tag %s exists in %s repository, Success.", self.tag_name, self.repository_name) else: - self.log.info( - "Tag %s doesn't exists in %s repository yet.", self.tag_name, self.repository_name - ) + self.log.info("Tag %s doesn't exists in %s repository yet.", self.tag_name, self.repository_name) return result diff --git a/airflow/providers/github/sensors/github_old.py b/airflow/providers/github/sensors/github_old.py deleted file mode 100644 index 74b047d4f43f1..0000000000000 --- a/airflow/providers/github/sensors/github_old.py +++ /dev/null @@ -1,68 +0,0 @@ -# # -# # Licensed to the Apache Software Foundation (ASF) under one -# # or more contributor license agreements. See the NOTICE file -# # distributed with this work for additional information -# # regarding copyright ownership. The ASF licenses this file -# # to you under the Apache License, Version 2.0 (the -# # "License"); you may not use this file except in compliance -# # with the License. You may obtain a copy of the License at -# # -# # http://www.apache.org/licenses/LICENSE-2.0 -# # -# # Unless required by applicable law or agreed to in writing, -# # software distributed under the License is distributed on an -# # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# # KIND, either express or implied. See the License for the -# # specific language governing permissions and limitations -# # under the License. -# from typing import Any, Callable, Dict, Optional -# -# from github import Github as GithubClient -# -# from airflow.providers.github.hooks.github import GithubHook -# from airflow.sensors.base import BaseSensorOperator -# -# -# class GithubTagSensor (BaseSensorOperator): -# """ -# Monitors a creation of Github tag. -# -# :param github_conn_id: reference to a pre-defined Github Connection -# :type github_conn_id: str -# """ -# -# def __init__( -# self, -# *, -# repository: str, -# tag_name: str, -# github_conn_id: str = 'github_default', -# **kwargs, -# ) -> None: -# super().__init__(**kwargs) -# self.github_conn_id = github_conn_id -# self.repository = repository -# self.tag_name = tag_name -# self.hook = GithubHook(github_conn_id=github_conn_id, **kwargs) -# -# def poke(self, context: Dict[Any, Any]) -> bool: -# -# self.log.info('Poking for tag: %s in repository: %s', self.tag_name, self.repository) -# try: -# response = self.hook.run( -# self.endpoint, -# data=self.request_params, -# headers=self.headers, -# extra_options=self.extra_options, -# ) -# if self.response_check: -# kwargs = determine_kwargs(self.response_check, [response], context) -# return self.response_check(response, **kwargs) -# except AirflowException as exc: -# if str(exc).startswith("404"): -# return False -# -# raise exc -# -# return True -# diff --git a/tests/providers/github/hooks/test_github.py b/tests/providers/github/hooks/test_github.py index f4ab2c6c0afb5..b4feab3183756 100644 --- a/tests/providers/github/hooks/test_github.py +++ b/tests/providers/github/hooks/test_github.py @@ -39,7 +39,9 @@ def setUp(self): ) ) - @patch("airflow.providers.github.hooks.github.GithubClient", autospec=True, return_value=github_client_mock) + @patch( + "airflow.providers.github.hooks.github.GithubClient", autospec=True, return_value=github_client_mock + ) def test_github_client_connection(self, github_mock): github_hook = GithubHook() diff --git a/tests/providers/github/operators/test_github.py b/tests/providers/github/operators/test_github.py index 8c69e3949b423..23461cbbdf006 100644 --- a/tests/providers/github/operators/test_github.py +++ b/tests/providers/github/operators/test_github.py @@ -28,6 +28,7 @@ DEFAULT_DATE = timezone.datetime(2017, 1, 1) github_client_mock = Mock(name="github_client_for_test") + class TestGithubOperator(unittest.TestCase): def setUp(self): args = {'owner': 'airflow', 'start_date': DEFAULT_DATE} @@ -43,9 +44,11 @@ def setUp(self): ) ) - @patch("airflow.providers.github.hooks.github.GithubClient", autospec=True, return_value=github_client_mock) + @patch( + "airflow.providers.github.hooks.github.GithubClient", autospec=True, return_value=github_client_mock + ) def test_find_repos(self, github_mock): - class MockRepository(object): + class MockRepository: pass repo = MockRepository() @@ -54,7 +57,7 @@ class MockRepository(object): github_mock.return_value.get_repo.return_value = repo github_operator = GithubOperator( - task_id='githib-test', + task_id='github-test', github_method="get_repo", github_method_args={'full_name_or_id': 'apache/airflow'}, result_processor=lambda r: r.full_name, diff --git a/tests/providers/github/sensors/test_github.py b/tests/providers/github/sensors/test_github.py index e58a30c9e639f..71cb0a75cacda 100644 --- a/tests/providers/github/sensors/test_github.py +++ b/tests/providers/github/sensors/test_github.py @@ -20,8 +20,6 @@ import unittest from unittest.mock import Mock, patch -from github.GitTag import GitTag - from airflow.models import Connection from airflow.models.dag import DAG from airflow.providers.github.sensors.github import GithubTagSensor @@ -46,10 +44,13 @@ def setUp(self): ) ) - @patch("airflow.providers.github.hooks.github.GithubClient", autospec=True, return_value=github_client_mock) + @patch( + "airflow.providers.github.hooks.github.GithubClient", autospec=True, return_value=github_client_mock + ) def test_github_tag_created(self, github_mock): - class MockTag(object): + class MockTag: pass + tag = MockTag() tag.name = "v1.0" From 47f8fc17686a3d666b113014ba728a61d3207f03 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Tue, 25 Jan 2022 17:46:58 +0000 Subject: [PATCH 13/22] 20300 - github Enterprise connection added --- .../github/example_dags/example_github.py | 2 +- airflow/providers/github/hooks/github.py | 26 ++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/airflow/providers/github/example_dags/example_github.py b/airflow/providers/github/example_dags/example_github.py index ee503c5fc7864..97ef04fc5b133 100644 --- a/airflow/providers/github/example_dags/example_github.py +++ b/airflow/providers/github/example_dags/example_github.py @@ -24,7 +24,7 @@ @task(task_id="github_task") def test_github_hook(): - bucket_name = 'test-influx' + bucket_name = 'test-github' github_hook = GithubHook() client = github_hook.get_conn() print(client) diff --git a/airflow/providers/github/hooks/github.py b/airflow/providers/github/hooks/github.py index 2844158cb2c2e..8df51e3306c17 100644 --- a/airflow/providers/github/hooks/github.py +++ b/airflow/providers/github/hooks/github.py @@ -17,7 +17,7 @@ # under the License. """This module allows to connect to a Github.""" - +import logging from typing import Dict, Optional from github import Github as GithubClient @@ -29,9 +29,9 @@ class GithubHook(BaseHook): """ Interact with Github. - Performs a connection to Github and retrieves client. + Performs a connection to GitHub and retrieves client. - :param github_conn_id: Reference to :ref:`Github connection id `. + :param github_conn_id: Reference to :ref:`GitHub connection id `. :type github_conn_id: str """ @@ -48,28 +48,34 @@ def __init__(self, github_conn_id: str = default_conn_name, *args, **kwargs) -> def get_conn(self) -> GithubClient: """ - Function that initiates a new Github connection - with token and hostname name + Function that initiates a new GitHub connection + with token and hostname ( for GitHub Enterprise ) """ if self.client is not None: return self.client - access_token = self.get_connection(self.github_conn_id).password + conn = self.get_connection(self.github_conn_id) + access_token = conn.password + host = conn.host + + if not host: + self.client = GithubClient(login_or_token=access_token) + else: + self.client = GithubClient(login_or_token=access_token, base_url=host) - self.client = GithubClient(login_or_token=access_token) return self.client @staticmethod def get_ui_field_behaviour() -> Dict: """Returns custom field behaviour""" return { - "hidden_fields": ['schema', 'port', 'host', 'login', 'extra'], + "hidden_fields": ['schema', 'port', 'login', 'extra'], "relabeling": { - # 'host': 'Github Enterprise Url', + 'host': 'Github Enterprise Url (Optional)', 'password': 'Github Access Token', }, "placeholders": { - # 'host': 'https://{hostname}/api/v3', + 'host': 'https://{hostname}/api/v3 (for Github Enterprise Connection)', 'password': 'token credentials auth', }, } From 19b684a2b719d4408d2bcbc1fe8f94f073e77560 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Tue, 25 Jan 2022 18:14:13 +0000 Subject: [PATCH 14/22] 20300 - PR comments resolved --- airflow/providers/github/operators/github.py | 2 +- airflow/providers/github/sensors/github.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airflow/providers/github/operators/github.py b/airflow/providers/github/operators/github.py index 56ddd299e5d89..49f1d4f4d2d3f 100644 --- a/airflow/providers/github/operators/github.py +++ b/airflow/providers/github/operators/github.py @@ -36,7 +36,7 @@ class GithubOperator(BaseOperator): :type github_method: str :param github_method_args: required method parameters for the github_method. (templated) :type github_method_args: dict - :param result_processor: function to further process the response from GitHub + :param result_processor: function to further process the response from GitHub API :type result_processor: function """ diff --git a/airflow/providers/github/sensors/github.py b/airflow/providers/github/sensors/github.py index 942e4a00db3fe..cbd21deb1aab0 100644 --- a/airflow/providers/github/sensors/github.py +++ b/airflow/providers/github/sensors/github.py @@ -67,11 +67,11 @@ def poke(self, context: Context) -> bool: return self.github_operator.execute(context=context) -class GithubRepositorySensor(GithubSensor): +class BaseGithubRepositorySensor(GithubSensor): """ - Base github sensor at Repository level. + Base GitHub sensor at Repository level. - :param github_conn_id: reference to a pre-defined Github Connection + :param github_conn_id: reference to a pre-defined GitHub Connection :type github_conn_id: str :param repository_name: full qualified name of the repository to be monitored, ex. "apache/airflow" :type repository_name: str @@ -101,7 +101,7 @@ def poke(self, context: Context) -> bool: raise AirflowException('Override me.') -class GithubTagSensor(GithubRepositorySensor): +class GithubTagSensor(BaseGithubRepositorySensor): """ Monitors a github tag for its creation. @@ -109,7 +109,7 @@ class GithubTagSensor(GithubRepositorySensor): :type github_conn_id: str :param tag_name: name of the tag to be monitored :type tag_name: str - :param repository_name: full qualified name of the repository to be monitored, ex. "apache/airflow" + :param repository_name: fully qualified name of the repository to be monitored, ex. "apache/airflow" :type repository_name: str """ From 929636e33c2a3a9b78bfd727cf2e2a8a6787c977 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Wed, 26 Jan 2022 13:48:21 +0000 Subject: [PATCH 15/22] 20300 - adding docs --- .../example_dags/example_github_query.py | 39 ------------------- .../commits.rst | 18 --------- .../connections/github.rst | 27 ++++++------- 3 files changed, 11 insertions(+), 73 deletions(-) delete mode 100644 airflow/providers/github/example_dags/example_github_query.py diff --git a/airflow/providers/github/example_dags/example_github_query.py b/airflow/providers/github/example_dags/example_github_query.py deleted file mode 100644 index e385309d4593a..0000000000000 --- a/airflow/providers/github/example_dags/example_github_query.py +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from datetime import datetime - -from airflow.models.dag import DAG -from airflow.providers.github.operators.github import GithubOperator - -dag = DAG( - 'example_github_operator', - start_date=datetime(2021, 1, 1), - tags=['example'], - catchup=False, -) - -# [START howto_operator_github] - -query_github_task = GithubOperator( - github_conn_id='github_conn_id', - task_id='query_github', - sql='from(bucket:"test-influx") |> range(start: -10m, stop: {{ds}})', - dag=dag, -) - -# [END howto_operator_github] diff --git a/docs/apache-airflow-providers-github/commits.rst b/docs/apache-airflow-providers-github/commits.rst index eabdbf9713a51..626606bffd374 100644 --- a/docs/apache-airflow-providers-github/commits.rst +++ b/docs/apache-airflow-providers-github/commits.rst @@ -19,21 +19,3 @@ Package apache-airflow-providers-github ------------------------------------------------------ - -`Github `__ - - -This is detailed commit list of changes for versions provider package: ``github``. -For high-level changelog, see :doc:`package information including changelog `. - - -1.0.0 -..... - -Latest change: 2021-09-29 - -================================================================================================= =========== ====================================================================== -Commit Committed Subject -================================================================================================= =========== ====================================================================== -`e84527509e `_ 2021-09-29 ``Updating the Github example DAG to use the TaskFlow API (#18596)`` -================================================================================================= =========== ====================================================================== diff --git a/docs/apache-airflow-providers-github/connections/github.rst b/docs/apache-airflow-providers-github/connections/github.rst index 95d39ae2f86fc..22707b3e3b844 100644 --- a/docs/apache-airflow-providers-github/connections/github.rst +++ b/docs/apache-airflow-providers-github/connections/github.rst @@ -20,28 +20,23 @@ Github Connection ==================== -The Github connection type provides connection to a Github database. +The Github connection type provides connection to a Github or Github Enterprise. Configuring the Connection -------------------------- -Host (required) - The host to connect to. +Access Token (required) + Personal Access token with required permissions. + - GitHub - Create token - https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token/ + - GitHub Enterprise - Create token - https://docs.github.com/en/enterprise-cloud@latest/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token/ -Extra (required) - Specify the extra parameters (as json dictionary) that can be used in Github +Host (optional) + Specify the Github Enterprise Url (as string) that can be used for Github Enterprise connection. - The following extras are required: + The following Url should be in following format: - - token - Create token - https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token/ + * ``hostname``: Url for Your GitHub Enterprise Deployment. - * ``token``: Create token using the influxdb cli or UI + .. code-block:: - Example "extras" field: - - .. code-block:: JSON - - { - "token": "343434343423234234234343434", - "org_name": "Test" - } + https://{hostname}/api/v3 From 5b1d5a7fa3b3f1a9bbd3e9b37ad136b6f4765f53 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Wed, 26 Jan 2022 15:10:24 +0000 Subject: [PATCH 16/22] 20300 - adding docs --- .../github/example_dags/example_github.py | 70 +++++++++++-------- airflow/providers/github/operators/github.py | 4 ++ .../apache-airflow-providers-github/index.rst | 29 ++++---- .../operators/index.rst | 24 +++++-- .../connections/influxdb.rst | 37 ++++------ 5 files changed, 87 insertions(+), 77 deletions(-) diff --git a/airflow/providers/github/example_dags/example_github.py b/airflow/providers/github/example_dags/example_github.py index 97ef04fc5b133..48d5e849ccc3a 100644 --- a/airflow/providers/github/example_dags/example_github.py +++ b/airflow/providers/github/example_dags/example_github.py @@ -14,44 +14,54 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +import logging from datetime import datetime -from airflow.decorators import task from airflow.models.dag import DAG -from airflow.providers.github.hooks.github import GithubHook +from airflow.providers.github.operators.github import GithubOperator +from airflow.providers.github.sensors.github import GithubTagSensor +dag = DAG( + 'example_github_operator', + start_date=datetime(2021, 1, 1), + tags=['example'], + catchup=False, +) -@task(task_id="github_task") -def test_github_hook(): - bucket_name = 'test-github' - github_hook = GithubHook() - client = github_hook.get_conn() - print(client) - print(f"Organization name {github_hook.org_name}") +# [START howto_tag_sensor_github] - # Make sure enough permissions to create bucket. - github_hook.create_bucket(bucket_name, "Bucket to test github connection", github_hook.org_name) - github_hook.write(bucket_name, "test_point", "location", "Prague", "temperature", 25.3, True) +tag_sensor = GithubTagSensor( + task_id='example_tag_sensor', + tag_name='v1.0', + repository_name="apache/airflow", + timeout=60, + poke_interval=10, + dag=dag, +) - tables = github_hook.query('from(bucket:"test-influx") |> range(start: -10m)') +# [END howto_tag_sensor_github] - for table in tables: - print(table) - for record in table.records: - print(record.values) - bucket_id = github_hook.find_bucket_id_by_name(bucket_name) - print(bucket_id) - # Delete bucket takes bucket id. - github_hook.delete_bucket(bucket_name) +# [START howto_operator_list_repos_github] +github_list_repos = GithubOperator( + task_id='github_list_repos', + github_method="get_user", + github_method_args={}, + result_processor=lambda user: logging.info(list(user.get_repos())), + dag=dag, +) -with DAG( - dag_id='github_example_dag', - schedule_interval=None, - start_date=datetime(2021, 1, 1), - max_active_runs=1, - tags=['example'], -) as dag: - test_github_hook() +# [END howto_operator_list_repos_github] + +# [START howto_operator_list_tags_github] + +list_repo_tags = GithubOperator( + task_id='list_repo_tags', + github_method="get_repo", + github_method_args={'full_name_or_id': 'apache/airflow'}, + result_processor=lambda repo: logging.info(list(repo.get_tags())), + dag=dag, +) + +# [END howto_operator_list_tags_github] diff --git a/airflow/providers/github/operators/github.py b/airflow/providers/github/operators/github.py index 49f1d4f4d2d3f..27a276d92c6ae 100644 --- a/airflow/providers/github/operators/github.py +++ b/airflow/providers/github/operators/github.py @@ -30,6 +30,10 @@ class GithubOperator(BaseOperator): GithubOperator to interact and perform action on GitHub API. This operator is designed to use GitHub Python SDK: https://github.com/PyGithub/PyGithub + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:GithubOperator` + :param github_conn_id: reference to a pre-defined GitHub Connection :type github_conn_id: str :param github_method: method name from GitHub Python SDK to be called diff --git a/docs/apache-airflow-providers-github/index.rst b/docs/apache-airflow-providers-github/index.rst index 093ae911da4f0..213637c7316d0 100644 --- a/docs/apache-airflow-providers-github/index.rst +++ b/docs/apache-airflow-providers-github/index.rst @@ -16,7 +16,7 @@ specific language governing permissions and limitations under the License. -``apache-airflow-providers-influxdb`` +``apache-airflow-providers-github`` ======================================= Content @@ -26,32 +26,32 @@ Content :maxdepth: 1 :caption: Guides - Connection types + Connection types Operators .. toctree:: :maxdepth: 1 :caption: References - Python API <_api/airflow/providers/influxdb/index> + Python API <_api/airflow/providers/github/index> .. toctree:: :maxdepth: 1 :caption: Resources - Example DAGs + Example DAGs .. toctree:: :maxdepth: 1 :caption: Resources - PyPI Repository + PyPI Repository .. toctree:: :maxdepth: 1 :caption: Resources - PyPI Repository + PyPI Repository Installing from sources .. toctree:: @@ -70,25 +70,25 @@ Content Detailed list of commits -Package apache-airflow-providers-influxdb +Package apache-airflow-providers-github ------------------------------------------------------ -`InfluxDB `__ +`Github `__ -Release: 1.1.0 +Release: 1.0.0 Provider package ---------------- -This is a provider package for ``influxdb`` provider. All classes for this provider package -are in ``airflow.providers.influxdb`` python package. +This is a provider package for ``github`` provider. All classes for this provider package +are in ``airflow.providers.github`` python package. Installation ------------ You can install this package on top of an existing Airflow 2.1+ installation via -``pip install apache-airflow-providers-influxdb`` +``pip install apache-airflow-providers-github`` PIP requirements ---------------- @@ -96,8 +96,7 @@ PIP requirements =================== ================== PIP package Version required =================== ================== -``influxdb-client`` ``>=1.19.0`` -``pandas`` ``>=0.17.1, <2.0`` +``PyGithub`` =================== ================== -.. include:: ../../airflow/providers/influxdb/CHANGELOG.rst +.. include:: ../../airflow/providers/github/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-github/operators/index.rst b/docs/apache-airflow-providers-github/operators/index.rst index 0787eb91cf821..d005b3c1aa793 100644 --- a/docs/apache-airflow-providers-github/operators/index.rst +++ b/docs/apache-airflow-providers-github/operators/index.rst @@ -15,19 +15,29 @@ specific language governing permissions and limitations under the License. - - .. _howto/operator:GithubOperator: GithubOperator ================= Use the :class:`~airflow.providers.github.operators.GithubOperator` to execute -SQL commands in a `Github `__ database. +Operations in a `Github `__. + +You can build your own sensor on Repository using :class:`~airflow.providers.github.operators.GithubOperator`, +and passing **github_method** and **github_method_args** from top level PyGithub methods. +You can further process the result using **result_processor** Callable as you like. + +An example of Listing all Repositories owned by a user, `client.get_user().get_repos()` can be implemented as following: + +.. exampleinclude:: /../../airflow/providers/github/example_dags/example_github.py + :language: python + :start-after: [START howto_operator_list_repos_github] + :end-before: [END howto_operator_list_repos_github] + -An example of running the query using the operator: +An example of Listing Tags in a Repository, `client.get_repo(full_name_or_id='apache/airflow').get_tags()` can be implemented as following: -.. exampleinclude:: /../../airflow/providers/github/example_dags/example_github_query.py +.. exampleinclude:: /../../airflow/providers/github/example_dags/example_github.py :language: python - :start-after: [START howto_operator_github] - :end-before: [END howto_operator_github] + :start-after: [START howto_operator_list_tags_github] + :end-before: [END howto_operator_list_tags_github] diff --git a/docs/apache-airflow-providers-influxdb/connections/influxdb.rst b/docs/apache-airflow-providers-influxdb/connections/influxdb.rst index 669bef7490a27..336dacb163991 100644 --- a/docs/apache-airflow-providers-influxdb/connections/influxdb.rst +++ b/docs/apache-airflow-providers-influxdb/connections/influxdb.rst @@ -16,34 +16,21 @@ specific language governing permissions and limitations under the License. -.. _howto/connection:influxdb: +.. _howto/operator: Github Sensors: -InfluxDB Connection -==================== -The InfluxDB connection type provides connection to a InfluxDB database. +You can build your own sensor on Repository using :class:`~airflow.providers.github.sensors.BaseGithubRepositorySensor`, +an example of this is :class:`~airflow.providers.github.sensors.GithubTagSensor` -Configuring the Connection --------------------------- -Host (required) - The host to connect to. +GithubTagSensor +================= -Extra (required) - Specify the extra parameters (as json dictionary) that can be used in InfluxDB - connection. +Use the :class:`~airflow.providers.github.sensors.GithubTagSensor` to wait for creation of +a Tag in `Github `__. - The following extras are required: +An example of waiting for **v1.0**: - - token - Create token - https://docs.influxdata.com/influxdb/cloud/security/tokens/create-token/ - - org_name - Create organization - https://docs.influxdata.com/influxdb/cloud/reference/cli/influx/org/create/ +.. exampleinclude:: /../../airflow/providers/github/example_dags/example_github.py + :language: python + :start-after: [START howto_tag_sensor_github] + :end-before: [END howto_tag_sensor_github] - * ``token``: Create token using the influxdb cli or UI - * ``org_name``: Create org name using influxdb cli or UI - - Example "extras" field: - - .. code-block:: JSON - - { - "token": "343434343423234234234343434", - "org_name": "Test" - } From 3868ab2a025ed60235bd2116e7e5942c95df549c Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Wed, 26 Jan 2022 15:24:15 +0000 Subject: [PATCH 17/22] 20300 - adding docs for sensors --- airflow/providers/github/operators/github.py | 1 + airflow/providers/github/sensors/github.py | 1 + docs/apache-airflow-providers-github/operators/index.rst | 9 +++++---- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/airflow/providers/github/operators/github.py b/airflow/providers/github/operators/github.py index 27a276d92c6ae..3504516751f95 100644 --- a/airflow/providers/github/operators/github.py +++ b/airflow/providers/github/operators/github.py @@ -15,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + from typing import Any, Callable, Optional from github import GithubException diff --git a/airflow/providers/github/sensors/github.py b/airflow/providers/github/sensors/github.py index cbd21deb1aab0..fed6475f549dd 100644 --- a/airflow/providers/github/sensors/github.py +++ b/airflow/providers/github/sensors/github.py @@ -15,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + from typing import Any, Callable, Optional from github import GithubException diff --git a/docs/apache-airflow-providers-github/operators/index.rst b/docs/apache-airflow-providers-github/operators/index.rst index d005b3c1aa793..0878ed37c438e 100644 --- a/docs/apache-airflow-providers-github/operators/index.rst +++ b/docs/apache-airflow-providers-github/operators/index.rst @@ -23,11 +23,11 @@ GithubOperator Use the :class:`~airflow.providers.github.operators.GithubOperator` to execute Operations in a `Github `__. -You can build your own sensor on Repository using :class:`~airflow.providers.github.operators.GithubOperator`, -and passing **github_method** and **github_method_args** from top level PyGithub methods. +You can build your own operator using :class:`~airflow.providers.github.operators.GithubOperator` +and passing **github_method** and **github_method_args** from top level `PyGithub `__ methods. You can further process the result using **result_processor** Callable as you like. -An example of Listing all Repositories owned by a user, `client.get_user().get_repos()` can be implemented as following: +An example of Listing all Repositories owned by a user, **client.get_user().get_repos()** can be implemented as following: .. exampleinclude:: /../../airflow/providers/github/example_dags/example_github.py :language: python @@ -35,7 +35,8 @@ An example of Listing all Repositories owned by a user, `client.get_user().get_r :end-before: [END howto_operator_list_repos_github] -An example of Listing Tags in a Repository, `client.get_repo(full_name_or_id='apache/airflow').get_tags()` can be implemented as following: + +An example of Listing Tags in a Repository, **client.get_repo(full_name_or_id='apache/airflow').get_tags()** can be implemented as following: .. exampleinclude:: /../../airflow/providers/github/example_dags/example_github.py :language: python From 0c38beb6ceefbaaa4aae73e608b71f300fb1b8ca Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Wed, 26 Jan 2022 15:35:15 +0000 Subject: [PATCH 18/22] 20300 - adding docs done --- airflow/providers/github/hooks/github.py | 1 - .../operators/index.rst | 25 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/airflow/providers/github/hooks/github.py b/airflow/providers/github/hooks/github.py index 8df51e3306c17..5338f495f835c 100644 --- a/airflow/providers/github/hooks/github.py +++ b/airflow/providers/github/hooks/github.py @@ -17,7 +17,6 @@ # under the License. """This module allows to connect to a Github.""" -import logging from typing import Dict, Optional from github import Github as GithubClient diff --git a/docs/apache-airflow-providers-github/operators/index.rst b/docs/apache-airflow-providers-github/operators/index.rst index 0878ed37c438e..b949a73460e4e 100644 --- a/docs/apache-airflow-providers-github/operators/index.rst +++ b/docs/apache-airflow-providers-github/operators/index.rst @@ -17,8 +17,8 @@ .. _howto/operator:GithubOperator: -GithubOperator -================= +Operators +========= Use the :class:`~airflow.providers.github.operators.GithubOperator` to execute Operations in a `Github `__. @@ -42,3 +42,24 @@ An example of Listing Tags in a Repository, **client.get_repo(full_name_or_id='a :language: python :start-after: [START howto_operator_list_tags_github] :end-before: [END howto_operator_list_tags_github] + + +Sensors +======= + +You can build your own sensor using :class:`~airflow.providers.github.sensors.GithubSensor`, + +You can also implement your own sensor on Repository using :class:`~airflow.providers.github.sensors.BaseGithubRepositorySensor`, +an example of this is :class:`~airflow.providers.github.sensors.GithubTagSensor` + + +Use the :class:`~airflow.providers.github.sensors.GithubTagSensor` to wait for creation of +a Tag in `Github `__. + +An example for tag **v1.0**: + +.. exampleinclude:: /../../airflow/providers/github/example_dags/example_github.py + :language: python + :start-after: [START howto_tag_sensor_github] + :end-before: [END howto_tag_sensor_github] + From a5071f3408ab5ffbbb57638e6696648a0eb2313e Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Wed, 26 Jan 2022 16:48:33 +0000 Subject: [PATCH 19/22] 20300 - fixing unintented docs changes --- .../connections/influxdb.rst | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/docs/apache-airflow-providers-influxdb/connections/influxdb.rst b/docs/apache-airflow-providers-influxdb/connections/influxdb.rst index 336dacb163991..669bef7490a27 100644 --- a/docs/apache-airflow-providers-influxdb/connections/influxdb.rst +++ b/docs/apache-airflow-providers-influxdb/connections/influxdb.rst @@ -16,21 +16,34 @@ specific language governing permissions and limitations under the License. -.. _howto/operator: Github Sensors: +.. _howto/connection:influxdb: -You can build your own sensor on Repository using :class:`~airflow.providers.github.sensors.BaseGithubRepositorySensor`, -an example of this is :class:`~airflow.providers.github.sensors.GithubTagSensor` +InfluxDB Connection +==================== +The InfluxDB connection type provides connection to a InfluxDB database. -GithubTagSensor -================= +Configuring the Connection +-------------------------- +Host (required) + The host to connect to. -Use the :class:`~airflow.providers.github.sensors.GithubTagSensor` to wait for creation of -a Tag in `Github `__. +Extra (required) + Specify the extra parameters (as json dictionary) that can be used in InfluxDB + connection. -An example of waiting for **v1.0**: + The following extras are required: -.. exampleinclude:: /../../airflow/providers/github/example_dags/example_github.py - :language: python - :start-after: [START howto_tag_sensor_github] - :end-before: [END howto_tag_sensor_github] + - token - Create token - https://docs.influxdata.com/influxdb/cloud/security/tokens/create-token/ + - org_name - Create organization - https://docs.influxdata.com/influxdb/cloud/reference/cli/influx/org/create/ + * ``token``: Create token using the influxdb cli or UI + * ``org_name``: Create org name using influxdb cli or UI + + Example "extras" field: + + .. code-block:: JSON + + { + "token": "343434343423234234234343434", + "org_name": "Test" + } From 8401865c28d8688b73f0a263c66d41cb17289cbf Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Thu, 27 Jan 2022 18:32:23 +0000 Subject: [PATCH 20/22] 20300 - more information added in docs --- .../github/example_dags/example_github.py | 35 ++++++++++++++++++- .../operators/index.rst | 6 ++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/airflow/providers/github/example_dags/example_github.py b/airflow/providers/github/example_dags/example_github.py index 48d5e849ccc3a..2b0769e061a9e 100644 --- a/airflow/providers/github/example_dags/example_github.py +++ b/airflow/providers/github/example_dags/example_github.py @@ -16,10 +16,14 @@ # under the License. import logging from datetime import datetime +from typing import Any, Optional +from github import GithubException + +from airflow import AirflowException from airflow.models.dag import DAG from airflow.providers.github.operators.github import GithubOperator -from airflow.providers.github.sensors.github import GithubTagSensor +from airflow.providers.github.sensors.github import GithubTagSensor, GithubSensor dag = DAG( 'example_github_operator', @@ -39,8 +43,37 @@ dag=dag, ) + # [END howto_tag_sensor_github] +# [START howto_sensor_github] + +def tag_checker(repo: Any, tag_name: str) -> Optional[bool]: + result = None + try: + if repo is not None and tag_name is not None: + all_tags = [x.name for x in repo.get_tags()] + result = tag_name in all_tags + + except GithubException as github_error: + raise AirflowException(f"Failed to execute GithubSensor, error: {str(github_error)}") + except Exception as e: + raise AirflowException(f"Github operator error: {str(e)}") + return result + + +github_sensor = GithubSensor( + task_id='example_sensor', + method_name="get_repo", + method_params={'full_name_or_id': "apache/airflow"}, + result_processor=lambda repo: tag_checker(repo, 'v1.0'), + timeout=60, + poke_interval=10, + dag=dag, +) + +# [END howto_sensor_github] + # [START howto_operator_list_repos_github] diff --git a/docs/apache-airflow-providers-github/operators/index.rst b/docs/apache-airflow-providers-github/operators/index.rst index b949a73460e4e..c784552d5a72c 100644 --- a/docs/apache-airflow-providers-github/operators/index.rst +++ b/docs/apache-airflow-providers-github/operators/index.rst @@ -63,3 +63,9 @@ An example for tag **v1.0**: :start-after: [START howto_tag_sensor_github] :end-before: [END howto_tag_sensor_github] +Similar Functionality can be achieved by directly using :class:`~airflow.providers.github.sensors.GithubSensor` , + +.. exampleinclude:: /../../airflow/providers/github/example_dags/example_github.py + :language: python + :start-after: [START howto_sensor_github] + :end-before: [END howto_sensor_github] From aa35f3321476d53a8149dbdfdde572ec377e7a03 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Thu, 27 Jan 2022 18:54:41 +0000 Subject: [PATCH 21/22] 20300 - fixing static checks error --- CONTRIBUTING.rst | 14 +++++++------- INSTALL | 14 +++++++------- .../github/example_dags/example_github.py | 3 ++- airflow/providers/github/provider.yaml | 5 +++++ 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 142ac8d77b660..dffc27b73d60a 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -609,13 +609,13 @@ apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, a apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol, -facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, -http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure, -microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, -opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds, -qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, -snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino, vertica, virtualenv, webhdfs, -winrm, yandex, zendesk +facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, +hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb, +microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, +openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, +qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, +slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino, vertica, virtualenv, +webhdfs, winrm, yandex, zendesk .. END EXTRAS HERE diff --git a/INSTALL b/INSTALL index 21ac12496b048..72bb4114cfe35 100644 --- a/INSTALL +++ b/INSTALL @@ -100,13 +100,13 @@ apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, a apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol, -facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, -http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure, -microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, -opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds, -qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, -snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino, vertica, virtualenv, webhdfs, -winrm, yandex, zendesk +facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, +hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb, +microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, +openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, +qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, +slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino, vertica, virtualenv, +webhdfs, winrm, yandex, zendesk # END EXTRAS HERE diff --git a/airflow/providers/github/example_dags/example_github.py b/airflow/providers/github/example_dags/example_github.py index 2b0769e061a9e..7f3fd8578b839 100644 --- a/airflow/providers/github/example_dags/example_github.py +++ b/airflow/providers/github/example_dags/example_github.py @@ -23,7 +23,7 @@ from airflow import AirflowException from airflow.models.dag import DAG from airflow.providers.github.operators.github import GithubOperator -from airflow.providers.github.sensors.github import GithubTagSensor, GithubSensor +from airflow.providers.github.sensors.github import GithubSensor, GithubTagSensor dag = DAG( 'example_github_operator', @@ -48,6 +48,7 @@ # [START howto_sensor_github] + def tag_checker(repo: Any, tag_name: str) -> Optional[bool]: result = None try: diff --git a/airflow/providers/github/provider.yaml b/airflow/providers/github/provider.yaml index 3ddf4c21ce5c3..fcdc1b2d3bb61 100644 --- a/airflow/providers/github/provider.yaml +++ b/airflow/providers/github/provider.yaml @@ -37,6 +37,11 @@ operators: python-modules: - airflow.providers.github.operators.github +sensors: + - integration-name: Github + python-modules: + - airflow.providers.github.sensors.github + connection-types: - hook-class-name: airflow.providers.github.hooks.github.GithubHook connection-type: github From 8369d9f35f111f1e0aff3b91d194b7cdcdb91223 Mon Sep 17 00:00:00 2001 From: Ashish Patel Date: Fri, 28 Jan 2022 12:14:18 +0000 Subject: [PATCH 22/22] 20300 - fixing static checks error --- airflow/providers/github/operators/github.py | 8 +++++--- airflow/providers/github/sensors/github.py | 12 +++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/airflow/providers/github/operators/github.py b/airflow/providers/github/operators/github.py index 3504516751f95..170b9a8ae7b82 100644 --- a/airflow/providers/github/operators/github.py +++ b/airflow/providers/github/operators/github.py @@ -16,14 +16,16 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, Callable, Optional +from typing import TYPE_CHECKING, Any, Callable, Optional from github import GithubException from airflow import AirflowException from airflow.models import BaseOperator from airflow.providers.github.hooks.github import GithubHook -from airflow.utils.context import Context + +if TYPE_CHECKING: + from airflow.utils.context import Context class GithubOperator(BaseOperator): @@ -62,7 +64,7 @@ def __init__( self.github_method_args = github_method_args self.result_processor = result_processor - def execute(self, context: Context) -> Any: + def execute(self, context: 'Context') -> Any: try: # Default method execution is on the top level GitHub client hook = GithubHook(github_conn_id=self.github_conn_id) diff --git a/airflow/providers/github/sensors/github.py b/airflow/providers/github/sensors/github.py index fed6475f549dd..3d0265e23d003 100644 --- a/airflow/providers/github/sensors/github.py +++ b/airflow/providers/github/sensors/github.py @@ -16,14 +16,16 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, Callable, Optional +from typing import TYPE_CHECKING, Any, Callable, Optional from github import GithubException from airflow import AirflowException from airflow.providers.github.operators.github import GithubOperator from airflow.sensors.base import BaseSensorOperator -from airflow.utils.context import Context + +if TYPE_CHECKING: + from airflow.utils.context import Context class GithubSensor(BaseSensorOperator): @@ -64,7 +66,7 @@ def __init__( result_processor=self.result_processor, ) - def poke(self, context: Context) -> bool: + def poke(self, context: 'Context') -> bool: return self.github_operator.execute(context=context) @@ -94,7 +96,7 @@ def __init__( **kwargs, ) - def poke(self, context: Context) -> bool: + def poke(self, context: 'Context') -> bool: """ Function that the sensors defined while deriving this class should override. @@ -133,7 +135,7 @@ def __init__( **kwargs, ) - def poke(self, context: Context) -> bool: + def poke(self, context: 'Context') -> bool: self.log.info('Poking for tag: %s in repository: %s', self.tag_name, self.repository_name) return GithubSensor.poke(self, context=context)