diff --git a/.github/workflows/ci-k8s.yml b/.github/workflows/ci-k8s.yml new file mode 100644 index 0000000000000..da60ad3ff7dbc --- /dev/null +++ b/.github/workflows/ci-k8s.yml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: "k8s tests" +on: [pull_request, push] + +jobs: + create-airflow-docker-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + name: checkout + - name: pip install airflow + run: pip install -e .[devel] + - name: build docker 1image + run: ./breeze build-image --production-image + launch-airflow-on-k8s: + runs-on: ubuntu-latest + strategy: + matrix: + k8s-version: [v1.15.7, v1.16.4, v1.17.2, v1.18.0] + steps: + - uses: actions/checkout@v2 + name: checkout + - name: create k8s Kind Cluster + uses: helm/kind-action@v1.0.0-rc.1 + with: + cluster_name: kind + node_image: kindest/node:${{ matrix.k8s-version }} + config: ./kind-cluster-conf.yaml + - name: test cluster running + run: | + kubectl cluster-info + kubectl get pods -n kube-system + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: '3.7' + - name: download helm + run: | + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh + - name: create namespace + run: kubectl create namespace airflow + - name: pull astronomer helm chart + run: | + helm repo add astronomer https://helm.astronomer.io + helm install airflow --namespace airflow astronomer/airflow \ + --set env[0].name=AIRFLOW__CORE__LOAD_EXAMPLES \ + --set env[0].value=True + kubectl get pods --namespace airflow + kubectl get svc --namespace airflow + kubectl port-forward --namespace airflow svc/airflow-webserver 8080:8080 & + - name: pip install airflow + run: pip install -e .[devel] + - name: run k8sexecutor tests + run: | + pytest tests/runtime/kubernetes/test_kubernetes_executor.py + env: + RUNTIME: kubernetes + - name: run k8sPodOperator tests + run: pytest tests/runtime/kubernetes/test_kubernetes_pod_operator.py + env: + RUNTIME: kubernetes diff --git a/scripts/ci/in_container/kubernetes/kind-cluster-conf.yaml b/.github/workflows/kind-cluster-conf.yaml similarity index 100% rename from scripts/ci/in_container/kubernetes/kind-cluster-conf.yaml rename to .github/workflows/kind-cluster-conf.yaml diff --git a/.travis.yml b/.travis.yml index 46f69ff32fda8..6f5fa40ea3a6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,26 +55,6 @@ jobs: PYTHON_MAJOR_MINOR_VERSION=3.6 stage: test script: ./scripts/ci/ci_docs.sh - - name: "Tests [Py3.6][Kubernetes][persistent]" - env: >- - BACKEND=postgres - PYTHON_MAJOR_MINOR_VERSION=3.6 - RUNTIME=kubernetes - ENABLE_KIND_CLUSTER=true - KUBERNETES_MODE=persistent_mode - KUBERNETES_VERSION=v1.15.3 - python: "3.6" - stage: test - - name: "Tests [Py3.7][Kubernetes][git]" - env: >- - BACKEND=postgres - PYTHON_MAJOR_MINOR_VERSION=3.7 - RUNTIME=kubernetes - ENABLE_KIND_CLUSTER=true - KUBERNETES_MODE=git_mode - KUBERNETES_VERSION=v1.15.3 - python: "3.6" - stage: test - name: "Tests [Postgres9.6][Py3.6][integrations]" env: >- BACKEND=postgres diff --git a/Dockerfile b/Dockerfile index 093fbf72be74b..8440e035ae5af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -328,5 +328,6 @@ WORKDIR ${AIRFLOW_HOME} ENV AIRFLOW__CORE__LOAD_EXAMPLES="false" +COPY scripts/include/clean-logs.sh /usr/local/bin/clean-airflow-logs ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] CMD ["airflow", "--help"] diff --git a/airflow/cli/cli_parser.py b/airflow/cli/cli_parser.py index d66d8a8f0ba64..77c10e9988632 100644 --- a/airflow/cli/cli_parser.py +++ b/airflow/cli/cli_parser.py @@ -76,7 +76,7 @@ def error(self, message): class Arg: """Class to keep information about command line argument""" # pylint: disable=redefined-builtin - def __init__(self, flags=None, help=None, action=None, default=None, nargs=None, + def __init__(self, flags=None, help=None, action=None, default=argparse.SUPPRESS, nargs=None, type=None, choices=None, required=None, metavar=None): self.flags = flags self.help = help @@ -434,6 +434,12 @@ def __init__(self, flags=None, help=None, action=None, default=None, nargs=None, ARG_CFG_PATH = Arg( ("--cfg-path",), help="Path to config file to use instead of airflow.cfg") +ARG_MIGRATION_TIMEOUT = Arg( + ("-t", "--migration-wait-timeout"), + help="timeout to wait for db to migrate ", + type=int, + default=0, + ) # webserver ARG_PORT = Arg( @@ -952,6 +958,12 @@ class GroupCommand(NamedTuple): func=lazy_load_command('airflow.cli.commands.db_command.initdb'), args=(), ), + ActionCommand( + name="wait-for-migrations", + help="wait until migration has finished (or until timeout", + func=lazy_load_command('airflow.cli.commands.db_command.wait_for_migrations'), + args=(ARG_MIGRATION_TIMEOUT,), + ), ActionCommand( name='reset', help="Burn down and rebuild the metadata database", @@ -1242,7 +1254,7 @@ def _add_command( def _add_action_command(sub: ActionCommand, sub_proc: argparse.ArgumentParser) -> None: for arg in _sort_args(sub.args): kwargs = { - k: v for k, v in vars(arg).items() if k != 'flags' and v + k: v for k, v in vars(arg).items() if k != 'flags' } sub_proc.add_argument(*arg.flags, **kwargs) sub_proc.set_defaults(func=sub.func) diff --git a/airflow/cli/commands/db_command.py b/airflow/cli/commands/db_command.py index 6779d419cb394..af916835b8a0e 100644 --- a/airflow/cli/commands/db_command.py +++ b/airflow/cli/commands/db_command.py @@ -15,14 +15,22 @@ # specific language governing permissions and limitations # under the License. """Database sub-commands""" -import os import textwrap from tempfile import NamedTemporaryFile -from airflow import settings from airflow.exceptions import AirflowException from airflow.utils import cli as cli_utils, db from airflow.utils.process_utils import execute_interactive +import importlib +import logging +import os +import time + +from alembic.config import Config +from alembic.runtime.migration import MigrationContext +from alembic.script import ScriptDirectory + +from airflow import settings, version def initdb(args): @@ -49,6 +57,14 @@ def upgradedb(args): print("DB: " + repr(settings.engine.url)) db.upgradedb() +@cli_utils.action_logging +def wait_for_migrations(args): + # """ + # Function to wait for all airflow migrations to complete. Used for launching airflow in k8s + # @param timeout: + # @return: + # """ + db.wait_for_migrations(timeout=args.migration_wait_timeout) @cli_utils.action_logging def shell(args): diff --git a/airflow/utils/db.py b/airflow/utils/db.py index 7b58e71fc2993..5f1a979461843 100644 --- a/airflow/utils/db.py +++ b/airflow/utils/db.py @@ -20,6 +20,20 @@ from sqlalchemy import Table +import textwrap +from tempfile import NamedTemporaryFile + +from airflow.exceptions import AirflowException +from airflow.utils import cli as cli_utils, db +from airflow.utils.process_utils import execute_interactive +import importlib +import logging +import os +import time + +from alembic.config import Config +from alembic.runtime.migration import MigrationContext +from alembic.script import ScriptDirectory from airflow import settings from airflow.configuration import conf # noinspection PyUnresolvedReferences @@ -543,6 +557,34 @@ def initdb(): from flask_appbuilder.models.sqla import Base Base.metadata.create_all(settings.engine) # pylint: disable=no-member +def wait_for_migrations(timeout): + """ + Function to wait for all airflow migrations to complete. Used for launching airflow in k8s + @param timeout: + @return: + """ + package_dir = os.path.dirname(importlib.util.find_spec('airflow').origin) + directory = os.path.join(package_dir, 'migrations') + config = Config(os.path.join(package_dir, 'alembic.ini')) + config.set_main_option('script_location', directory) + config.set_main_option('sqlalchemy.url', settings.SQL_ALCHEMY_CONN) + script_ = ScriptDirectory.from_config(config) + with settings.engine.connect() as connection: + context = MigrationContext.configure(connection) + ticker = 0 + while True: + source_heads = set(script_.get_heads()) + db_heads = set(context.get_current_heads()) + if source_heads == db_heads: + logging.info('Airflow version: %s', version.version) + logging.info('Current heads: %s', db_heads) + break + if ticker >= timeout: + raise TimeoutError("There are still unapplied migrations after {} " + "seconds".format(ticker, timeout)) + ticker += 1 + time.sleep(1) + logging.info('Waiting for migrations... %s second(s)', ticker) def upgradedb(): """ diff --git a/scripts/ci/in_container/kubernetes/app/secrets.yaml b/kind-cluster-conf.yaml similarity index 64% rename from scripts/ci/in_container/kubernetes/app/secrets.yaml rename to kind-cluster-conf.yaml index fa8ae164ad380..c7a44337b1d19 100644 --- a/scripts/ci/in_container/kubernetes/app/secrets.yaml +++ b/kind-cluster-conf.yaml @@ -15,12 +15,22 @@ # specific language governing permissions and limitations # under the License. --- -apiVersion: v1 -kind: Secret -metadata: - name: airflow-secrets -type: Opaque -data: - # The sql_alchemy_conn value is a base64 encoded representation of this connection string: - # postgresql+psycopg2://root:root@postgres-airflow:5432/airflow - sql_alchemy_conn: cG9zdGdyZXNxbCtwc3ljb3BnMjovL3Jvb3Q6cm9vdEBwb3N0Z3Jlcy1haXJmbG93OjU0MzIvYWlyZmxvdwo= +kind: Cluster +apiVersion: kind.sigs.k8s.io/v1alpha3 +networking: + apiServerAddress: 0.0.0.0 + apiServerPort: 19090 +nodes: + - role: control-plane + - role: worker + extraPortMappings: + - containerPort: 30809 + hostPort: 30809 +kubeadmConfigPatchesJson6902: + - group: kubeadm.k8s.io + version: v1beta2 + kind: ClusterConfiguration + patch: | + - op: add + path: /apiServer/certSANs/- + value: docker diff --git a/scripts/ci/ci_run_airflow_testing.sh b/scripts/ci/ci_run_airflow_testing.sh index 7cb962adb782c..df3cdc749df77 100755 --- a/scripts/ci/ci_run_airflow_testing.sh +++ b/scripts/ci/ci_run_airflow_testing.sh @@ -78,37 +78,3 @@ do done RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:=""} - -if [[ ${RUNTIME:=} == "kubernetes" ]]; then - export KUBERNETES_MODE=${KUBERNETES_MODE:="git_mode"} - export KUBERNETES_VERSION=${KUBERNETES_VERSION:="v1.15.3"} - - set +u - # shellcheck disable=SC2016 - docker-compose --log-level INFO \ - -f "${MY_DIR}/docker-compose/base.yml" \ - -f "${MY_DIR}/docker-compose/backend-${BACKEND}.yml" \ - -f "${MY_DIR}/docker-compose/runtime-kubernetes.yml" \ - "${INTEGRATIONS[@]}" \ - "${DOCKER_COMPOSE_LOCAL[@]}" \ - run airflow \ - '/opt/airflow/scripts/ci/in_container/entrypoint_ci.sh "${@}"' \ - /opt/airflow/scripts/ci/in_container/entrypoint_ci.sh "${@}" - # Note the command is there twice (!) because it is passed via bash -c - # and bash -c starts passing parameters from $0. TODO: fixme - set -u -else - set +u - # shellcheck disable=SC2016 - docker-compose --log-level INFO \ - -f "${MY_DIR}/docker-compose/base.yml" \ - -f "${MY_DIR}/docker-compose/backend-${BACKEND}.yml" \ - "${INTEGRATIONS[@]}" \ - "${DOCKER_COMPOSE_LOCAL[@]}" \ - run airflow \ - '/opt/airflow/scripts/ci/in_container/entrypoint_ci.sh "${@}"' \ - /opt/airflow/scripts/ci/in_container/entrypoint_ci.sh "${@}" - # Note the command is there twice (!) because it is passed via bash -c - # and bash -c starts passing parameters from $0. TODO: fixme - set -u -fi diff --git a/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh b/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh deleted file mode 100755 index a124fba263cc4..0000000000000 --- a/scripts/ci/in_container/deploy_airflow_to_kubernetes.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# Script to run flake8 on all code. Can be started from any working directory -# shellcheck source=scripts/ci/in_container/_in_container_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh" - -"${MY_DIR}/kubernetes/docker/rebuild_airflow_image.sh" -"${MY_DIR}/kubernetes/app/deploy_app.sh" diff --git a/scripts/ci/in_container/kubernetes/app/deploy_app.sh b/scripts/ci/in_container/kubernetes/app/deploy_app.sh deleted file mode 100755 index 5ffeb61a1add6..0000000000000 --- a/scripts/ci/in_container/kubernetes/app/deploy_app.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -MY_DIR=$(cd "$(dirname "$0")" && pwd) - -AIRFLOW_SOURCES=$( - cd "${MY_DIR}/../../../../../" || exit 1 - pwd -) -export AIRFLOW_SOURCES - -# We keep _utils here because we are not in the in_container directory -# shellcheck source=scripts/ci/in_container/_in_container_utils.sh -. "${MY_DIR}/../../_in_container_utils.sh" - -assert_in_container - -in_container_script_start - -function end_and_dump_logs() { - dump_logs - in_container_script_end -} - -trap in_container_script_end EXIT - -export TEMPLATE_DIRNAME="${MY_DIR}/templates" -export BUILD_DIRNAME="${MY_DIR}/build" - -# shellcheck source=common/_image_variables.sh -. "${AIRFLOW_SOURCES}/common/_image_variables.sh" - -# Source branch will be set in DockerHub -SOURCE_BRANCH=${SOURCE_BRANCH:=${DEFAULT_BRANCH}} -# if BRANCH_NAME is not set it will be set to either SOURCE_BRANCH (if overridden) -# or default branch for the sources -BRANCH_NAME=${BRANCH_NAME:=${SOURCE_BRANCH}} - -if [[ ! -d "${BUILD_DIRNAME}" ]]; then - mkdir -p "${BUILD_DIRNAME}" -fi - -rm -f "${BUILD_DIRNAME}"/* - -if [[ "${KUBERNETES_MODE}" == "persistent_mode" ]]; then - INIT_DAGS_VOLUME_NAME=airflow-dags - POD_AIRFLOW_DAGS_VOLUME_NAME=airflow-dags - CONFIGMAP_DAGS_FOLDER=/root/airflow/dags - CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT= - CONFIGMAP_DAGS_VOLUME_CLAIM=airflow-dags -else - INIT_DAGS_VOLUME_NAME=airflow-dags-fake - POD_AIRFLOW_DAGS_VOLUME_NAME=airflow-dags-git - CONFIGMAP_DAGS_FOLDER=/root/airflow/dags/repo/airflow/example_dags - CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT=/root/airflow/dags - CONFIGMAP_DAGS_VOLUME_CLAIM= -fi - -if [[ ${TRAVIS_PULL_REQUEST:=} != "" && ${TRAVIS_PULL_REQUEST} != "false" ]]; then - CONFIGMAP_GIT_REPO=${TRAVIS_PULL_REQUEST_SLUG} - CONFIGMAP_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH} -else - CONFIGMAP_GIT_REPO=${TRAVIS_REPO_SLUG:-apache/airflow} - CONFIGMAP_BRANCH=${TRAVIS_BRANCH:=master} -fi - - -if [[ "${KUBERNETES_MODE}" == "persistent_mode" ]]; then - sed -e "s/{{INIT_GIT_SYNC}}//g" \ - "${TEMPLATE_DIRNAME}/airflow.template.yaml" >"${BUILD_DIRNAME}/airflow.yaml" -else - sed -e "/{{INIT_GIT_SYNC}}/{r ${TEMPLATE_DIRNAME}/init_git_sync.template.yaml" -e 'd}' \ - "${TEMPLATE_DIRNAME}/airflow.template.yaml" >"${BUILD_DIRNAME}/airflow.yaml" -fi -sed -i "s|{{AIRFLOW_KUBERNETES_IMAGE}}|${AIRFLOW_KUBERNETES_IMAGE}|g" "${BUILD_DIRNAME}/airflow.yaml" - -sed -i "s|{{CONFIGMAP_GIT_REPO}}|${CONFIGMAP_GIT_REPO}|g" "${BUILD_DIRNAME}/airflow.yaml" -sed -i "s|{{CONFIGMAP_BRANCH}}|${CONFIGMAP_BRANCH}|g" "${BUILD_DIRNAME}/airflow.yaml" -sed -i "s|{{INIT_DAGS_VOLUME_NAME}}|${INIT_DAGS_VOLUME_NAME}|g" "${BUILD_DIRNAME}/airflow.yaml" -sed -i "s|{{POD_AIRFLOW_DAGS_VOLUME_NAME}}|${POD_AIRFLOW_DAGS_VOLUME_NAME}|g" \ - "${BUILD_DIRNAME}/airflow.yaml" - -sed "s|{{CONFIGMAP_DAGS_FOLDER}}|${CONFIGMAP_DAGS_FOLDER}|g" \ - "${TEMPLATE_DIRNAME}/configmaps.template.yaml" >"${BUILD_DIRNAME}/configmaps.yaml" -sed -i "s|{{CONFIGMAP_GIT_REPO}}|${CONFIGMAP_GIT_REPO}|g" "${BUILD_DIRNAME}/configmaps.yaml" -sed -i "s|{{CONFIGMAP_BRANCH}}|${CONFIGMAP_BRANCH}|g" "${BUILD_DIRNAME}/configmaps.yaml" -sed -i "s|{{CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT}}|${CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT}|g" \ - "${BUILD_DIRNAME}/configmaps.yaml" -sed -i "s|{{CONFIGMAP_DAGS_VOLUME_CLAIM}}|${CONFIGMAP_DAGS_VOLUME_CLAIM}|g" \ - "${BUILD_DIRNAME}/configmaps.yaml" -sed -i "s|{{AIRFLOW_KUBERNETES_IMAGE_NAME}}|${AIRFLOW_KUBERNETES_IMAGE_NAME}|g" \ - "${BUILD_DIRNAME}/configmaps.yaml" -sed -i "s|{{AIRFLOW_KUBERNETES_IMAGE_TAG}}|${AIRFLOW_KUBERNETES_IMAGE_TAG}|g" \ - "${BUILD_DIRNAME}/configmaps.yaml" - -cat "${BUILD_DIRNAME}/airflow.yaml" -cat "${BUILD_DIRNAME}/configmaps.yaml" - -kubectl delete -f "${MY_DIR}/postgres.yaml" || true -kubectl delete -f "${BUILD_DIRNAME}/airflow.yaml" || true -kubectl delete -f "${MY_DIR}/secrets.yaml" || true - -set -e - -kubectl apply -f "${MY_DIR}/secrets.yaml" -kubectl apply -f "${BUILD_DIRNAME}/configmaps.yaml" -kubectl apply -f "${MY_DIR}/postgres.yaml" -kubectl apply -f "${MY_DIR}/volumes.yaml" -kubectl apply -f "${BUILD_DIRNAME}/airflow.yaml" - -dump_logs() { - POD=$(kubectl get pods -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | grep airflow | head -1) - echo "------- pod description -------" - kubectl describe pod "${POD}" - echo "------- webserver init container logs - init -------" - kubectl logs "${POD}" -c init || true - if [[ "${KUBERNETES_MODE}" != "persistent_mode" ]]; then - echo "------- webserver init container logs - git-sync-clone -------" - kubectl logs "${POD}" -c git-sync-clone || true - fi - echo "------- webserver logs -------" - kubectl logs "${POD}" -c webserver || true - echo "------- scheduler logs -------" - kubectl logs "${POD}" -c scheduler || true - echo "--------------" -} - -set +x -set +o pipefail -# wait for up to 10 minutes for everything to be deployed -PODS_ARE_READY="0" -for i in {1..150}; do - echo "------- Running kubectl get pods: $i -------" - PODS=$(kubectl get pods | awk 'NR>1 {print $0}') - echo "$PODS" - NUM_AIRFLOW_READY=$(echo "${PODS}" | grep airflow | awk '{print $2}' | grep -cE '([0-9])\/(\1)' | xargs) - NUM_POSTGRES_READY=$(echo "${PODS}" | grep postgres | awk '{print $2}' | grep -cE '([0-9])\/(\1)' | xargs) - if [[ "${NUM_AIRFLOW_READY}" == "1" && "${NUM_POSTGRES_READY}" == "1" ]]; then - PODS_ARE_READY="1" - break - fi - sleep 4 -done -POD=$(kubectl get pods -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' | grep airflow | head -1) - -if [[ "${PODS_ARE_READY}" == "1" ]]; then - echo "PODS are ready." -else - echo >&2 "PODS are not ready after waiting for a long time. Exiting..." - dump_logs - exit 1 -fi - -# Wait until Airflow webserver is up -KUBERNETES_HOST=${CLUSTER_NAME}-worker -AIRFLOW_WEBSERVER_IS_READY="0" -CONSECUTIVE_SUCCESS_CALLS=0 -for i in {1..30}; do - echo "------- Wait until webserver is up: $i -------" - PODS=$(kubectl get pods | awk 'NR>1 {print $0}') - echo "$PODS" - HTTP_CODE=$(curl -LI "http://${KUBERNETES_HOST}:30809/health" -o /dev/null -w '%{http_code}\n' -sS) || true - if [[ "${HTTP_CODE}" == 200 ]]; then - ((CONSECUTIVE_SUCCESS_CALLS += 1)) - else - CONSECUTIVE_SUCCESS_CALLS="0" - fi - if [[ "${CONSECUTIVE_SUCCESS_CALLS}" == 3 ]]; then - AIRFLOW_WEBSERVER_IS_READY="1" - break - fi - sleep 10 -done -set -o pipefail - -if [[ "${AIRFLOW_WEBSERVER_IS_READY}" == "1" ]]; then - echo "Airflow webserver is ready." -else - echo >&2 "Airflow webserver is not ready after waiting for a long time. Exiting..." - exit 1 -fi diff --git a/scripts/ci/in_container/kubernetes/app/postgres.yaml b/scripts/ci/in_container/kubernetes/app/postgres.yaml deleted file mode 100644 index c6a4db7d8898d..0000000000000 --- a/scripts/ci/in_container/kubernetes/app/postgres.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -kind: Deployment -apiVersion: extensions/v1beta1 -metadata: - name: postgres-airflow -spec: - replicas: 1 - template: - metadata: - labels: - name: postgres-airflow - spec: - restartPolicy: Always - containers: - - name: postgres - image: postgres - imagePullPolicy: IfNotPresent - ports: - - containerPort: 5432 - protocol: TCP - volumeMounts: - - name: dbvol - mountPath: /var/lib/postgresql/data/pgdata - subPath: pgdata - env: - - name: POSTGRES_USER - value: root - - name: POSTGRES_PASSWORD - value: root - - name: POSTGRES_DB - value: airflow - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - livenessProbe: - initialDelaySeconds: 60 - timeoutSeconds: 5 - failureThreshold: 5 - exec: - command: - - /bin/sh - - -c - - > - exec pg_isready --host $POD_IP || - if [[ $(psql -qtAc --host $POD_IP 'SELECT pg_is_in_recovery') != "f" ]]; - then exit 0 else; - exit 1; fi - readinessProbe: - initialDelaySeconds: 5 - timeoutSeconds: 5 - periodSeconds: 5 - exec: - command: - - /bin/sh - - -c - - exec pg_isready --host $POD_IP - resources: - requests: - memory: .5Gi - cpu: .5 - volumes: - - name: dbvol - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: postgres-airflow -spec: - clusterIP: None - ports: - - port: 5432 - targetPort: 5432 - selector: - name: postgres-airflow diff --git a/scripts/ci/in_container/kubernetes/app/templates/airflow.template.yaml b/scripts/ci/in_container/kubernetes/app/templates/airflow.template.yaml deleted file mode 100644 index 08b5afb6bd308..0000000000000 --- a/scripts/ci/in_container/kubernetes/app/templates/airflow.template.yaml +++ /dev/null @@ -1,144 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: admin-rbac -subjects: - - kind: ServiceAccount - # Reference to upper's `metadata.name` - name: default - # Reference to upper's `metadata.namespace` - namespace: default -roleRef: - kind: ClusterRole - name: cluster-admin - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: airflow -spec: - replicas: 1 - template: - metadata: - labels: - name: airflow - spec: - initContainers: - - name: "init" - image: {{AIRFLOW_KUBERNETES_IMAGE}} - imagePullPolicy: IfNotPresent - volumeMounts: - - name: airflow-configmap - mountPath: /root/airflow/airflow.cfg - subPath: airflow.cfg - - name: {{INIT_DAGS_VOLUME_NAME}} - mountPath: /root/airflow/dags - - name: test-volume - mountPath: /root/test_volume - env: - - name: SQL_ALCHEMY_CONN - valueFrom: - secretKeyRef: - name: airflow-secrets - key: sql_alchemy_conn - command: - - "bash" - args: - - "-cx" - - "/tmp/airflow-test-env-init.sh" -{{INIT_GIT_SYNC}} - containers: - - name: webserver - image: {{AIRFLOW_KUBERNETES_IMAGE}} - imagePullPolicy: IfNotPresent - ports: - - name: webserver - containerPort: 8080 - args: ["webserver"] - env: - - name: AIRFLOW__KUBERNETES__NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SQL_ALCHEMY_CONN - valueFrom: - secretKeyRef: - name: airflow-secrets - key: sql_alchemy_conn - volumeMounts: - - name: airflow-configmap - mountPath: /root/airflow/airflow.cfg - subPath: airflow.cfg - - name: {{POD_AIRFLOW_DAGS_VOLUME_NAME}} - mountPath: /root/airflow/dags - - name: airflow-logs - mountPath: /root/airflow/logs - - name: scheduler - image: {{AIRFLOW_KUBERNETES_IMAGE}} - imagePullPolicy: IfNotPresent - args: ["scheduler"] - env: - - name: AIRFLOW__KUBERNETES__NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SQL_ALCHEMY_CONN - valueFrom: - secretKeyRef: - name: airflow-secrets - key: sql_alchemy_conn - volumeMounts: - - name: airflow-configmap - mountPath: /root/airflow/airflow.cfg - subPath: airflow.cfg - - name: {{POD_AIRFLOW_DAGS_VOLUME_NAME}} - mountPath: /root/airflow/dags - - name: airflow-logs - mountPath: /root/airflow/logs - volumes: - - name: airflow-dags - persistentVolumeClaim: - claimName: airflow-dags - - name: airflow-dags-fake - emptyDir: {} - - name: airflow-dags-git - emptyDir: {} - - name: test-volume - persistentVolumeClaim: - claimName: test-volume - - name: airflow-logs - persistentVolumeClaim: - claimName: airflow-logs - - name: airflow-configmap - configMap: - name: airflow-configmap ---- -apiVersion: v1 -kind: Service -metadata: - name: airflow -spec: - type: NodePort - ports: - - port: 8080 - nodePort: 30809 - selector: - name: airflow diff --git a/scripts/ci/in_container/kubernetes/app/templates/configmaps.template.yaml b/scripts/ci/in_container/kubernetes/app/templates/configmaps.template.yaml deleted file mode 100644 index 722150371be44..0000000000000 --- a/scripts/ci/in_container/kubernetes/app/templates/configmaps.template.yaml +++ /dev/null @@ -1,330 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: airflow-configmap -data: - # yamllint disable rule:line-length - airflow.cfg: | - [core] - dags_folder = {{CONFIGMAP_DAGS_FOLDER}} - executor = KubernetesExecutor - parallelism = 32 - load_examples = False - load_default_connections = True - plugins_folder = /root/airflow/plugins - sql_alchemy_conn = $SQL_ALCHEMY_CONN - - [logging] - base_log_folder = /root/airflow/logs - logging_level = INFO - - [scheduler] - dag_dir_list_interval = 300 - child_process_log_directory = /root/airflow/logs/scheduler - # Task instances listen for external kill signal (when you clear tasks - # from the CLI or the UI), this defines the frequency at which they should - # listen (in seconds). - job_heartbeat_sec = 5 - max_threads = 2 - - # The scheduler constantly tries to trigger new tasks (look at the - # scheduler section in the docs for more information). This defines - # how often the scheduler should run (in seconds). - scheduler_heartbeat_sec = 5 - - # after how much time a new DAGs should be picked up from the filesystem - min_file_process_interval = 0 - - statsd_on = False - statsd_host = localhost - statsd_port = 8125 - statsd_prefix = airflow - - # How many seconds to wait between file-parsing loops to prevent the logs from being spammed. - min_file_parsing_loop_time = 1 - - print_stats_interval = 30 - scheduler_zombie_task_threshold = 300 - max_tis_per_query = 0 - authenticate = False - - # Turn off scheduler catchup by setting this to False. - # Default behavior is unchanged and - # Command Line Backfills still work, but the scheduler - # will not do scheduler catchup if this is False, - # however it can be set on a per DAG basis in the - # DAG definition (catchup) - catchup_by_default = True - - [webserver] - # The base url of your website as airflow cannot guess what domain or - # cname you are using. This is used in automated emails that - # airflow sends to point links to the right web server - base_url = http://localhost:8080 - - # The ip specified when starting the web server - web_server_host = 0.0.0.0 - - # The port on which to run the web server - web_server_port = 8080 - - # Paths to the SSL certificate and key for the web server. When both are - # provided SSL will be enabled. This does not change the web server port. - web_server_ssl_cert = - web_server_ssl_key = - - # Number of seconds the webserver waits before killing gunicorn master that doesn't respond - web_server_master_timeout = 120 - - # Number of seconds the gunicorn webserver waits before timing out on a worker - web_server_worker_timeout = 120 - - # Number of workers to refresh at a time. When set to 0, worker refresh is - # disabled. When nonzero, airflow periodically refreshes webserver workers by - # bringing up new ones and killing old ones. - worker_refresh_batch_size = 1 - - # Number of seconds to wait before refreshing a batch of workers. - worker_refresh_interval = 30 - - # Secret key used to run your flask app - secret_key = temporary_key - - # Number of workers to run the Gunicorn web server - workers = 4 - - # The worker class gunicorn should use. Choices include - # sync (default), eventlet, gevent - worker_class = sync - - # Log files for the gunicorn webserver. '-' means log to stderr. - access_logfile = - - error_logfile = - - - # Expose the configuration file in the web server - expose_config = False - - # Default DAG view. Valid values are: - # tree, graph, duration, gantt, landing_times - dag_default_view = tree - - # Default DAG orientation. Valid values are: - # LR (Left->Right), TB (Top->Bottom), RL (Right->Left), BT (Bottom->Top) - dag_orientation = LR - - # Puts the webserver in demonstration mode; blurs the names of Operators for - # privacy. - demo_mode = False - - # The amount of time (in secs) webserver will wait for initial handshake - # while fetching logs from other worker machine - log_fetch_timeout_sec = 5 - - # By default, the webserver shows paused DAGs. Flip this to hide paused - # DAGs by default - hide_paused_dags_by_default = False - - # Consistent page size across all listing views in the UI - page_size = 100 - - [smtp] - # If you want airflow to send emails on retries, failure, and you want to use - # the airflow.utils.email.send_email_smtp function, you have to configure an - # smtp server here - smtp_host = localhost - smtp_starttls = True - smtp_ssl = False - # Uncomment and set the user/pass settings if you want to use SMTP AUTH - # smtp_user = airflow - # smtp_password = airflow - smtp_port = 25 - smtp_mail_from = airflow@example.com - - [kubernetes] - airflow_configmap = airflow-configmap - worker_container_repository = {{AIRFLOW_KUBERNETES_IMAGE_NAME}} - worker_container_tag = {{AIRFLOW_KUBERNETES_IMAGE_TAG}} - worker_container_image_pull_policy = IfNotPresent - delete_worker_pods = True - dags_in_image = False - git_repo = https://github.com/{{CONFIGMAP_GIT_REPO}}.git - git_branch = {{CONFIGMAP_BRANCH}} - git_subpath = airflow/example_dags/ - git_user = - git_password = - git_sync_root = /git - git_sync_path = repo - git_dags_folder_mount_point = {{CONFIGMAP_GIT_DAGS_FOLDER_MOUNT_POINT}} - dags_volume_claim = {{CONFIGMAP_DAGS_VOLUME_CLAIM}} - dags_volume_subpath = - logs_volume_claim = airflow-logs - logs_volume_subpath = - dags_volume_host = - logs_volume_host = - in_cluster = True - namespace = default - - # Example affinity and toleration definitions. - affinity = {"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"kubernetes.io/hostname","operator":"NotIn","values":["4e5e6a99-e28a-450b-bba9-e0124853de9b"]}]}]}}} - tolerations = [{ "key": "dedicated", "operator": "Equal", "value": "airflow", "effect": "NoSchedule" }, { "key": "prod", "operator": "Exists" }] - - # Example kubertenes worker pods annotations declaration. - worker_annotations = { "iam.amazonaws.com/role" : "role-arn", "iam.amazonaws.com/external-id" : "external-id" } - - # For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync - git_sync_container_repository = gcr.io/google-containers/git-sync-amd64 - git_sync_container_tag = v2.0.5 - git_sync_init_container_name = git-sync-clone - - [kubernetes_node_selectors] - # The Key-value pairs to be given to worker pods. - # The worker pods will be scheduled to the nodes of the specified key-value pairs. - # Should be supplied in the format: key = value - - [kubernetes_secrets] - SQL_ALCHEMY_CONN = airflow-secrets=sql_alchemy_conn - - [hive] - # Default mapreduce queue for HiveOperator tasks - default_hive_mapred_queue = - - [celery] - # This section only applies if you are using the CeleryExecutor in - # [core] section above - - # The app name that will be used by celery - celery_app_name = airflow.executors.celery_executor - - # The concurrency that will be used when starting workers with the - # "airflow celery worker" command. This defines the number of task instances that - # a worker will take, so size up your workers based on the resources on - # your worker box and the nature of your tasks - worker_concurrency = 16 - - # When you start an airflow worker, airflow starts a tiny web server - # subprocess to serve the workers local log files to the airflow main - # web server, who then builds pages and sends them to users. This defines - # the port on which the logs are served. It needs to be unused, and open - # visible from the main web server to connect into the workers. - worker_log_server_port = 8793 - - # The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally - # a sqlalchemy database. Refer to the Celery documentation for more - # information. - # http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings - broker_url = sqla+mysql://airflow:airflow@localhost:3306/airflow - - # The Celery result_backend. When a job finishes, it needs to update the - # metadata of the job. Therefore it will post a message on a message bus, - # or insert it into a database (depending of the backend) - # This status is used by the scheduler to update the state of the task - # The use of a database is highly recommended - # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings - result_backend = db+mysql://airflow:airflow@localhost:3306/airflow - - # Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start - # it `airflow celery flower`. This defines the IP that Celery Flower runs on - flower_host = 0.0.0.0 - - # The root URL for Flower - # Ex: flower_url_prefix = /flower - flower_url_prefix = - - # This defines the port that Celery Flower runs on - flower_port = 5555 - - # Securing Flower with Basic Authentication - # Accepts user:password pairs separated by a comma - # Example: flower_basic_auth = user1:password1,user2:password2 - flower_basic_auth = - - # Default queue that tasks get assigned to and that worker listen on. - default_queue = default - - # How many processes CeleryExecutor uses to sync task state. - # 0 means to use max(1, number of cores - 1) processes. - sync_parallelism = 0 - - # Import path for celery configuration options - celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG - - [celery_broker_transport_options] - # The visibility timeout defines the number of seconds to wait for the worker - # to acknowledge the task before the message is redelivered to another worker. - # Make sure to increase the visibility timeout to match the time of the longest - # ETA you're planning to use. Especially important in case of using Redis or SQS - visibility_timeout = 21600 - - # In case of using SSL - ssl_active = False - ssl_key = - ssl_cert = - ssl_cacert = - - [dask] - # This section only applies if you are using the DaskExecutor in - # [core] section above - - # The IP address and port of the Dask cluster's scheduler. - cluster_address = 127.0.0.1:8786 - # TLS/ SSL settings to access a secured Dask scheduler. - tls_ca = - tls_cert = - tls_key = - - [ldap] - # set this to ldaps://: - uri = - user_filter = objectClass=* - user_name_attr = uid - group_member_attr = memberOf - superuser_filter = - data_profiler_filter = - bind_user = cn=Manager,dc=example,dc=com - bind_password = insecure - basedn = dc=example,dc=com - cacert = /etc/ca/ldap_ca.crt - search_scope = LEVEL - - [kerberos] - ccache = /tmp/airflow_krb5_ccache - # gets augmented with fqdn - principal = airflow - reinit_frequency = 3600 - kinit_path = kinit - keytab = airflow.keytab - - [cli] - api_client = airflow.api.client.json_client - endpoint_url = http://localhost:8080 - - [api] - auth_backend = airflow.api.auth.backend.default - - [github_enterprise] - api_rev = v3 - - [admin] - # UI to hide sensitive variable fields when set to True - hide_sensitive_variable_fields = True - - [elasticsearch] - host = - # yamllint enable rule:line-length diff --git a/scripts/ci/in_container/kubernetes/app/templates/init_git_sync.template.yaml b/scripts/ci/in_container/kubernetes/app/templates/init_git_sync.template.yaml deleted file mode 100644 index 343ad8c1511ee..0000000000000 --- a/scripts/ci/in_container/kubernetes/app/templates/init_git_sync.template.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - - name: git-sync-clone - env: - - name: GIT_SYNC_REPO - value: https://github.com/{{CONFIGMAP_GIT_REPO}}.git - - name: GIT_SYNC_BRANCH - value: {{CONFIGMAP_BRANCH}} - - name: GIT_SYNC_ROOT - value: /git - - name: GIT_SYNC_DEST - value: repo - - name: GIT_SYNC_ONE_TIME - value: "true" - image: gcr.io/google-containers/git-sync-amd64:v2.0.5 - imagePullPolicy: IfNotPresent - securityContext: - runAsUser: 0 - volumeMounts: - - mountPath: /git - name: airflow-dags-git diff --git a/scripts/ci/in_container/kubernetes/app/volumes.yaml b/scripts/ci/in_container/kubernetes/app/volumes.yaml deleted file mode 100644 index 86c5c8251eb01..0000000000000 --- a/scripts/ci/in_container/kubernetes/app/volumes.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -kind: PersistentVolume -apiVersion: v1 -metadata: - name: airflow-dags -spec: - accessModes: - - ReadOnlyMany - capacity: - storage: 2Gi - hostPath: - path: /airflow-dags/ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: airflow-dags -spec: - accessModes: - - ReadOnlyMany - resources: - requests: - storage: 2Gi ---- -kind: PersistentVolume -apiVersion: v1 -metadata: - name: airflow-logs -spec: - accessModes: - - ReadOnlyMany - capacity: - storage: 2Gi - hostPath: - path: /airflow-logs/ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: airflow-logs -spec: - accessModes: - - ReadOnlyMany - resources: - requests: - storage: 2Gi ---- -kind: PersistentVolume -apiVersion: v1 -metadata: - name: test-volume -spec: - accessModes: - - ReadWriteOnce - capacity: - storage: 2Gi - hostPath: - path: /airflow-dags/ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: test-volume -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi diff --git a/scripts/ci/in_container/kubernetes/docker/airflow-test-env-init.sh b/scripts/ci/in_container/kubernetes/docker/airflow-test-env-init.sh deleted file mode 100755 index 18a8ec5607c77..0000000000000 --- a/scripts/ci/in_container/kubernetes/docker/airflow-test-env-init.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -x - -cd /opt/airflow/airflow && \ -cp -R example_dags/* /root/airflow/dags/ && \ -find /root/airflow/dags/ && \ -airflow db init && \ -alembic upgrade heads && \ -(airflow users create --username airflow --lastname airflow --firstname jon --email airflow@apache.org --role Admin --password airflow || true) && \ -echo "retrieved from mount" > /root/test_volume/test.txt diff --git a/scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh b/scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh deleted file mode 100755 index 60ebffacc3395..0000000000000 --- a/scripts/ci/in_container/kubernetes/docker/rebuild_airflow_image.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -MY_DIR=$(cd "$(dirname "$0")" && pwd) -AIRFLOW_SOURCES=$(cd "${MY_DIR}/../../../../../" || exit 1 ; pwd) -export AIRFLOW_SOURCES - -# We keep _utils here because we are not in the in_container directory -# shellcheck source=scripts/ci/in_container/_in_container_utils.sh -. "${MY_DIR}/../../_in_container_utils.sh" - -export OUTPUT_LOG=${AIRFLOW_SOURCES}/logs/rebuild_airflow_image.log - -assert_in_container - -in_container_script_start - -cd "${AIRFLOW_SOURCES}" || exit 1 - -# Required to rebuild images from inside container -mkdir -pv scripts/docker/ -cp /entrypoint.sh scripts/docker/ - -echo -echo "Building image from ${AIRFLOW_CI_IMAGE} with latest sources" -echo -start_output_heartbeat "Rebuilding Kubernetes image" 3 -docker build \ - --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ - --build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \ - --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ - --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ - --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH}" \ - --build-arg AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" \ - --build-arg UPGRADE_TO_LATEST_REQUIREMENTS="${UPGRADE_TO_LATEST_REQUIREMENTS}" \ - --build-arg HOME="${HOME}" \ - --cache-from "${AIRFLOW_CI_IMAGE}" \ - --tag="${AIRFLOW_CI_IMAGE}" \ - --target="main" \ - -f Dockerfile.ci . >> "${OUTPUT_LOG}" -echo -echo "Adding kubernetes-specific scripts to basic CI image." -echo "Building ${AIRFLOW_KUBERNETES_IMAGE} from ${AIRFLOW_CI_IMAGE}" -echo -docker build \ - --build-arg AIRFLOW_CI_IMAGE="${AIRFLOW_CI_IMAGE}" \ - --cache-from "${AIRFLOW_CI_IMAGE}" \ - --tag="${AIRFLOW_KUBERNETES_IMAGE}" \ - -f- . >> "${OUTPUT_LOG}" <&1); then - echo "${OUTPUT}" - fi - stop_output_heartbeat - else - kind create cluster \ - --name "${CLUSTER_NAME}" \ - --config "${MY_DIR}/kind-cluster-conf.yaml" \ - --image "kindest/node:${KUBERNETES_VERSION}" - fi - echo - echo "Created cluster ${CLUSTER_NAME}" - echo - - echo - echo "Connecting Kubernetes' worker and ccontrol plane to the network used by Breeze" - echo - docker network connect docker-compose_default "${CLUSTER_NAME}-control-plane" - docker network connect docker-compose_default "${CLUSTER_NAME}-worker" - echo - echo "Connected Kubernetes' worker and ccontrol plane to the network used by Breeze" - echo - - - echo - echo "Replacing cluster host address with https://${CLUSTER_NAME}-control-plane:6443" - echo - kubectl config set "clusters.kind-${CLUSTER_NAME}.server" "https://${CLUSTER_NAME}-control-plane:6443" - - echo - echo "Replaced cluster host address with https://${CLUSTER_NAME}-control-plane:6443" - echo - - echo - echo "Patching CoreDNS to avoid loop and to use 8.8.8.8 DNS as forward address." - echo - echo "============================================================================" - echo " Original coredns configmap:" - echo "============================================================================" - kubectl get configmaps --namespace=kube-system coredns -o yaml - kubectl get configmaps --namespace=kube-system coredns -o yaml | \ - sed 's/forward \. .*$/forward . 8.8.8.8/' | kubectl apply -f - - - echo - echo "============================================================================" - echo " Updated coredns configmap with new forward directive:" - echo "============================================================================" - kubectl get configmaps --namespace=kube-system coredns -o yaml - - - echo - echo "Restarting CoreDNS" - echo - kubectl scale deployment --namespace=kube-system coredns --replicas=0 - kubectl scale deployment --namespace=kube-system coredns --replicas=2 - echo - echo "Restarted CoreDNS" - echo -} - -function delete_cluster() { - kind delete cluster --name "${CLUSTER_NAME}" - echo - echo "Deleted cluster ${CLUSTER_NAME}" - echo - rm -rf "${HOME}/.kube/*" -} - -ALL_CLUSTERS=$(kind get clusters || true) -if [[ ${ALL_CLUSTERS} == *"${CLUSTER_NAME}"* ]]; then - echo - echo "Cluster ${CLUSTER_NAME} is already created" - echo - if [[ ${KIND_CLUSTER_OPERATION} == "start" ]]; then - echo - echo "Reusing previously created cluster" - echo - elif [[ ${KIND_CLUSTER_OPERATION} == "recreate" ]]; then - echo - echo "Recreating cluster" - echo - delete_cluster - create_cluster - elif [[ ${KIND_CLUSTER_OPERATION} == "stop" ]]; then - echo - echo "Deleting cluster" - echo - delete_cluster - exit - else - echo - echo "Wrong cluster operation: ${KIND_CLUSTER_OPERATION}. Should be one of start/stop/recreate" - echo - exit 1 - fi -else - if [[ ${KIND_CLUSTER_OPERATION} == "start" ]]; then - echo - echo "Creating cluster" - echo - create_cluster - elif [[ ${KIND_CLUSTER_OPERATION} == "restart" ]]; then - echo - echo "Cluster ${CLUSTER_NAME} does not exist. Creating rather than recreating" - echo - echo "Creating cluster" - echo - create_cluster - elif [[ ${KIND_CLUSTER_OPERATION} == "stop" ]]; then - echo - echo "Cluster ${CLUSTER_NAME} does not exist. It should exist for stop operation" - echo - exit 1 - else - echo - echo "Wrong cluster operation: ${KIND_CLUSTER_OPERATION}. Should be one of start/stop/recreate" - echo - exit 1 - fi -fi - -kubectl cluster-info - -kubectl get nodes -echo -echo "Showing storageClass" -echo -kubectl get storageclass -echo -echo "Showing kube-system pods" -echo -kubectl get -n kube-system pods -echo -echo "Airflow environment on kubernetes is good to go!" -echo -kubectl create namespace test-namespace - -in_container_script_end diff --git a/scripts/ci/in_container/kubernetes/docker/bootstrap.sh b/scripts/include/clean-logs.sh old mode 100644 new mode 100755 similarity index 65% rename from scripts/ci/in_container/kubernetes/docker/bootstrap.sh rename to scripts/include/clean-logs.sh index 0a9f34d642dd7..7fb0f30319a93 --- a/scripts/ci/in_container/kubernetes/docker/bootstrap.sh +++ b/scripts/include/clean-logs.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,12 +17,21 @@ # specific language governing permissions and limitations # under the License. -if [[ "$1" = "webserver" ]] -then - exec airflow webserver -fi +set -e + +DIRECTORY=${AIRFLOW_HOME:-/usr/local/airflow} +RETENTION=${AIRFLOW__LOG_RETENTION_DAYS:-15} + +trap "exit" INT TERM + +EVERY=$((15*60)) + +echo "Cleaning logs every $EVERY seconds" + +while true; do + seconds=$(( $(date -u +%s) % EVERY)) + [[ $seconds -lt 1 ]] || sleep $((EVERY - seconds)) -if [[ "$1" = "scheduler" ]] -then - exec airflow scheduler -fi + echo "Trimming airflow logs to ${RETENTION} days." + find "${DIRECTORY}"/logs -mtime +"${RETENTION}" -name '*.log' -delete +done diff --git a/setup.py b/setup.py index c1cacc14491b8..88f06df7e5525 100644 --- a/setup.py +++ b/setup.py @@ -623,6 +623,7 @@ def do_setup(): entry_points={ "console_scripts": [ "airflow = airflow.__main__:main", + "airflow-migration-spinner = airflow.include.airflow_migration_spinner:main", ], }, install_requires=INSTALL_REQUIREMENTS, diff --git a/tests/cli/commands/test_db_command.py b/tests/cli/commands/test_db_command.py index a60c105e6a381..15e953eba54ac 100644 --- a/tests/cli/commands/test_db_command.py +++ b/tests/cli/commands/test_db_command.py @@ -42,6 +42,13 @@ def test_cli_resetdb(self, mock_resetdb): mock_resetdb.assert_called_once_with() + @mock.patch("airflow.cli.commands.db_command.db.wait_for_migrations") + def test_cli_wait_for_migrations(self, mock_wait_for_migrations): + a =self.parser.parse_args(['db', 'wait-for-migrations',]) + db_command.wait_for_migrations(self.parser.parse_args(['db', 'wait-for-migrations'])) + + mock_wait_for_migrations.assert_called_once_with(timeout=0) + @mock.patch("airflow.cli.commands.db_command.db.upgradedb") def test_cli_upgradedb(self, mock_upgradedb): db_command.upgradedb(self.parser.parse_args(['db', 'upgrade'])) diff --git a/tests/runtime/kubernetes/test_kubernetes_executor.py b/tests/runtime/kubernetes/test_kubernetes_executor.py index b1739760531a8..3c1165577743c 100644 --- a/tests/runtime/kubernetes/test_kubernetes_executor.py +++ b/tests/runtime/kubernetes/test_kubernetes_executor.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import os import re import time import unittest @@ -26,7 +25,7 @@ from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry -KUBERNETES_HOST = (os.environ.get('CLUSTER_NAME') or "docker") + "-worker:30809" +KUBERNETES_HOST = "localhost:8080" @pytest.mark.runtime("kubernetes")