[AIRFLOW-4117] Multi-staging Image - Travis CI tests [Step 3/3]

.dockerignore

@@ -30,16 +30,21 @@
 !airflow
 !dags
 !dev
-#!/docs - we do not need docs in the Docker!
+!docs
 !licenses
 !scripts
 !tests
 
 !.coveragerc
 !.rat-excludes
+!.flake8
+!pylintrc
 !LICENSE
 !MANIFEST.in
 !NOTICE
+!CHANGELOG.txt
+!.github
+!run-tests
 
 # Avoid triggering context change on README change (new companies using Airflow)
 # So please do not uncomment this line ;)
@@ -98,3 +103,7 @@ airflow/www/static/docs
 # Exclude auto-generated Finder files on Mac OS
 **/.DS_Store
 **/Thumbs.db
+
+# Exclude docs generated files
+docs/_build/
+docs/_api/

.flake8

@@ -1,5 +1,5 @@
 [flake8]
 max-line-length = 110
 ignore = E731,W504
-exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg,*/_vendor/*,node_modules
+exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,.eggs,*.egg,*/_vendor/*,node_modules
 format = ${cyan}%(path)s${reset}:${yellow_bold}%(row)d${reset}:${green_bold}%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s

.gitignore

@@ -158,3 +158,7 @@ airflow-*.pid
 .mypy_cache/
 .dmypy.json
 dmypy.json
+
+# Needed for CI Dockerfile build system
+.build
+/tmp

.travis.yml

@@ -18,77 +18,61 @@
 #
 dist: xenial
 language: python
-python:
-  - "3.6"
 env:
   global:
-    - TRAVIS_CACHE=$HOME/.travis_cache/
-    - CASS_DRIVER_BUILD_CONCURRENCY=8
-    - CASS_DRIVER_NO_CYTHON=1
+    - BUILD_ID=${TRAVIS_BUILD_ID}
+    - AIRFLOW_CONTAINER_BRANCH_NAME=${TRAVIS_BRANCH}
   matrix:
-    - TOX_ENV=py35-backend_mysql-env_docker PYTHON_VERSION=3
-    - TOX_ENV=py35-backend_sqlite-env_docker PYTHON_VERSION=3
-    - TOX_ENV=py35-backend_postgres-env_docker PYTHON_VERSION=3
-    - TOX_ENV=py35-backend_postgres-env_kubernetes KUBERNETES_VERSION=v1.13.0 PYTHON_VERSION=3
-
+    - BACKEND=mysql ENV=docker
+    - BACKEND=postgres ENV=docker
+    - BACKEND=sqlite ENV=docker
+    - BACKEND=postgres ENV=kubernetes KUBERNETES_VERSION=v1.13.0
+python:
+  - "3.6"
+  - "3.5"
+matrix:
+  exclude:
+    - env: BACKEND=postgres ENV=docker
+      python: "3.5"
+    - env: BACKEND=mysql ENV=docker
+      python: "3.5"
+    - env: BACKEND=postgres ENV=kubernetes KUBERNETES_VERSION=v1.13.0
+      python: "3.5"
+    - env: BACKEND=sqlite ENV=docker
+      python: "3.6"
 stages:
   - pre-test
   - test
-
 jobs:
   include:
     - name: Flake8
       stage: pre-test
-      install: pip install flake8
-      script: flake8
+      script: ./scripts/ci/ci_flake8.sh
+      install: skip
     - name: mypy
       stage: pre-test
-      install: pip install mypy
-      script: mypy airflow tests
+      script: ./scripts/ci/ci_mypy.sh
+      install: skip
     - name: Check license header
       stage: pre-test
+      script: ./scripts/ci/ci_check_license.sh
       install: skip
-      script: scripts/ci/6-check-license.sh
     - name: Lint Dockerfile
       stage: pre-test
+      script: ./scripts/ci/ci_lint_dockerfile.sh
       install: skip
-      script: scripts/ci/ci_lint_dockerfile.sh
     - name: Check docs
-      stage: pre-test
-      install: pip install -e .[doc]
-      script: docs/build.sh
+      stage: test
+      script: ./scripts/ci/ci_docs.sh
+      install: skip
     - name: Pylint
       stage: pre-test
-      install: pip install pylint~=2.3.1  # Ensure the same version as in setup.py
-      script: scripts/ci/ci_pylint.sh
-      cache: false
-cache:
-  directories:
-    - $HOME/.wheelhouse/
-    - $HOME/.cache/pip
-    - $HOME/.travis_cache/
+      script: ./scripts/ci/ci_pylint.sh
+      install: skip
+services:
+  - docker
 before_install:
-  # Required for K8s v1.10.x. See
-  # https://github.com/kubernetes/kubernetes/issues/61058#issuecomment-372764783
-  - if [ ! -z "$KUBERNETES_VERSION" ]; then sudo mount --make-shared / && sudo service docker restart; fi
+  - ./scripts/ci/ci_before_install.sh
 install:
-  - pip install --upgrade pip
-  - docker-compose -f scripts/ci/docker-compose.yml pull --quiet
-script:
-  - if [ -z "$KUBERNETES_VERSION" ]; then
-      docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml run airflow-testing /app/scripts/ci/run-ci.sh;
-    fi
-  - if [ ! -z "$KUBERNETES_VERSION" ]; then
-      ./scripts/ci/kubernetes/minikube/stop_minikube.sh &&
-      ./scripts/ci/kubernetes/setup_kubernetes.sh &&
-      ./scripts/ci/kubernetes/kube/deploy.sh -d persistent_mode &&
-      MINIKUBE_IP=$(minikube ip) docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml -f scripts/ci/docker-compose-kubernetes.yml run airflow-testing /app/scripts/ci/run-ci.sh;
-    fi
-  - if [ ! -z "$KUBERNETES_VERSION" ]; then
-      ./scripts/ci/kubernetes/minikube/stop_minikube.sh &&
-      ./scripts/ci/kubernetes/setup_kubernetes.sh &&
-      ./scripts/ci/kubernetes/kube/deploy.sh -d git_mode &&
-      MINIKUBE_IP=$(minikube ip) docker-compose --log-level ERROR -f scripts/ci/docker-compose.yml -f scripts/ci/docker-compose-kubernetes.yml run airflow-testing /app/scripts/ci/run-ci.sh;
-    fi
-before_cache:
-  - sudo chown -R travis:travis $HOME/.cache/pip $HOME/.wheelhouse/
+  - ./hooks/build
+script: "./scripts/ci/ci_run_airflow_testing.sh"