Skip to content

[AIRFLOW-3885] Improve Travis CI cycle time #4703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
drewsonne wants to merge 2 commits into from
Closed

[AIRFLOW-3885] Improve Travis CI cycle time #4703

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d575d60
[AIRFLOW-3885] Split license script into install and run
drewsonne Feb 13, 2019
038f780
Merge branch 'master' into feature/nose-fast-fail
Jun 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ jobs:
script: mypy airflow tests
- name: Check license header
stage: pre-test
install: skip
script: scripts/ci/6-check-license.sh
install: scripts/ci/6-check-license.sh install
script: scripts/ci/6-check-license.sh run
- name: Lint Dockerfile
stage: pre-test
install: skip
Expand Down
86 changes: 51 additions & 35 deletions scripts/ci/6-check-license.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
Expand All @@ -17,13 +16,29 @@
# limitations under the License.
#

declare -r RAT_VERSION=0.12
declare -r URL="http://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar"

acquire_rat_jar () {
declare -r TMP_DIR=/tmp
declare -r rat_jar="${TMP_DIR}"/lib/apache-rat-${RAT_VERSION}.jar

@nvtkaszpir nvtkaszpir May 26, 2019

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe double quote after TMP_DIR var should be at the end of the line.

@ashb ashb May 28, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's perhaps more conventional there but this works fine

$ echo "$HOME"
/Users/ash
$ echo "$HOME"def
/Users/ashdef

@nvtkaszpir nvtkaszpir May 28, 2019

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I'm aware it works, but looking at the fact that there is furhter on ${RAT_VERSION} I would rather expect the whole declare -r rat_jar= to be double quoted.


URL="http://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar"
declare -r DEFAULT_JAVA_CMD="$JAVA_HOME/bin/java"

# Go to the Airflow project root directory
FWDIR="$(cd "`dirname "$0"`"/../..; pwd)"
cd "$FWDIR"

acquire_rat_jar () {
set -e

JAR="$rat_jar"

if [ -f "JAR" ]; then
return 0
fi

mkdir -p ${TMP_DIR}/lib

# Download rat launch jar if it hasn't been downloaded yet
if [ ! -f "$JAR" ]; then
# Download
Expand All @@ -44,50 +59,51 @@ acquire_rat_jar () {
# We failed to download
rm "$JAR"
printf "Our attempt to download rat locally to ${JAR} failed. Please install rat manually.\n"
exit -1
return -1
fi
printf "Done downloading.\n"

set +e
}

# Go to the Airflow project root directory
FWDIR="$(cd "`dirname "$0"`"/../..; pwd)"
cd "$FWDIR"
run_rat () {

TMP_DIR=/tmp
if test -x $DEFAULT_JAVA_CMD; then
declare -r java_cmd=$DEFAULT_JAVA_CMD
else
declare -r java_cmd=java
fi

if test -x "$JAVA_HOME/bin/java"; then
declare java_cmd="$JAVA_HOME/bin/java"
else
declare java_cmd=java
fi
# This is the target of a symlink in airflow/www/static/docs - and rat exclude doesn't cope with the symlink target doesn't exist
mkdir -p docs/_build/html/

export RAT_VERSION=0.12
export rat_jar="${TMP_DIR}"/lib/apache-rat-${RAT_VERSION}.jar
mkdir -p ${TMP_DIR}/lib
echo "Running license checks. This can take a while."
$java_cmd -jar "$rat_jar" -E "$FWDIR"/.rat-excludes -d "$FWDIR" > rat-results.txt

if [ $? -ne 0 ]; then
echo "RAT exited abnormally"
exit 1
fi

[[ -f "$rat_jar" ]] || acquire_rat_jar || {
echo "Download failed. Obtain the rat jar manually and place it at $rat_jar"
exit 1
}
ERRORS="$(cat rat-results.txt | grep -e "??")"

# This is the target of a symlink in airflow/www/static/docs - and rat exclude doesn't cope with the symlink target doesn't exist
mkdir -p docs/_build/html/
if test ! -z "$ERRORS"; then
echo >&2 "Could not find Apache license headers in the following files:"
echo >&2 "$ERRORS"
exit 1
else
echo -e "RAT checks passed."
fi

}

echo "Running license checks. This can take a while."
$java_cmd -jar "$rat_jar" -E "$FWDIR"/.rat-excludes -d "$FWDIR" > rat-results.txt
declare -r ACTION=${1:-install_run}

if [ $? -ne 0 ]; then
echo "RAT exited abnormally"
exit 1
if [[ "${ACTION,,}" == *"install"* ]]; then
acquire_rat_jar
fi

ERRORS="$(cat rat-results.txt | grep -e "??")"

if test ! -z "$ERRORS"; then
echo >&2 "Could not find Apache license headers in the following files:"
echo >&2 "$ERRORS"
exit 1
else
echo -e "RAT checks passed."
if [[ "${ACTION,,}" == *"run"* ]]; then
run_rat
fi