From feac6fbd61e68a18e5a080c3cc158c1bee4e202e Mon Sep 17 00:00:00 2001 From: Abhishek Agarwal <1477457+abhishekagarwal87@users.noreply.github.com> Date: Mon, 16 Jan 2023 11:32:07 +0530 Subject: [PATCH 1/8] Switch to Hadoop 3 by default --- .github/workflows/static-checks.yml | 9 +++--- .travis.yml | 5 ---- check_test_suite.py | 2 +- distribution/asf-release-process-guide.md | 28 ------------------- distribution/pom.xml | 6 ++-- docs/development/build.md | 18 ------------ extensions-contrib/thrift-extensions/pom.xml | 10 +++---- extensions-core/avro-extensions/pom.xml | 10 +++---- extensions-core/druid-ranger-security/pom.xml | 10 +++---- extensions-core/hdfs-storage/pom.xml | 10 +++---- extensions-core/orc-extensions/pom.xml | 10 +++---- extensions-core/parquet-extensions/pom.xml | 10 +++---- indexing-hadoop/pom.xml | 10 +++---- indexing-service/pom.xml | 10 +++---- integration-tests/pom.xml | 10 +++---- pom.xml | 13 ++++----- 16 files changed, 59 insertions(+), 112 deletions(-) diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index f247bf01a194..798148c32f2e 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -86,21 +86,20 @@ jobs: if: ${{ matrix.java == 'jdk8' }} run: ./.github/scripts/license_checks_script.sh - - name: license checks for hadoop3 + - name: license checks for hadoop 2 if: ${{ matrix.java == 'jdk8' }} env: - HADOOP_PROFILE: -Phadoop3 + HADOOP_PROFILE: -Phadoop2 run: ./.github/scripts/license_checks_script.sh - name: analyze dependencies if: ${{ matrix.java == 'jdk8' }} run: | ./.github/scripts/analyze_dependencies_script.sh - - - name: analyze dependencies for hadoop3 + - name: analyze dependencies for hadoop 2 if: ${{ matrix.java == 'jdk8' }} env: - HADOOP_PROFILE: -Phadoop3 + HADOOP_PROFILE: -Phadoop2 run: | ./.github/scripts/analyze_dependencies_script.sh diff --git a/.travis.yml b/.travis.yml index c9afd77e14b5..1e2da982a225 100644 --- a/.travis.yml +++ b/.travis.yml @@ -660,11 +660,6 @@ jobs: " && false; } - - <<: *security_vulnerabilities - name: "security vulnerabilities with Hadoop3" - env: - - HADOOP_PROFILE='-Phadoop3' - # Travis CI only supports per build (and not per-job notifications): https://github.com/travis-ci/travis-ci/issues/9888 notifications: email: diff --git a/check_test_suite.py b/check_test_suite.py index 741b14180aab..8fd739111426 100755 --- a/check_test_suite.py +++ b/check_test_suite.py @@ -22,7 +22,7 @@ # this script does some primitive examination of git diff to determine if a test suite needs to be run or not # these jobs should always be run, no matter what -always_run_jobs = ['license checks', 'license checks with Hadoop3', '(openjdk8) packaging check', '(openjdk11) packaging check'] +always_run_jobs = ['license checks', '(openjdk8) packaging check', '(openjdk11) packaging check'] # ignore changes to these files completely since they don't impact CI, if the changes are only to these files then all # of CI can be skipped. however, jobs which are always run will still be run even if only these files are changed diff --git a/distribution/asf-release-process-guide.md b/distribution/asf-release-process-guide.md index 4264f2ca5344..0af6e6e880ec 100644 --- a/distribution/asf-release-process-guide.md +++ b/distribution/asf-release-process-guide.md @@ -320,30 +320,6 @@ apache-druid-0.17.0-src.tar.gz.asc apache-druid-0.17.0-src.tar.gz.sha512 ``` -#### Build artifacts for Hadoop-3 - -```bash -$ mvn clean install -Phadoop3,apache-release,dist,rat -DskipTests -Dgpg.keyname= -``` - -This should produce the following artifacts: - -```plaintext -apache-druid-0.17.0-bin.tar.gz -apache-druid-0.17.0-bin.tar.gz.asc -apache-druid-0.17.0-bin.tar.gz.sha512 -apache-druid-0.17.0-src.tar.gz -apache-druid-0.17.0-src.tar.gz.asc -apache-druid-0.17.0-src.tar.gz.sha512 -``` - -You can ignore the src artifacts as they are the same as the main profile. The binary artifacts should be renamed to include the suffix `hadoop3`. So the final artifacts would be as follows: -```plaintext -apache-druid-0.17.0-hadoop3-bin.tar.gz -apache-druid-0.17.0-hadoop3-bin.tar.gz.asc -apache-druid-0.17.0-hadoop3-bin.tar.gz.sha512 -``` - Ensure that the GPG key fingerprint used in the `mvn install` command matches your release signing key in https://dist.apache.org/repos/dist/release/druid/KEYS. ### Verify checksums @@ -353,8 +329,6 @@ $ diff <(shasum -a512 apache-druid-0.17.0-bin.tar.gz | cut -d ' ' -f1) <(cat apa ... $ diff <(shasum -a512 apache-druid-0.17.0-src.tar.gz | cut -d ' ' -f1) <(cat apache-druid-0.17.0-src.tar.gz.sha512 ; echo) ... -$ diff <(shasum -a512 apache-druid-0.17.0-hadoop3-bin.tar.gz | cut -d ' ' -f1) <(cat apache-druid-0.17.0-hadoop3-bin.tar.gz.sha512 ; echo) -... ``` ### Verify GPG signatures @@ -364,8 +338,6 @@ $ gpg --verify apache-druid-0.17.0-bin.tar.gz.asc apache-druid-0.17.0-bin.tar.gz ... $ gpg --verify apache-druid-0.17.0-src.tar.gz.asc apache-druid-0.17.0-src.tar.gz ... -$ gpg --verify apache-druid-0.17.0-hadoop3-bin.tar.gz.asc apache-druid-0.17.0-hadoop3-bin.tar.gz -... ``` ### Commit artifacts to SVN repo diff --git a/distribution/pom.xml b/distribution/pom.xml index 7b92e3436b45..732bc5fa54ea 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -116,7 +116,7 @@ - dist + dist-hadoop2 false @@ -190,6 +190,7 @@ -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies + -Dhadoop2.enabled=true org.apache.druid.cli.Main tools pull-deps @@ -302,7 +303,7 @@ - dist-hadoop3 + dist false @@ -373,7 +374,6 @@ -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies - -Dhadoop3.enabled=true org.apache.druid.cli.Main tools pull-deps diff --git a/docs/development/build.md b/docs/development/build.md index b093b2e4f99c..b1e6777ee56d 100644 --- a/docs/development/build.md +++ b/docs/development/build.md @@ -71,24 +71,6 @@ Putting these together, if you wish to build the source and binary distributions mvn clean install -Papache-release,dist,rat -DskipTests ``` -### Building hadoop 3 distribution - -By default, druid ships hadoop 2.x.x jars along with the distribution. Exact version can be found in the -main [pom](https://github.com/apache/druid/blob/master/pom.xml). To build druid with hadoop 3.x.x jars, hadoop3 profile -needs to be activated. - -To generate build with hadoop 3 dependencies, run: - -```bash -mvn clean install -Phadoop3 -``` - -To generate distribution with hadoop3 dependencies, run : - -```bash -mvn clean install -Papache-release,dist-hadoop3,rat,hadoop3 -DskipTests -``` - #### Potential issues ##### Missing `pyyaml` diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 5d61693a2868..f9e99cd129fe 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -136,7 +136,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -161,10 +164,7 @@ hadoop3 - - hadoop3.enabled - true - + true diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index 53dff82a980c..fcb3005670eb 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -284,7 +284,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -326,10 +329,7 @@ hadoop3 - - hadoop3.enabled - true - + true diff --git a/extensions-core/druid-ranger-security/pom.xml b/extensions-core/druid-ranger-security/pom.xml index edb31fe29336..9baed5dbffcb 100644 --- a/extensions-core/druid-ranger-security/pom.xml +++ b/extensions-core/druid-ranger-security/pom.xml @@ -172,7 +172,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -401,10 +404,7 @@ hadoop3 - - hadoop3.enabled - true - + true diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 587474b5475f..2fe248a37c68 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -150,7 +150,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -436,10 +439,7 @@ hadoop3 - - hadoop3.enabled - true - + true diff --git a/extensions-core/orc-extensions/pom.xml b/extensions-core/orc-extensions/pom.xml index 9de5c3a91fd6..050b96d920a3 100644 --- a/extensions-core/orc-extensions/pom.xml +++ b/extensions-core/orc-extensions/pom.xml @@ -248,7 +248,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -504,10 +507,7 @@ hadoop3 - - hadoop3.enabled - true - + true diff --git a/extensions-core/parquet-extensions/pom.xml b/extensions-core/parquet-extensions/pom.xml index ab834af8fa7a..633c77a297c0 100644 --- a/extensions-core/parquet-extensions/pom.xml +++ b/extensions-core/parquet-extensions/pom.xml @@ -184,7 +184,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -432,10 +435,7 @@ hadoop3 - - hadoop3.enabled - true - + true diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 7f56e7f7562d..b5a9c4b46199 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -172,7 +172,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -227,10 +230,7 @@ hadoop3 - - hadoop3.enabled - true - + true diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 1ea333873843..8135ad8673ad 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -291,7 +291,10 @@ hadoop2 - true + + hadoop2.enabled + true + @@ -335,10 +338,7 @@ hadoop3 - - hadoop3.enabled - true - + true diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index e61094d085e5..b32a3f5261e4 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -39,8 +39,8 @@ - "org.apache.hadoop:hadoop-client:${hadoop.compile.version}", "org.apache.hadoop:hadoop-azure:${hadoop.compile.version}" - org.apache.hadoop.fs.s3native.NativeS3FileSystem + "org.apache.hadoop:hadoop-client-api:${hadoop.compile.version}", "org.apache.hadoop:hadoop-client-runtime:${hadoop.compile.version}", "org.apache.hadoop:hadoop-azure:${hadoop.compile.version}" + org.apache.hadoop.fs.s3a.S3AFileSystem @@ -559,10 +559,10 @@ - hadoop3 + hadoop2 - "org.apache.hadoop:hadoop-client-api:${hadoop.compile.version}", "org.apache.hadoop:hadoop-client-runtime:${hadoop.compile.version}", "org.apache.hadoop:hadoop-azure:${hadoop.compile.version}" - org.apache.hadoop.fs.s3a.S3AFileSystem + "org.apache.hadoop:hadoop-client:${hadoop.compile.version}", "org.apache.hadoop:hadoop-azure:${hadoop.compile.version}" + org.apache.hadoop.fs.s3native.NativeS3FileSystem diff --git a/pom.xml b/pom.xml index ac577819c61a..ca3df0a91e0b 100644 --- a/pom.xml +++ b/pom.xml @@ -110,12 +110,12 @@ 1.3.1 1.7.36 - 2.8.5 + 3.3.1 4.3.1 1.12.317 2.8.0 0.8.7 - 5.2.5.Final + 5.3.6.Final 4.5.13 3.5.9 @@ -2013,17 +2013,16 @@ - hadoop3 + hadoop2 - hadoop3.enabled + hadoop2.enabled true - 3.3.1 - 5.3.6.Final - 4.5.13 + 2.8.5 + 5.2.5.Final From b35fb7c37f29062f8046e0d5dcbbc13a7382c037 Mon Sep 17 00:00:00 2001 From: Abhishek Agarwal <1477457+abhishekagarwal87@users.noreply.github.com> Date: Mon, 16 Jan 2023 23:04:36 +0530 Subject: [PATCH 2/8] Fix license checks --- .github/workflows/static-checks.yml | 6 ------ distribution/bin/check-licenses.py | 2 ++ extensions-core/druid-kerberos/pom.xml | 29 ++++++++++++++++++++++++++ licenses.yaml | 20 ++++++++++++++++++ 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 798148c32f2e..0963da94992a 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -86,12 +86,6 @@ jobs: if: ${{ matrix.java == 'jdk8' }} run: ./.github/scripts/license_checks_script.sh - - name: license checks for hadoop 2 - if: ${{ matrix.java == 'jdk8' }} - env: - HADOOP_PROFILE: -Phadoop2 - run: ./.github/scripts/license_checks_script.sh - - name: analyze dependencies if: ${{ matrix.java == 'jdk8' }} run: | diff --git a/distribution/bin/check-licenses.py b/distribution/bin/check-licenses.py index b5a2c2e933dd..ff77eeace0bf 100755 --- a/distribution/bin/check-licenses.py +++ b/distribution/bin/check-licenses.py @@ -293,6 +293,8 @@ def build_compatible_license_names(): compatible_licenses['The MIT License (MIT)'] = 'MIT License' compatible_licenses['Bouncy Castle Licence'] = 'MIT License' + compatible_licenses['The Go license'] = 'The Go license' + compatible_licenses['-'] = '-' return compatible_licenses diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index 9a36a98ba6eb..15df58eb81d1 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -142,10 +142,19 @@ com.google.code.findbugs jsr305 + org.mortbay.jetty jetty-util + + org.eclipse.jetty + jetty-webapp + + + org.eclipse.jetty + jetty-xml + org.apache.hadoop hadoop-annotations @@ -283,6 +292,26 @@ jetty-client provided + + org.eclipse.jetty + jetty-server + provided + + + org.eclipse.jetty + jetty-util + provided + + + org.eclipse.jetty + jetty-io + provided + + + org.eclipse.jetty + jetty-servlet + provided + diff --git a/licenses.yaml b/licenses.yaml index 77e85f1f4a39..bcb8badd978b 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -3786,6 +3786,16 @@ libraries: --- +name: Hadoop Client API +license_category: binary +module: extensions/druid-hdfs-storage +license_name: Apache License version 2.0 +version: 3.3.1 +libraries: + - org.apache.hadoop: hadoop-client-api + +--- + name: xmlenc license_category: binary module: extensions/druid-hdfs-storage @@ -5064,6 +5074,16 @@ libraries: --- +name: RE2/J +license_category: binary +module: java-core +license_name: The Go license +version: 1.1 +license_file_path: licenses/bin/re2j.GO +libraries: + - com.google.re2j: re2j + +--- # Web console modules start name: "@babel/code-frame" From 2dfc683fb14b239a4dc00fd6b84bba4691d42f13 Mon Sep 17 00:00:00 2001 From: Abhishek Agarwal <1477457+abhishekagarwal87@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:48:42 +0530 Subject: [PATCH 3/8] Add hadoop-client-api as compile dependency --- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/parquet-extensions/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 744bff5f85cc..8340a9aa31f9 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -171,7 +171,7 @@ org.apache.hadoop hadoop-client-api ${hadoop.compile.version} - provided + compile org.apache.hadoop diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index 71addc2ce3c3..3d0cb94d7fb0 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -323,7 +323,7 @@ org.apache.hadoop hadoop-client-api ${hadoop.compile.version} - provided + compile com.sun.jersey diff --git a/extensions-core/parquet-extensions/pom.xml b/extensions-core/parquet-extensions/pom.xml index 9476bcf757aa..41730a3276bf 100644 --- a/extensions-core/parquet-extensions/pom.xml +++ b/extensions-core/parquet-extensions/pom.xml @@ -429,7 +429,7 @@ org.apache.hadoop hadoop-client-api ${hadoop.compile.version} - provided + compile org.apache.hadoop From 5daa975a9bacce6c7294c225f458ea32f9767950 Mon Sep 17 00:00:00 2001 From: Abhishek Agarwal <1477457+abhishekagarwal87@users.noreply.github.com> Date: Wed, 8 Mar 2023 13:01:11 +0530 Subject: [PATCH 4/8] Add client runtime to extensions --- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-core/druid-ranger-security/pom.xml | 2 +- extensions-core/orc-extensions/pom.xml | 2 +- extensions-core/parquet-extensions/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 8340a9aa31f9..43337e02deef 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -177,7 +177,7 @@ org.apache.hadoop hadoop-client-runtime ${hadoop.compile.version} - test + runtime diff --git a/extensions-core/druid-ranger-security/pom.xml b/extensions-core/druid-ranger-security/pom.xml index 04a76ee45be1..0dfc98b5a880 100644 --- a/extensions-core/druid-ranger-security/pom.xml +++ b/extensions-core/druid-ranger-security/pom.xml @@ -417,7 +417,7 @@ org.apache.hadoop hadoop-client-runtime ${hadoop.compile.version} - test + runtime diff --git a/extensions-core/orc-extensions/pom.xml b/extensions-core/orc-extensions/pom.xml index d11e73c018a9..5bdaa7b67594 100644 --- a/extensions-core/orc-extensions/pom.xml +++ b/extensions-core/orc-extensions/pom.xml @@ -512,7 +512,7 @@ org.apache.hadoop hadoop-client-runtime ${hadoop.compile.version} - test + runtime diff --git a/extensions-core/parquet-extensions/pom.xml b/extensions-core/parquet-extensions/pom.xml index 41730a3276bf..48872c6ef760 100644 --- a/extensions-core/parquet-extensions/pom.xml +++ b/extensions-core/parquet-extensions/pom.xml @@ -435,7 +435,7 @@ org.apache.hadoop hadoop-client-runtime ${hadoop.compile.version} - test + runtime From f37d36848558be85f553cb0af462d8f808b124d8 Mon Sep 17 00:00:00 2001 From: Abhishek Agarwal <1477457+abhishekagarwal87@users.noreply.github.com> Date: Sun, 12 Mar 2023 09:17:55 +0530 Subject: [PATCH 5/8] Add missing extension to the profile --- distribution/pom.xml | 4 ++-- integration-tests-ex/image/docker/launch.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index 732bc5fa54ea..51980a581848 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -226,8 +226,6 @@ -c org.apache.druid.extensions:druid-multi-stage-query -c - org.apache.druid.extensions:druid-catalog - -c org.apache.druid.extensions:druid-protobuf-extensions -c org.apache.druid.extensions:mysql-metadata-storage @@ -442,6 +440,8 @@ org.apache.druid.extensions:druid-ranger-security -c org.apache.druid.extensions:druid-kubernetes-extensions + -c + org.apache.druid.extensions:druid-catalog ${druid.distribution.pulldeps.opts} diff --git a/integration-tests-ex/image/docker/launch.sh b/integration-tests-ex/image/docker/launch.sh index 8116ab963231..13fcc98a4dee 100644 --- a/integration-tests-ex/image/docker/launch.sh +++ b/integration-tests-ex/image/docker/launch.sh @@ -88,7 +88,7 @@ fi # Assemble Java options JAVA_OPTS="$DRUID_SERVICE_JAVA_OPTS $DRUID_COMMON_JAVA_OPTS -XX:HeapDumpPath=$LOG_DIR/$INSTANCE_NAME $DEBUG_OPTS" -LOG4J_CONFIG=$SHARED_DIR/conf/log4j2.xml +LOG4J_CONFIG=$SHARED_DIR/resources/log4j2.xml if [ -f $LOG4J_CONFIG ]; then JAVA_OPTS="$JAVA_OPTS -Dlog4j.configurationFile=$LOG4J_CONFIG" fi From 3259c4149489509c69cd551e5c57b9ecf4325c3f Mon Sep 17 00:00:00 2001 From: "tejaswini.bandlamudi" Date: Fri, 31 Mar 2023 13:55:29 +0530 Subject: [PATCH 6/8] Update hadoop version and exclusively download aws-java-sdk-s3 dependency --- extensions-core/hdfs-storage/pom.xml | 12 ++++++++++++ pom.xml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 4cf99b89473e..2e9cffe87c57 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -45,6 +45,12 @@ hadoop-aws ${hadoop.compile.version} runtime + + + com.amazonaws + aws-java-sdk-bundle + + commons-io @@ -447,6 +453,12 @@ ${hadoop.compile.version} test + + com.amazonaws + aws-java-sdk-s3 + ${aws.sdk.version} + runtime + log4j log4j diff --git a/pom.xml b/pom.xml index 1111b237923e..407f19c8f3a5 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ 1.3.1 1.7.36 - 3.3.1 + 3.3.5 4.3.1 1.12.317 2.8.0 From c5a9f89c9d731eb03b1cc924ef2ae9228641726d Mon Sep 17 00:00:00 2001 From: "tejaswini.bandlamudi" Date: Fri, 31 Mar 2023 14:01:04 +0530 Subject: [PATCH 7/8] nit static-checks updates --- .github/scripts/analyze_dependencies_script.sh | 2 +- .github/scripts/license_checks_script.sh | 2 +- .github/workflows/static-checks.yml | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/scripts/analyze_dependencies_script.sh b/.github/scripts/analyze_dependencies_script.sh index 1212dbd1cd2b..c92d90030f71 100755 --- a/.github/scripts/analyze_dependencies_script.sh +++ b/.github/scripts/analyze_dependencies_script.sh @@ -15,7 +15,7 @@ #!bin/bash -${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true || +${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${HADOOP_PROFILE} || { echo " The dependency analysis has found a dependency that is either: diff --git a/.github/scripts/license_checks_script.sh b/.github/scripts/license_checks_script.sh index 163214f82988..410ac60375fa 100755 --- a/.github/scripts/license_checks_script.sh +++ b/.github/scripts/license_checks_script.sh @@ -20,7 +20,7 @@ set -e ./.github/scripts/setup_generate_license.sh ${MVN} apache-rat:check -Prat --fail-at-end \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ --Drat.consoleOutput=true +-Drat.consoleOutput=true ${HADOOP_PROFILE} # Generate dependency reports and checks they are valid. mkdir -p target distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2 diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index cfc25ec60d29..1f2b758e5d8e 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -88,8 +88,10 @@ jobs: run: | ./.github/scripts/analyze_dependencies_script.sh - - name: analyze dependencies for hadoop + - name: analyze dependencies for hadoop2 if: ${{ matrix.java == 'jdk8' }} + env: + HADOOP_PROFILE: -Phadoop2 run: | ./.github/scripts/analyze_dependencies_script.sh From 56589f32f3318a7dcfb10f3f3a0dd78e11f50e13 Mon Sep 17 00:00:00 2001 From: "tejaswini.bandlamudi" Date: Mon, 24 Apr 2023 21:10:05 +0530 Subject: [PATCH 8/8] update license information --- extensions-core/druid-kerberos/pom.xml | 2 +- licenses.yaml | 57 ++++++++++++++++++-------- 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index c697166a784a..dfffd516d12f 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -202,7 +202,7 @@ json-smart - com.sun.jersey + com.github.pjfanning jersey-json diff --git a/licenses.yaml b/licenses.yaml index ed6ca43d7c41..ba0e3ca2171d 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -644,7 +644,7 @@ name: Apache Commons Configuration license_category: binary module: java-core license_name: Apache License version 2.0 -version: 2.1.1 +version: 2.8.0 libraries: - org.apache.commons: commons-configuration2 @@ -2913,22 +2913,10 @@ name: Apache Hadoop license_category: binary module: hadoop-client license_name: Apache License version 2.0 -version: 3.3.1 +version: 3.3.5 libraries: - - org.apache.hadoop: hadoop-annotations - org.apache.hadoop: hadoop-auth - - org.apache.hadoop: hadoop-client - org.apache.hadoop: hadoop-common - - org.apache.hadoop: hadoop-hdfs-client - - org.apache.hadoop: hadoop-mapreduce-client-app - - org.apache.hadoop: hadoop-mapreduce-client-common - - org.apache.hadoop: hadoop-mapreduce-client-core - - org.apache.hadoop: hadoop-mapreduce-client-jobclient - - org.apache.hadoop: hadoop-mapreduce-client-shuffle - - org.apache.hadoop: hadoop-yarn-api - - org.apache.hadoop: hadoop-yarn-client - - org.apache.hadoop: hadoop-yarn-common - - org.apache.hadoop: hadoop-yarn-server-common --- @@ -3485,6 +3473,43 @@ notices: --- +name: reload4j +license_category: binary +module: hadoop-common +license_name: Apache License version 2.0 +version: 1.2.22 +libraries: + - ch.qos.reload4j: reload4j +notices: + - reload4j: | + Apache log4j + Copyright 2010 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + +--- + +name: slf4j-reload4j +license_category: binary +module: hadoop-common +license_name: MIT License +version: 1.7.36 +libraries: + - org.slf4j: slf4j-reload4j + +--- + +name: com.github.pjfanning jersey-json +license_category: binary +module: druid-kerberos +license_name: CDDL 1.1 +version: 1.20 +libraries: + - com.github.pjfanning: jersey-json + +--- + name: Kafka Schema Registry Client version: 5.5.1 license_category: binary @@ -3790,7 +3815,7 @@ name: Hadoop Client API license_category: binary module: extensions/druid-hdfs-storage license_name: Apache License version 2.0 -version: 3.3.1 +version: 3.3.5 libraries: - org.apache.hadoop: hadoop-client-api @@ -5047,7 +5072,7 @@ name: Woodstox license_category: binary module: java-core license_name: Apache License version 2.0 -version: 5.3.0 +version: 5.4.0 libraries: - com.fasterxml.woodstox: woodstox-core