Update Hadoop3 as default build version

.github/workflows/static-checks.yml

@@ -85,21 +85,15 @@ jobs:
         if: ${{ matrix.java == 'jdk8' }}
         run: ./.github/scripts/license_checks_script.sh
 
-      - name: license checks for hadoop3
-        if: ${{ matrix.java == 'jdk8' }}
-        env:
-          HADOOP_PROFILE: -Phadoop3
-        run: ./.github/scripts/license_checks_script.sh
-
       - name: analyze dependencies
         if: ${{ matrix.java == 'jdk8' }}
         run: |
           ./.github/scripts/analyze_dependencies_script.sh
 
-      - name: analyze dependencies for hadoop3
+      - name: analyze dependencies for hadoop2
         if: ${{ matrix.java == 'jdk8' }}
         env:
-          HADOOP_PROFILE: -Phadoop3
+          HADOOP_PROFILE: -Phadoop2
         run: |
           ./.github/scripts/analyze_dependencies_script.sh
 

check_test_suite.py

@@ -22,7 +22,7 @@
 # this script does some primitive examination of git diff to determine if a test suite needs to be run or not
 
 # these jobs should always be run, no matter what
-always_run_jobs = ['license checks', 'license checks with Hadoop3', '(openjdk8) packaging check', '(openjdk11) packaging check']
+always_run_jobs = ['license checks', '(openjdk8) packaging check', '(openjdk11) packaging check']
 
 # ignore changes to these files completely since they don't impact CI, if the changes are only to these files then all
 # of CI can be skipped. however, jobs which are always run will still be run even if only these files are changed

distribution/asf-release-process-guide.md

@@ -328,30 +328,6 @@ apache-druid-0.17.0-src.tar.gz.asc
 apache-druid-0.17.0-src.tar.gz.sha512
 ```
 
-#### Build artifacts for Hadoop-3
-
-```bash
-$ mvn clean install -Phadoop3,apache-release,dist,rat -DskipTests -Dgpg.keyname=<your GPG key fingerprint>
-```
-
-This should produce the following artifacts:
-
-```plaintext
-apache-druid-0.17.0-bin.tar.gz
-apache-druid-0.17.0-bin.tar.gz.asc
-apache-druid-0.17.0-bin.tar.gz.sha512
-apache-druid-0.17.0-src.tar.gz
-apache-druid-0.17.0-src.tar.gz.asc
-apache-druid-0.17.0-src.tar.gz.sha512
-```
-
-You can ignore the src artifacts as they are the same as the main profile. The binary artifacts should be renamed to include the suffix `hadoop3`. So the final artifacts would be as follows:  
-```plaintext
-apache-druid-0.17.0-hadoop3-bin.tar.gz
-apache-druid-0.17.0-hadoop3-bin.tar.gz.asc
-apache-druid-0.17.0-hadoop3-bin.tar.gz.sha512
-```
-
 Ensure that the GPG key fingerprint used in the `mvn install` command matches your release signing key in https://dist.apache.org/repos/dist/release/druid/KEYS.                                                                                               
 
 ### Verify checksums
@@ -361,8 +337,6 @@ $ diff <(shasum -a512 apache-druid-0.17.0-bin.tar.gz | cut -d ' ' -f1) <(cat apa
 ...
 $ diff <(shasum -a512 apache-druid-0.17.0-src.tar.gz | cut -d ' ' -f1) <(cat apache-druid-0.17.0-src.tar.gz.sha512 ; echo)
 ...
-$ diff <(shasum -a512 apache-druid-0.17.0-hadoop3-bin.tar.gz | cut -d ' ' -f1) <(cat apache-druid-0.17.0-hadoop3-bin.tar.gz.sha512 ; echo)
-...
 ```
 
 ### Verify GPG signatures
@@ -372,8 +346,6 @@ $ gpg --verify apache-druid-0.17.0-bin.tar.gz.asc apache-druid-0.17.0-bin.tar.gz
 ...
 $ gpg --verify apache-druid-0.17.0-src.tar.gz.asc apache-druid-0.17.0-src.tar.gz
 ...
-$ gpg --verify apache-druid-0.17.0-hadoop3-bin.tar.gz.asc apache-druid-0.17.0-hadoop3-bin.tar.gz
-...
 ```
 
 ### Commit artifacts to SVN repo

distribution/bin/check-licenses.py

@@ -293,6 +293,8 @@ def build_compatible_license_names():
     compatible_licenses['The MIT License (MIT)'] = 'MIT License'
     compatible_licenses['Bouncy Castle Licence'] = 'MIT License'
 
+    compatible_licenses['The Go license'] = 'The Go license'
+
     compatible_licenses['-'] = '-'
     return compatible_licenses
 

distribution/pom.xml

@@ -116,7 +116,7 @@
 
     <profiles>
         <profile>
-            <id>dist</id>
+            <id>dist-hadoop2</id>
             <activation>
                 <activeByDefault>false</activeByDefault>
                 <property>
@@ -190,6 +190,7 @@
                                         <argument>
                                             -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies
                                         </argument>
+                                        <argument>-Dhadoop2.enabled=true</argument>
                                         <argument>org.apache.druid.cli.Main</argument>
                                         <argument>tools</argument>
                                         <argument>pull-deps</argument>
@@ -225,8 +226,6 @@
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-multi-stage-query</argument>
                                         <argument>-c</argument>
-                                        <argument>org.apache.druid.extensions:druid-catalog</argument>
-                                        <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-protobuf-extensions</argument>
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:mysql-metadata-storage</argument>
@@ -302,7 +301,7 @@
             </build>
         </profile>
         <profile>
-            <id>dist-hadoop3</id>
+            <id>dist</id>
             <activation>
                 <activeByDefault>false</activeByDefault>
             </activation>
@@ -373,7 +372,6 @@
                                         <argument>
                                             -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies
                                         </argument>
-                                        <argument>-Dhadoop3.enabled=true</argument>
                                         <argument>org.apache.druid.cli.Main</argument>
                                         <argument>tools</argument>
                                         <argument>pull-deps</argument>
@@ -442,6 +440,8 @@
                                         <argument>org.apache.druid.extensions:druid-ranger-security</argument>
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-kubernetes-extensions</argument>
+                                        <argument>-c</argument>
+                                        <argument>org.apache.druid.extensions:druid-catalog</argument>
                                         <argument>${druid.distribution.pulldeps.opts}</argument>
                                     </arguments>
                                 </configuration>

docs/development/build.md

@@ -72,24 +72,6 @@ Putting these together, if you wish to build the source and binary distributions
 mvn clean install -Papache-release,dist,rat -DskipTests
 ```
 
-### Building hadoop 3 distribution
-
-By default, druid ships hadoop 2.x.x jars along with the distribution. Exact version can be found in the
-main [pom](https://github.com/apache/druid/blob/master/pom.xml). To build druid with hadoop 3.x.x jars, hadoop3 profile
-needs to be activated.
-
-To generate build with hadoop 3 dependencies, run:
-
-```bash
-mvn clean install -Phadoop3
-```
-
-To generate distribution with hadoop3 dependencies, run :
-
-```bash
-mvn clean install -Papache-release,dist-hadoop3,rat,hadoop3 -DskipTests
-```
-
 #### Potential issues
 
 ##### Missing `pyyaml`

extensions-contrib/thrift-extensions/pom.xml

@@ -136,7 +136,10 @@
     <profile>
       <id>hadoop2</id>
       <activation>
-        <activeByDefault>true</activeByDefault>
+        <property>
+          <name>hadoop2.enabled</name>
+          <value>true</value>
+        </property>
       </activation>
       <dependencies>
         <dependency>
@@ -161,23 +164,20 @@
     <profile>
       <id>hadoop3</id>
       <activation>
-        <property>
-          <name>hadoop3.enabled</name>
-          <value>true</value>
-        </property>
+        <activeByDefault>true</activeByDefault>
       </activation>
       <dependencies>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-client-api</artifactId>
           <version>${hadoop.compile.version}</version>
-          <scope>provided</scope>
+          <scope>compile</scope>
         </dependency>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-client-runtime</artifactId>
           <version>${hadoop.compile.version}</version>
-          <scope>test</scope>
+          <scope>runtime</scope>
         </dependency>
       </dependencies>
     </profile>

extensions-core/avro-extensions/pom.xml

@@ -271,7 +271,10 @@
     <profile>
       <id>hadoop2</id>
       <activation>
-        <activeByDefault>true</activeByDefault>
+        <property>
+          <name>hadoop2.enabled</name>
+          <value>true</value>
+        </property>
       </activation>
       <dependencies>
         <dependency>
@@ -313,17 +316,14 @@
     <profile>
       <id>hadoop3</id>
       <activation>
-        <property>
-          <name>hadoop3.enabled</name>
-          <value>true</value>
-        </property>
+        <activeByDefault>true</activeByDefault>
       </activation>
       <dependencies>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-client-api</artifactId>
           <version>${hadoop.compile.version}</version>
-          <scope>provided</scope>
+          <scope>compile</scope>
         </dependency>
         <dependency>
         <groupId>com.sun.jersey</groupId>

extensions-core/druid-kerberos/pom.xml

@@ -136,10 +136,19 @@
           <groupId>com.google.code.findbugs</groupId>
           <artifactId>jsr305</artifactId>
         </exclusion>
+        <!-- multiple jetty-util exclusions because of different Hadoop profiles (2.x, 3.x) -->
         <exclusion>
           <groupId>org.mortbay.jetty</groupId>
           <artifactId>jetty-util</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-webapp</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-xml</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-annotations</artifactId>
@@ -193,7 +202,7 @@
           <artifactId>json-smart</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.sun.jersey</groupId>
+          <groupId>com.github.pjfanning</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
         <exclusion>
@@ -277,6 +286,26 @@
       <artifactId>jetty-client</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-server</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-util</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-io</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlet</artifactId>
+      <scope>provided</scope>
+    </dependency>
     <!-- we require not shaded version of hadoop deps in case of hadoop3 as we are extending
     org.apache.hadoop.security.authentication.server.AuthenticationHandler -->
     <dependency>

extensions-core/druid-ranger-security/pom.xml

@@ -172,7 +172,10 @@
         <profile>
             <id>hadoop2</id>
             <activation>
-                <activeByDefault>true</activeByDefault>
+                <property>
+                    <name>hadoop2.enabled</name>
+                    <value>true</value>
+                </property>
             </activation>
             <dependencies>
                 <dependency>
@@ -401,10 +404,7 @@
         <profile>
             <id>hadoop3</id>
             <activation>
-                <property>
-                    <name>hadoop3.enabled</name>
-                    <value>true</value>
-                </property>
+                <activeByDefault>true</activeByDefault>
             </activation>
             <dependencies>
                 <dependency>
@@ -417,7 +417,7 @@
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-client-runtime</artifactId>
                     <version>${hadoop.compile.version}</version>
-                    <scope>test</scope>
+                    <scope>runtime</scope>
                 </dependency>
             </dependencies>
         </profile>

extensions-core/hdfs-storage/pom.xml

@@ -45,6 +45,12 @@
           <artifactId>hadoop-aws</artifactId>
           <version>${hadoop.compile.version}</version>
           <scope>runtime</scope>
+          <exclusions>
+            <exclusion>
+              <groupId>com.amazonaws</groupId>
+              <artifactId>aws-java-sdk-bundle</artifactId>
+            </exclusion>
+          </exclusions>
         </dependency>
         <dependency>
             <groupId>commons-io</groupId>
@@ -137,7 +143,10 @@
     <profile>
       <id>hadoop2</id>
       <activation>
-        <activeByDefault>true</activeByDefault>
+        <property>
+          <name>hadoop2.enabled</name>
+          <value>true</value>
+        </property>
       </activation>
       <dependencies>
         <dependency>
@@ -423,10 +432,7 @@
     <profile>
       <id>hadoop3</id>
       <activation>
-        <property>
-          <name>hadoop3.enabled</name>
-          <value>true</value>
-        </property>
+        <activeByDefault>true</activeByDefault>
       </activation>
       <dependencies>
         <dependency>
@@ -447,6 +453,12 @@
           <version>${hadoop.compile.version}</version>
           <scope>test</scope>
         </dependency>
+        <dependency>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-java-sdk-s3</artifactId>
+          <version>${aws.sdk.version}</version>
+          <scope>runtime</scope>
+        </dependency>
         <dependency>
           <groupId>log4j</groupId>
           <artifactId>log4j</artifactId>