diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 737efa8f7846b..a87e0af0b542f 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1072,20 +1072,9 @@ object DependencyOverrides { object ExcludedDependencies { lazy val settings = Seq( libraryDependencies ~= { libs => libs.filterNot(_.name == "groovy-all") }, - // SPARK-33705: Due to sbt compiler issues, it brings exclusions defined in maven pom back to - // the classpath directly and assemble test scope artifacts to assembly/target/scala-xx/jars, - // which is also will be added to the classpath of some unit tests that will build a subprocess - // to run `spark-submit`, e.g. HiveThriftServer2Test. - // - // These artifacts are for the jersey-1 API but Spark use jersey-2 ones, so it cause test - // flakiness w/ jar conflicts issues. - // - // Also jersey-1 is only used by yarn module(see resource-managers/yarn/pom.xml) for testing - // purpose only. Here we exclude them from the whole project scope and add them w/ yarn only. excludeDependencies ++= Seq( - ExclusionRule(organization = "com.sun.jersey"), ExclusionRule(organization = "ch.qos.logback"), - ExclusionRule("javax.ws.rs", "jsr311-api")) + ExclusionRule("javax.servlet", "javax.servlet-api")) ) } @@ -1229,10 +1218,6 @@ object YARN { val hadoopProvidedProp = "spark.yarn.isHadoopProvided" lazy val settings = Seq( - excludeDependencies --= Seq( - ExclusionRule(organization = "com.sun.jersey"), - ExclusionRule("javax.servlet", "javax.servlet-api"), - ExclusionRule("javax.ws.rs", "jsr311-api")), Compile / unmanagedResources := (Compile / unmanagedResources).value.filter(!_.getName.endsWith(s"$propFileName")), genConfigProperties := { diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index 770a550030f51..5a10aa797c1b1 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -29,43 +29,8 @@ Spark Project YARN yarn - 1.19 - - - hadoop-3 - - true - - - - org.apache.hadoop - hadoop-client-runtime - ${hadoop.version} - ${hadoop.deps.scope} - - - org.apache.hadoop - hadoop-client-minicluster - ${hadoop.version} - test - - - - org.bouncycastle - bcprov-jdk18on - test - - - org.bouncycastle - bcpkix-jdk18on - test - - - - - org.apache.spark @@ -102,6 +67,35 @@ org.apache.hadoop hadoop-client-api ${hadoop.version} + ${hadoop.deps.scope} + + + org.apache.hadoop + hadoop-client-runtime + ${hadoop.version} + ${hadoop.deps.scope} + + + org.apache.hadoop + hadoop-client-minicluster + ${hadoop.version} + test + + + + javax.xml.bind + jaxb-api + test + + + org.bouncycastle + bcprov-jdk18on + test + + + org.bouncycastle + bcpkix-jdk18on + test @@ -135,22 +129,6 @@ - - - org.eclipse.jetty.orbit - javax.servlet.jsp - 2.2.0.v201112011158 - test - - - org.eclipse.jetty.orbit - javax.servlet.jsp.jstl - 1.2.0.v201105211821 - test - - org.mockito mockito-core @@ -166,65 +144,6 @@ byte-buddy-agent test - - - - com.sun.jersey - jersey-core - test - ${jersey-1.version} - - - com.sun.jersey - jersey-json - test - ${jersey-1.version} - - - com.sun.jersey - jersey-server - test - ${jersey-1.version} - - - com.sun.jersey.contribs - jersey-guice - test - ${jersey-1.version} - - - com.sun.jersey - jersey-servlet - test - ${jersey-1.version} - - - - - ${hive.group} - hive-exec - ${hive.classifier} - provided - - - ${hive.group} - hive-metastore - provided - - - org.apache.thrift - libthrift - provided - - - org.apache.thrift - libfb303 - provided -