From d457f85057ba4c9bb422a464e6db36e3e2730a42 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 14 Aug 2023 13:55:45 +0800 Subject: [PATCH 01/15] github-repo --- pom.xml | 9 +++++++++ project/SparkBuild.scala | 3 +++ .../scala/org/apache/spark/sql/internal/SQLConf.scala | 3 +++ 3 files changed, 15 insertions(+) diff --git a/pom.xml b/pom.xml index 05c6f9841e61e..8de10621e9d8c 100644 --- a/pom.xml +++ b/pom.xml @@ -359,6 +359,15 @@ false + + + github-deploy + https://raw.github.com/pan3793/mvn-repo/release-2.3.101/ + + true + always + + diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 5bb7745d77bf2..5b524f1556b59 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -276,6 +276,9 @@ object SparkBuild extends PomBuild { DefaultMavenRepository, Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) + ) ++ Seq( + "github-deploy" at "https://raw.github.com/pan3793/mvn-repo/release-2.3.101/", + Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ), externalResolvers := resolvers.value, otherResolvers := SbtPomKeys.mvnLocalRepository(dotM2 => Seq(Resolver.file("dotM2", dotM2))).value, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index df75985043d0d..6d930772444f5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -4155,6 +4155,9 @@ object SQLConf { .stringConf .createWithDefault( sys.env.getOrElse("DEFAULT_ARTIFACT_REPOSITORY", + // scalastyle:off line.size.limit + "https://raw.github.com/pan3793/mvn-repo/release-2.3.101/," + + // scalastyle:on line.size.limit "https://maven-central.storage-download.googleapis.com/maven2/")) val LEGACY_FROM_DAYTIME_STRING = From 647b0a43d7eda6b55d75cd741437e817abcf1d2e Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 14 Aug 2023 13:52:01 +0800 Subject: [PATCH 02/15] Hive 2.3.101 --- assembly/pom.xml | 2 +- dev/deps/spark-deps-hadoop-3-hive-2.3 | 37 ++++++++++--------- dev/test-dependencies.sh | 2 +- pom.xml | 34 ++++++++--------- project/SparkBuild.scala | 2 +- .../IntervalExpressionsSuite.scala | 2 +- .../sql-tests/results/ansi/interval.sql.out | 4 +- .../sql-tests/results/interval.sql.out | 4 +- .../hive/service/auth/KerberosSaslHelper.java | 5 ++- .../hive/service/auth/PlainSaslHelper.java | 3 +- .../service/auth/TSetIpAddressProcessor.java | 5 ++- .../cli/thrift/ThriftBinaryCLIService.java | 6 --- .../service/cli/thrift/ThriftCLIService.java | 10 +++++ sql/hive/pom.xml | 4 -- .../spark/sql/hive/client/package.scala | 2 +- .../spark/sql/hive/HiveSparkSubmitSuite.scala | 6 +-- 16 files changed, 65 insertions(+), 63 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 6c31ec745b5bd..60cbe23b4f1c9 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -78,7 +78,7 @@ com.google.guava diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 5d933e34e40ba..3ef6508980b81 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -64,12 +64,14 @@ derby/10.16.1.1//derby-10.16.1.1.jar derbyshared/10.16.1.1//derbyshared-10.16.1.1.jar derbytools/10.16.1.1//derbytools-10.16.1.1.jar dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar +error_prone_annotations/2.23.0//error_prone_annotations-2.23.0.jar esdk-obs-java/3.20.4.2//esdk-obs-java-3.20.4.2.jar +failureaccess/1.0.2//failureaccess-1.0.2.jar flatbuffers-java/23.5.26//flatbuffers-java-23.5.26.jar gcs-connector/hadoop3-2.2.21/shaded/gcs-connector-hadoop3-2.2.21-shaded.jar gmetric4j/1.0.10//gmetric4j-1.0.10.jar gson/2.2.4//gson-2.2.4.jar -guava/14.0.1//guava-14.0.1.jar +guava/33.0.0-jre//guava-33.0.0-jre.jar hadoop-aliyun/3.4.0//hadoop-aliyun-3.4.0.jar hadoop-annotations/3.4.0//hadoop-annotations-3.4.0.jar hadoop-aws/3.4.0//hadoop-aws-3.4.0.jar @@ -81,19 +83,19 @@ hadoop-cloud-storage/3.4.0//hadoop-cloud-storage-3.4.0.jar hadoop-huaweicloud/3.4.0//hadoop-huaweicloud-3.4.0.jar hadoop-shaded-guava/1.2.0//hadoop-shaded-guava-1.2.0.jar hadoop-yarn-server-web-proxy/3.4.0//hadoop-yarn-server-web-proxy-3.4.0.jar -hive-beeline/2.3.9//hive-beeline-2.3.9.jar -hive-cli/2.3.9//hive-cli-2.3.9.jar -hive-common/2.3.9//hive-common-2.3.9.jar -hive-exec/2.3.9/core/hive-exec-2.3.9-core.jar -hive-jdbc/2.3.9//hive-jdbc-2.3.9.jar -hive-llap-common/2.3.9//hive-llap-common-2.3.9.jar -hive-metastore/2.3.9//hive-metastore-2.3.9.jar -hive-serde/2.3.9//hive-serde-2.3.9.jar -hive-service-rpc/4.0.0//hive-service-rpc-4.0.0.jar -hive-shims-0.23/2.3.9//hive-shims-0.23-2.3.9.jar -hive-shims-common/2.3.9//hive-shims-common-2.3.9.jar -hive-shims-scheduler/2.3.9//hive-shims-scheduler-2.3.9.jar -hive-shims/2.3.9//hive-shims-2.3.9.jar +hive-beeline/2.3.101//hive-beeline-2.3.101.jar +hive-cli/2.3.101//hive-cli-2.3.101.jar +hive-common/2.3.101//hive-common-2.3.101.jar +hive-exec/2.3.101/core/hive-exec-2.3.101-core.jar +hive-jdbc/2.3.101//hive-jdbc-2.3.101.jar +hive-llap-common/2.3.101//hive-llap-common-2.3.101.jar +hive-metastore/2.3.101//hive-metastore-2.3.101.jar +hive-serde/2.3.101//hive-serde-2.3.101.jar +hive-service-rpc/3.1.3//hive-service-rpc-3.1.3.jar +hive-shims-0.23/2.3.101//hive-shims-0.23-2.3.101.jar +hive-shims-common/2.3.101//hive-shims-common-2.3.101.jar +hive-shims-scheduler/2.3.101//hive-shims-scheduler-2.3.101.jar +hive-shims/2.3.101//hive-shims-2.3.101.jar hive-storage-api/2.8.1//hive-storage-api-2.8.1.jar hk2-api/3.0.3//hk2-api-3.0.3.jar hk2-locator/3.0.3//hk2-locator-3.0.3.jar @@ -104,14 +106,13 @@ icu4j/72.1//icu4j-72.1.jar ini4j/0.5.4//ini4j-0.5.4.jar istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar ivy/2.5.2//ivy-2.5.2.jar +j2objc-annotations/2.8//j2objc-annotations-2.8.jar jackson-annotations/2.17.0//jackson-annotations-2.17.0.jar -jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar jackson-core/2.17.0//jackson-core-2.17.0.jar jackson-databind/2.17.0//jackson-databind-2.17.0.jar jackson-dataformat-cbor/2.17.0//jackson-dataformat-cbor-2.17.0.jar jackson-dataformat-yaml/2.17.0//jackson-dataformat-yaml-2.17.0.jar jackson-datatype-jsr310/2.17.0//jackson-datatype-jsr310-2.17.0.jar -jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar jackson-module-scala_2.13/2.17.0//jackson-module-scala_2.13-2.17.0.jar jakarta.annotation-api/2.0.0//jakarta.annotation-api-2.0.0.jar jakarta.inject-api/2.0.1//jakarta.inject-api-2.0.1.jar @@ -123,6 +124,7 @@ janino/3.1.9//janino-3.1.9.jar java-diff-utils/4.12//java-diff-utils-4.12.jar java-xmlbuilder/1.2//java-xmlbuilder-1.2.jar javassist/3.29.2-GA//javassist-3.29.2-GA.jar +javax.annotation-api/1.3.2//javax.annotation-api-1.3.2.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar javax.servlet-api/4.0.1//javax.servlet-api-4.0.1.jar javolution/5.5.1//javolution-5.5.1.jar @@ -184,7 +186,8 @@ kubernetes-model-storageclass/6.12.1//kubernetes-model-storageclass-6.12.1.jar lapack/3.0.3//lapack-3.0.3.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar -libthrift/0.12.0//libthrift-0.12.0.jar +libthrift/0.16.0//libthrift-0.16.0.jar +listenablefuture/9999.0-empty-to-avoid-conflict-with-guava//listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar log4j-1.2-api/2.22.1//log4j-1.2-api-2.22.1.jar log4j-api/2.22.1//log4j-api-2.22.1.jar log4j-core/2.22.1//log4j-core-2.22.1.jar diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index 175f59a700941..197f88341a2e2 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -49,7 +49,7 @@ OLD_VERSION=$($MVN -q \ --non-recursive \ org.codehaus.mojo:exec-maven-plugin:1.6.0:exec | grep -E '[0-9]+\.[0-9]+\.[0-9]+') # dependency:get for guava and jetty-io are workaround for SPARK-37302. -GUAVA_VERSION=$(build/mvn help:evaluate -Dexpression=guava.version -q -DforceStdout | grep -E "^[0-9.]+$") +GUAVA_VERSION=$(build/mvn help:evaluate -Dexpression=guava.version -q -DforceStdout | grep -E "^[0-9\.]+") build/mvn dependency:get -Dartifact=com.google.guava:guava:${GUAVA_VERSION} -q JETTY_VERSION=$(build/mvn help:evaluate -Dexpression=jetty.version -q -DforceStdout | grep -E "[0-9]+\.[0-9]+\.[0-9]+") build/mvn dependency:get -Dartifact=org.eclipse.jetty:jetty-io:${JETTY_VERSION} -q diff --git a/pom.xml b/pom.xml index 8de10621e9d8c..8a0f4f7982606 100644 --- a/pom.xml +++ b/pom.xml @@ -132,8 +132,8 @@ org.apache.hive core - 2.3.9 - 2.3.9 + 2.3.101 + 2.3.101 2.3 @@ -199,14 +199,14 @@ 2.12.0 4.1.17 - 14.0.1 + 33.0.0-jre 3.1.9 3.0.12 2.12.7 3.5.2 3.0.0 2.2.11 - 0.12.0 + 0.16.0 4.13.1 1.1 4.17.0 @@ -1951,19 +1951,6 @@ - - - org.codehaus.jackson - jackson-core-asl - ${codehaus.jackson.version} - ${hive.jackson.scope} - - - org.codehaus.jackson - jackson-mapper-asl - ${codehaus.jackson.version} - ${hive.jackson.scope} - ${hive.group} hive-beeline @@ -2300,8 +2287,8 @@ janino - org.pentaho - pentaho-aggdesigner-algorithm + net.hydromatic + aggdesigner-algorithm @@ -2371,6 +2358,10 @@ org.codehaus.groovy groovy-all + + com.lmax + disruptor + @@ -2822,6 +2813,10 @@ org.slf4j slf4j-api + + javax.annotation + javax.annotation-api + @@ -3461,6 +3456,7 @@ org.spark-project.spark:unused com.google.guava:guava + com.google.guava:failureaccess org.jpmml:* diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 5b524f1556b59..9ed0fa3292986 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1055,7 +1055,7 @@ object KubernetesIntegrationTests { * Overrides to work around sbt's dependency resolution being different from Maven's. */ object DependencyOverrides { - lazy val guavaVersion = sys.props.get("guava.version").getOrElse("14.0.1") + lazy val guavaVersion = sys.props.get("guava.version").getOrElse("33.0.0-jre") lazy val settings = Seq( dependencyOverrides += "com.google.guava" % "guava" % guavaVersion, dependencyOverrides += "xerces" % "xercesImpl" % "2.12.2", diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala index a60ab778623eb..eed9b9aa20325 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala @@ -341,7 +341,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Seq( (Period.ofMonths(2), Int.MaxValue) -> "overflow", - (Period.ofMonths(Int.MinValue), 10d) -> "not in range", + (Period.ofMonths(Int.MinValue), 10d) -> "out of range", (Period.ofMonths(-100), Float.NaN) -> "input is infinite or NaN", (Period.ofMonths(200), Double.PositiveInfinity) -> "input is infinite or NaN", (Period.ofMonths(-200), Float.NegativeInfinity) -> "input is infinite or NaN" diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out index 4e220ba9885c1..08afa0a5297d1 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out @@ -2866,7 +2866,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1.0D struct<> -- !query output java.lang.ArithmeticException -not in range +rounded value is out of range for input 2.147483648E9 and rounding mode HALF_UP -- !query @@ -2946,7 +2946,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1.0D struct<> -- !query output java.lang.ArithmeticException -not in range +rounded value is out of range for input 9.223372036854776E18 and rounding mode HALF_UP -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index e2b9e11eb6331..a701f4c3c18e5 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -2679,7 +2679,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1.0D struct<> -- !query output java.lang.ArithmeticException -not in range +rounded value is out of range for input 2.147483648E9 and rounding mode HALF_UP -- !query @@ -2759,7 +2759,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1.0D struct<> -- !query output java.lang.ArithmeticException -not in range +rounded value is out of range for input 9.223372036854776E18 and rounding mode HALF_UP -- !query diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java index 175412ed98c6c..ef91f94eeec2b 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java @@ -30,6 +30,7 @@ import org.apache.thrift.TProcessorFactory; import org.apache.thrift.transport.TSaslClientTransport; import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; public final class KerberosSaslHelper { @@ -68,8 +69,8 @@ public static TTransport createSubjectAssumedTransport(String principal, new TSaslClientTransport("GSSAPI", null, names[0], names[1], saslProps, null, underlyingTransport); return new TSubjectAssumingTransport(saslTransport); - } catch (SaslException se) { - throw new IOException("Could not instantiate SASL transport", se); + } catch (SaslException | TTransportException se) { + throw new IOException("Could not instantiate transport", se); } } diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java index c06f6ec34653f..5ac29950f4f85 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java @@ -38,6 +38,7 @@ import org.apache.thrift.transport.TSaslClientTransport; import org.apache.thrift.transport.TSaslServerTransport; import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; import org.apache.thrift.transport.TTransportFactory; public final class PlainSaslHelper { @@ -64,7 +65,7 @@ public static TTransportFactory getPlainTransportFactory(String authTypeStr) } public static TTransport getPlainTransport(String username, String password, - TTransport underlyingTransport) throws SaslException { + TTransport underlyingTransport) throws SaslException, TTransportException { return new TSaslClientTransport("PLAIN", null, null, null, new HashMap(), new PlainCallbackHandler(username, password), underlyingTransport); } diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java index 1205d21be6be6..b727b4e27de8d 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java @@ -45,11 +45,12 @@ public TSetIpAddressProcessor(Iface iface) { } @Override - public boolean process(final TProtocol in, final TProtocol out) throws TException { + public void process(final TProtocol in, final TProtocol out) throws TException { setIpAddress(in); setUserName(in); try { - return super.process(in, out); + super.process(in, out); + return; } finally { THREAD_LOCAL_USER_NAME.remove(); THREAD_LOCAL_IP_ADDRESS.remove(); diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java index 4d99496876fdc..c7fa7b5f3e0ac 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java @@ -91,16 +91,10 @@ protected void initializeServer() { // Server args int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE); - int requestTimeout = (int) hiveConf.getTimeVar( - HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_TIMEOUT, TimeUnit.SECONDS); - int beBackoffSlotLength = (int) hiveConf.getTimeVar( - HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_BEBACKOFF_SLOT_LENGTH, TimeUnit.MILLISECONDS); TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket) .processorFactory(processorFactory).transportFactory(transportFactory) .protocolFactory(new TBinaryProtocol.Factory()) .inputProtocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize)) - .requestTimeout(requestTimeout).requestTimeoutUnit(TimeUnit.SECONDS) - .beBackoffSlotLength(beBackoffSlotLength).beBackoffSlotLengthUnit(TimeUnit.MILLISECONDS) .executorService(executorService); // TCP Server diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java index 4b18e2950a3de..ac0a6d59bee8e 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java @@ -83,6 +83,16 @@ public void setSessionHandle(SessionHandle sessionHandle) { public SessionHandle getSessionHandle() { return sessionHandle; } + + @Override + public T unwrap(Class aClass) { + return null; + } + + @Override + public boolean isWrapperFor(Class aClass) { + return false; + } } public ThriftCLIService(CLIService service, String serviceName) { diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 3895d9dc5a634..eba61e172fb3f 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -135,10 +135,6 @@ org.apache.httpcomponents httpclient - - org.codehaus.jackson - jackson-mapper-asl - javax.servlet diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala index 564c87a0fca8e..00cfd579a1770 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala @@ -59,7 +59,7 @@ package object client { "org.pentaho:pentaho-aggdesigner-algorithm")) // Since HIVE-23980, calcite-core included in Hive package jar. - case object v2_3 extends HiveVersion("2.3.9", + case object v2_3 extends HiveVersion("2.3.101", exclusions = Seq("org.apache.calcite:calcite-core", "org.apache.calcite:calcite-druid", "org.apache.calcite.avatica:avatica", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index c7aa412959097..de39575cbad4c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -387,7 +387,7 @@ object SetMetastoreURLTest extends Logging { val builder = SparkSession.builder() .config(sparkConf) .config(UI_ENABLED.key, "false") - .config(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.9") + .config(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.101") // The issue described in SPARK-16901 only appear when // spark.sql.hive.metastore.jars is not set to builtin. .config(HiveUtils.HIVE_METASTORE_JARS.key, "maven") @@ -698,7 +698,7 @@ object SparkSQLConfTest extends Logging { val filteredSettings = super.getAll.filterNot(e => isMetastoreSetting(e._1)) // Always add these two metastore settings at the beginning. - (HiveUtils.HIVE_METASTORE_VERSION.key -> "2.3.9") +: + (HiveUtils.HIVE_METASTORE_VERSION.key -> "2.3.101") +: (HiveUtils.HIVE_METASTORE_JARS.key -> "maven") +: filteredSettings } @@ -726,7 +726,7 @@ object SPARK_9757 extends QueryTest { val hiveWarehouseLocation = Utils.createTempDir() val sparkContext = new SparkContext( new SparkConf() - .set(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.9") + .set(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.101") .set(HiveUtils.HIVE_METASTORE_JARS.key, "maven") .set(UI_ENABLED, false) .set(WAREHOUSE_PATH.key, hiveWarehouseLocation.toString)) From f504753f214527dfe9ef8dcd0ff73e48154f4a66 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sun, 3 Mar 2024 16:04:05 +0800 Subject: [PATCH 03/15] jackson --- .../main/scala/org/apache/spark/sql/hive/client/package.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala index 00cfd579a1770..995237d72d24b 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala @@ -63,7 +63,6 @@ package object client { exclusions = Seq("org.apache.calcite:calcite-core", "org.apache.calcite:calcite-druid", "org.apache.calcite.avatica:avatica", - "com.fasterxml.jackson.core:*", "org.apache.curator:*", "org.pentaho:pentaho-aggdesigner-algorithm", "org.apache.hive:hive-vector-code-gen")) From 2ecdfd8236e06f1af0506c6322a6f86f2840b338 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 4 Mar 2024 08:18:07 +0800 Subject: [PATCH 04/15] Bump Hive 2.3.102 and exclude javax.servlet --- connector/connect/server/pom.xml | 4 --- core/pom.xml | 4 --- dev/deps/spark-deps-hadoop-3-hive-2.3 | 25 +++++++++---------- pom.xml | 13 +++------- project/SparkBuild.scala | 2 +- .../apache/spark/sql/internal/SQLConf.scala | 2 +- sql/hive-thriftserver/pom.xml | 5 ---- sql/hive/pom.xml | 4 --- .../spark/sql/hive/client/package.scala | 2 +- .../spark/sql/hive/HiveSparkSubmitSuite.scala | 6 ++--- 10 files changed, 21 insertions(+), 46 deletions(-) diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml index f093f4f19622a..4ce31c2c4597f 100644 --- a/connector/connect/server/pom.xml +++ b/connector/connect/server/pom.xml @@ -160,10 +160,6 @@ jakarta.servlet jakarta.servlet-api - - javax.servlet - javax.servlet-api - com.google.guava guava diff --git a/core/pom.xml b/core/pom.xml index 6468f500db046..ef9f74e51edde 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -166,10 +166,6 @@ jetty-servlets compile - - javax.servlet - javax.servlet-api - jakarta.servlet jakarta.servlet-api diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 3ef6508980b81..ac01b43d2cf7c 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -83,19 +83,19 @@ hadoop-cloud-storage/3.4.0//hadoop-cloud-storage-3.4.0.jar hadoop-huaweicloud/3.4.0//hadoop-huaweicloud-3.4.0.jar hadoop-shaded-guava/1.2.0//hadoop-shaded-guava-1.2.0.jar hadoop-yarn-server-web-proxy/3.4.0//hadoop-yarn-server-web-proxy-3.4.0.jar -hive-beeline/2.3.101//hive-beeline-2.3.101.jar -hive-cli/2.3.101//hive-cli-2.3.101.jar -hive-common/2.3.101//hive-common-2.3.101.jar -hive-exec/2.3.101/core/hive-exec-2.3.101-core.jar -hive-jdbc/2.3.101//hive-jdbc-2.3.101.jar -hive-llap-common/2.3.101//hive-llap-common-2.3.101.jar -hive-metastore/2.3.101//hive-metastore-2.3.101.jar -hive-serde/2.3.101//hive-serde-2.3.101.jar +hive-beeline/2.3.102//hive-beeline-2.3.102.jar +hive-cli/2.3.102//hive-cli-2.3.102.jar +hive-common/2.3.102//hive-common-2.3.102.jar +hive-exec/2.3.102/core/hive-exec-2.3.102-core.jar +hive-jdbc/2.3.102//hive-jdbc-2.3.102.jar +hive-llap-common/2.3.102//hive-llap-common-2.3.102.jar +hive-metastore/2.3.102//hive-metastore-2.3.102.jar +hive-serde/2.3.102//hive-serde-2.3.102.jar hive-service-rpc/3.1.3//hive-service-rpc-3.1.3.jar -hive-shims-0.23/2.3.101//hive-shims-0.23-2.3.101.jar -hive-shims-common/2.3.101//hive-shims-common-2.3.101.jar -hive-shims-scheduler/2.3.101//hive-shims-scheduler-2.3.101.jar -hive-shims/2.3.101//hive-shims-2.3.101.jar +hive-shims-0.23/2.3.102//hive-shims-0.23-2.3.102.jar +hive-shims-common/2.3.102//hive-shims-common-2.3.102.jar +hive-shims-scheduler/2.3.102//hive-shims-scheduler-2.3.102.jar +hive-shims/2.3.102//hive-shims-2.3.102.jar hive-storage-api/2.8.1//hive-storage-api-2.8.1.jar hk2-api/3.0.3//hk2-api-3.0.3.jar hk2-locator/3.0.3//hk2-locator-3.0.3.jar @@ -126,7 +126,6 @@ java-xmlbuilder/1.2//java-xmlbuilder-1.2.jar javassist/3.29.2-GA//javassist-3.29.2-GA.jar javax.annotation-api/1.3.2//javax.annotation-api-1.3.2.jar javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar -javax.servlet-api/4.0.1//javax.servlet-api-4.0.1.jar javolution/5.5.1//javolution-5.5.1.jar jaxb-api/2.2.11//jaxb-api-2.2.11.jar jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar diff --git a/pom.xml b/pom.xml index 8a0f4f7982606..ec21d4a9dc08c 100644 --- a/pom.xml +++ b/pom.xml @@ -132,8 +132,8 @@ org.apache.hive core - 2.3.101 - 2.3.101 + 2.3.102 + 2.3.102 2.3 @@ -145,8 +145,6 @@ shaded-protobuf 11.0.20 5.0.0 - - 4.0.1 0.10.0 2.5.2 2.0.8 @@ -362,7 +360,7 @@ github-deploy - https://raw.github.com/pan3793/mvn-repo/release-2.3.101/ + https://raw.github.com/pan3793/mvn-repo/release-2.3.102/ true always @@ -2937,11 +2935,6 @@ jakarta.servlet-api ${jakartaservlet.version} - - javax.servlet - javax.servlet-api - ${javaxservlet.version} - diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 9ed0fa3292986..1380ac48b2195 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -277,7 +277,7 @@ object SparkBuild extends PomBuild { Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ) ++ Seq( - "github-deploy" at "https://raw.github.com/pan3793/mvn-repo/release-2.3.101/", + "github-deploy" at "https://raw.github.com/pan3793/mvn-repo/release-2.3.102/", Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ), externalResolvers := resolvers.value, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 6d930772444f5..965eac5c151df 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -4156,7 +4156,7 @@ object SQLConf { .createWithDefault( sys.env.getOrElse("DEFAULT_ARTIFACT_REPOSITORY", // scalastyle:off line.size.limit - "https://raw.github.com/pan3793/mvn-repo/release-2.3.101/," + + "https://raw.github.com/pan3793/mvn-repo/release-2.3.102/," + // scalastyle:on line.size.limit "https://maven-central.storage-download.googleapis.com/maven2/")) diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 6a352f8a530d7..176d258915491 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -95,11 +95,6 @@ jetty-servlet provided - - javax.servlet - javax.servlet-api - provided - org.seleniumhq.selenium diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index eba61e172fb3f..8861736cd978d 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -136,10 +136,6 @@ httpclient - - javax.servlet - javax.servlet-api - commons-codec commons-codec diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala index 995237d72d24b..e8eff2ace2e3d 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala @@ -59,7 +59,7 @@ package object client { "org.pentaho:pentaho-aggdesigner-algorithm")) // Since HIVE-23980, calcite-core included in Hive package jar. - case object v2_3 extends HiveVersion("2.3.101", + case object v2_3 extends HiveVersion("2.3.102", exclusions = Seq("org.apache.calcite:calcite-core", "org.apache.calcite:calcite-druid", "org.apache.calcite.avatica:avatica", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index de39575cbad4c..8889999e6c96b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -387,7 +387,7 @@ object SetMetastoreURLTest extends Logging { val builder = SparkSession.builder() .config(sparkConf) .config(UI_ENABLED.key, "false") - .config(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.101") + .config(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.102") // The issue described in SPARK-16901 only appear when // spark.sql.hive.metastore.jars is not set to builtin. .config(HiveUtils.HIVE_METASTORE_JARS.key, "maven") @@ -698,7 +698,7 @@ object SparkSQLConfTest extends Logging { val filteredSettings = super.getAll.filterNot(e => isMetastoreSetting(e._1)) // Always add these two metastore settings at the beginning. - (HiveUtils.HIVE_METASTORE_VERSION.key -> "2.3.101") +: + (HiveUtils.HIVE_METASTORE_VERSION.key -> "2.3.102") +: (HiveUtils.HIVE_METASTORE_JARS.key -> "maven") +: filteredSettings } @@ -726,7 +726,7 @@ object SPARK_9757 extends QueryTest { val hiveWarehouseLocation = Utils.createTempDir() val sparkContext = new SparkContext( new SparkConf() - .set(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.101") + .set(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.102") .set(HiveUtils.HIVE_METASTORE_JARS.key, "maven") .set(UI_ENABLED, false) .set(WAREHOUSE_PATH.key, hiveWarehouseLocation.toString)) From 59323ceabb017d655e6ee4341fdd400b44f4e2e0 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 29 Mar 2024 13:48:12 +0800 Subject: [PATCH 05/15] Hive 2.3.10-3 --- dev/deps/spark-deps-hadoop-3-hive-2.3 | 24 +++++++++---------- pom.xml | 6 ++--- project/SparkBuild.scala | 2 +- .../apache/spark/sql/internal/SQLConf.scala | 2 +- .../spark/sql/hive/client/package.scala | 2 +- .../spark/sql/hive/HiveSparkSubmitSuite.scala | 6 ++--- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index ac01b43d2cf7c..011df052a46d4 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -83,19 +83,19 @@ hadoop-cloud-storage/3.4.0//hadoop-cloud-storage-3.4.0.jar hadoop-huaweicloud/3.4.0//hadoop-huaweicloud-3.4.0.jar hadoop-shaded-guava/1.2.0//hadoop-shaded-guava-1.2.0.jar hadoop-yarn-server-web-proxy/3.4.0//hadoop-yarn-server-web-proxy-3.4.0.jar -hive-beeline/2.3.102//hive-beeline-2.3.102.jar -hive-cli/2.3.102//hive-cli-2.3.102.jar -hive-common/2.3.102//hive-common-2.3.102.jar -hive-exec/2.3.102/core/hive-exec-2.3.102-core.jar -hive-jdbc/2.3.102//hive-jdbc-2.3.102.jar -hive-llap-common/2.3.102//hive-llap-common-2.3.102.jar -hive-metastore/2.3.102//hive-metastore-2.3.102.jar -hive-serde/2.3.102//hive-serde-2.3.102.jar +hive-beeline/2.3.103//hive-beeline-2.3.103.jar +hive-cli/2.3.103//hive-cli-2.3.103.jar +hive-common/2.3.103//hive-common-2.3.103.jar +hive-exec/2.3.103/core/hive-exec-2.3.103-core.jar +hive-jdbc/2.3.103//hive-jdbc-2.3.103.jar +hive-llap-common/2.3.103//hive-llap-common-2.3.103.jar +hive-metastore/2.3.103//hive-metastore-2.3.103.jar +hive-serde/2.3.103//hive-serde-2.3.103.jar hive-service-rpc/3.1.3//hive-service-rpc-3.1.3.jar -hive-shims-0.23/2.3.102//hive-shims-0.23-2.3.102.jar -hive-shims-common/2.3.102//hive-shims-common-2.3.102.jar -hive-shims-scheduler/2.3.102//hive-shims-scheduler-2.3.102.jar -hive-shims/2.3.102//hive-shims-2.3.102.jar +hive-shims-0.23/2.3.103//hive-shims-0.23-2.3.103.jar +hive-shims-common/2.3.103//hive-shims-common-2.3.103.jar +hive-shims-scheduler/2.3.103//hive-shims-scheduler-2.3.103.jar +hive-shims/2.3.103//hive-shims-2.3.103.jar hive-storage-api/2.8.1//hive-storage-api-2.8.1.jar hk2-api/3.0.3//hk2-api-3.0.3.jar hk2-locator/3.0.3//hk2-locator-3.0.3.jar diff --git a/pom.xml b/pom.xml index ec21d4a9dc08c..d213662352779 100644 --- a/pom.xml +++ b/pom.xml @@ -132,8 +132,8 @@ org.apache.hive core - 2.3.102 - 2.3.102 + 2.3.103 + 2.3.103 2.3 @@ -360,7 +360,7 @@ github-deploy - https://raw.github.com/pan3793/mvn-repo/release-2.3.102/ + https://raw.github.com/pan3793/mvn-repo/release-2.3.103/ true always diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 1380ac48b2195..a60b55390aaa3 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -277,7 +277,7 @@ object SparkBuild extends PomBuild { Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ) ++ Seq( - "github-deploy" at "https://raw.github.com/pan3793/mvn-repo/release-2.3.102/", + "github-deploy" at "https://raw.github.com/pan3793/mvn-repo/release-2.3.103/", Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ), externalResolvers := resolvers.value, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 965eac5c151df..9d58791ea86d9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -4156,7 +4156,7 @@ object SQLConf { .createWithDefault( sys.env.getOrElse("DEFAULT_ARTIFACT_REPOSITORY", // scalastyle:off line.size.limit - "https://raw.github.com/pan3793/mvn-repo/release-2.3.102/," + + "https://raw.github.com/pan3793/mvn-repo/release-2.3.103/," + // scalastyle:on line.size.limit "https://maven-central.storage-download.googleapis.com/maven2/")) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala index e8eff2ace2e3d..21afd707e54b7 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala @@ -59,7 +59,7 @@ package object client { "org.pentaho:pentaho-aggdesigner-algorithm")) // Since HIVE-23980, calcite-core included in Hive package jar. - case object v2_3 extends HiveVersion("2.3.102", + case object v2_3 extends HiveVersion("2.3.103", exclusions = Seq("org.apache.calcite:calcite-core", "org.apache.calcite:calcite-druid", "org.apache.calcite.avatica:avatica", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index 8889999e6c96b..aecbd7e940bf9 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -387,7 +387,7 @@ object SetMetastoreURLTest extends Logging { val builder = SparkSession.builder() .config(sparkConf) .config(UI_ENABLED.key, "false") - .config(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.102") + .config(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.103") // The issue described in SPARK-16901 only appear when // spark.sql.hive.metastore.jars is not set to builtin. .config(HiveUtils.HIVE_METASTORE_JARS.key, "maven") @@ -698,7 +698,7 @@ object SparkSQLConfTest extends Logging { val filteredSettings = super.getAll.filterNot(e => isMetastoreSetting(e._1)) // Always add these two metastore settings at the beginning. - (HiveUtils.HIVE_METASTORE_VERSION.key -> "2.3.102") +: + (HiveUtils.HIVE_METASTORE_VERSION.key -> "2.3.103") +: (HiveUtils.HIVE_METASTORE_JARS.key -> "maven") +: filteredSettings } @@ -726,7 +726,7 @@ object SPARK_9757 extends QueryTest { val hiveWarehouseLocation = Utils.createTempDir() val sparkContext = new SparkContext( new SparkConf() - .set(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.102") + .set(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.103") .set(HiveUtils.HIVE_METASTORE_JARS.key, "maven") .set(UI_ENABLED, false) .set(WAREHOUSE_PATH.key, hiveWarehouseLocation.toString)) From c8ca897c102bff09ae40880a6b9a2fe666cfbae6 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 29 Mar 2024 13:52:24 +0800 Subject: [PATCH 06/15] remove jodd --- LICENSE-binary | 1 - dev/deps/spark-deps-hadoop-3-hive-2.3 | 1 - licenses-binary/LICENSE-jodd.txt | 24 ------------------------ pom.xml | 6 ------ sql/hive/pom.xml | 4 ---- 5 files changed, 36 deletions(-) delete mode 100644 licenses-binary/LICENSE-jodd.txt diff --git a/LICENSE-binary b/LICENSE-binary index 40271c9924bc4..034215f0ab157 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -436,7 +436,6 @@ com.esotericsoftware:reflectasm org.codehaus.janino:commons-compiler org.codehaus.janino:janino jline:jline -org.jodd:jodd-core com.github.wendykierp:JTransforms pl.edu.icm:JLargeArrays diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 011df052a46d4..6afd40e727541 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -145,7 +145,6 @@ jline/2.14.6//jline-2.14.6.jar jline/3.24.1//jline-3.24.1.jar jna/5.13.0//jna-5.13.0.jar joda-time/2.12.7//joda-time-2.12.7.jar -jodd-core/3.5.2//jodd-core-3.5.2.jar jpam/1.1//jpam-1.1.jar json/1.8//json-1.8.jar json4s-ast_2.13/4.0.7//json4s-ast_2.13-4.0.7.jar diff --git a/licenses-binary/LICENSE-jodd.txt b/licenses-binary/LICENSE-jodd.txt deleted file mode 100644 index cc6b458adb386..0000000000000 --- a/licenses-binary/LICENSE-jodd.txt +++ /dev/null @@ -1,24 +0,0 @@ -Copyright (c) 2003-present, Jodd Team (https://jodd.org) -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, -this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/pom.xml b/pom.xml index d213662352779..9b93be29925e5 100644 --- a/pom.xml +++ b/pom.xml @@ -201,7 +201,6 @@ 3.1.9 3.0.12 2.12.7 - 3.5.2 3.0.0 2.2.11 0.16.0 @@ -2781,11 +2780,6 @@ joda-time ${joda.version} - - org.jodd - jodd-core - ${jodd.version} - org.datanucleus datanucleus-core diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 8861736cd978d..fa8de0d95d4b5 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -144,10 +144,6 @@ joda-time joda-time - - org.jodd - jodd-core - com.google.code.findbugs jsr305 From eb7cc46a17f984d96b1d29d1286e5e0130cd9b64 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sun, 21 Apr 2024 12:06:38 +0800 Subject: [PATCH 07/15] Apache Hive 2.3.10 RC0 --- dev/deps/spark-deps-hadoop-3-hive-2.3 | 27 +++++++++---------- pom.xml | 8 +++--- project/SparkBuild.scala | 2 +- .../apache/spark/sql/internal/SQLConf.scala | 2 +- .../spark/sql/hive/client/package.scala | 2 +- .../spark/sql/hive/HiveSparkSubmitSuite.scala | 6 ++--- 6 files changed, 23 insertions(+), 24 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 6afd40e727541..ba79e7d08113e 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -46,7 +46,6 @@ commons-compress/1.26.1//commons-compress-1.26.1.jar commons-crypto/1.1.0//commons-crypto-1.1.0.jar commons-dbcp/1.4//commons-dbcp-1.4.jar commons-io/2.16.1//commons-io-2.16.1.jar -commons-lang/2.6//commons-lang-2.6.jar commons-lang3/3.14.0//commons-lang3-3.14.0.jar commons-math3/3.6.1//commons-math3-3.6.1.jar commons-pool/1.5.4//commons-pool-1.5.4.jar @@ -83,19 +82,19 @@ hadoop-cloud-storage/3.4.0//hadoop-cloud-storage-3.4.0.jar hadoop-huaweicloud/3.4.0//hadoop-huaweicloud-3.4.0.jar hadoop-shaded-guava/1.2.0//hadoop-shaded-guava-1.2.0.jar hadoop-yarn-server-web-proxy/3.4.0//hadoop-yarn-server-web-proxy-3.4.0.jar -hive-beeline/2.3.103//hive-beeline-2.3.103.jar -hive-cli/2.3.103//hive-cli-2.3.103.jar -hive-common/2.3.103//hive-common-2.3.103.jar -hive-exec/2.3.103/core/hive-exec-2.3.103-core.jar -hive-jdbc/2.3.103//hive-jdbc-2.3.103.jar -hive-llap-common/2.3.103//hive-llap-common-2.3.103.jar -hive-metastore/2.3.103//hive-metastore-2.3.103.jar -hive-serde/2.3.103//hive-serde-2.3.103.jar -hive-service-rpc/3.1.3//hive-service-rpc-3.1.3.jar -hive-shims-0.23/2.3.103//hive-shims-0.23-2.3.103.jar -hive-shims-common/2.3.103//hive-shims-common-2.3.103.jar -hive-shims-scheduler/2.3.103//hive-shims-scheduler-2.3.103.jar -hive-shims/2.3.103//hive-shims-2.3.103.jar +hive-beeline/2.3.10//hive-beeline-2.3.10.jar +hive-cli/2.3.10//hive-cli-2.3.10.jar +hive-common/2.3.10//hive-common-2.3.10.jar +hive-exec/2.3.10/core/hive-exec-2.3.10-core.jar +hive-jdbc/2.3.10//hive-jdbc-2.3.10.jar +hive-llap-common/2.3.10//hive-llap-common-2.3.10.jar +hive-metastore/2.3.10//hive-metastore-2.3.10.jar +hive-serde/2.3.10//hive-serde-2.3.10.jar +hive-service-rpc/4.0.0//hive-service-rpc-4.0.0.jar +hive-shims-0.23/2.3.10//hive-shims-0.23-2.3.10.jar +hive-shims-common/2.3.10//hive-shims-common-2.3.10.jar +hive-shims-scheduler/2.3.10//hive-shims-scheduler-2.3.10.jar +hive-shims/2.3.10//hive-shims-2.3.10.jar hive-storage-api/2.8.1//hive-storage-api-2.8.1.jar hk2-api/3.0.3//hk2-api-3.0.3.jar hk2-locator/3.0.3//hk2-locator-3.0.3.jar diff --git a/pom.xml b/pom.xml index 9b93be29925e5..32d9bcea1f5ad 100644 --- a/pom.xml +++ b/pom.xml @@ -132,8 +132,8 @@ org.apache.hive core - 2.3.103 - 2.3.103 + 2.3.10 + 2.3.10 2.3 @@ -358,8 +358,8 @@ - github-deploy - https://raw.github.com/pan3793/mvn-repo/release-2.3.103/ + hive-staging-repo + https://repository.apache.org/content/repositories/orgapachehive-1128/ true always diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index a60b55390aaa3..9c2cfcbb3ba1c 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -277,7 +277,7 @@ object SparkBuild extends PomBuild { Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ) ++ Seq( - "github-deploy" at "https://raw.github.com/pan3793/mvn-repo/release-2.3.103/", + "hive-staging-repo" at "https://repository.apache.org/content/repositories/orgapachehive-1128/", Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ), externalResolvers := resolvers.value, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 9d58791ea86d9..f6dc095fa4310 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -4156,7 +4156,7 @@ object SQLConf { .createWithDefault( sys.env.getOrElse("DEFAULT_ARTIFACT_REPOSITORY", // scalastyle:off line.size.limit - "https://raw.github.com/pan3793/mvn-repo/release-2.3.103/," + + "https://repository.apache.org/content/repositories/orgapachehive-1128/," + // scalastyle:on line.size.limit "https://maven-central.storage-download.googleapis.com/maven2/")) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala index 21afd707e54b7..81d75649733a2 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala @@ -59,7 +59,7 @@ package object client { "org.pentaho:pentaho-aggdesigner-algorithm")) // Since HIVE-23980, calcite-core included in Hive package jar. - case object v2_3 extends HiveVersion("2.3.103", + case object v2_3 extends HiveVersion("2.3.10", exclusions = Seq("org.apache.calcite:calcite-core", "org.apache.calcite:calcite-druid", "org.apache.calcite.avatica:avatica", diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index aecbd7e940bf9..8df8a7b8fbbb5 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -387,7 +387,7 @@ object SetMetastoreURLTest extends Logging { val builder = SparkSession.builder() .config(sparkConf) .config(UI_ENABLED.key, "false") - .config(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.103") + .config(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.10") // The issue described in SPARK-16901 only appear when // spark.sql.hive.metastore.jars is not set to builtin. .config(HiveUtils.HIVE_METASTORE_JARS.key, "maven") @@ -698,7 +698,7 @@ object SparkSQLConfTest extends Logging { val filteredSettings = super.getAll.filterNot(e => isMetastoreSetting(e._1)) // Always add these two metastore settings at the beginning. - (HiveUtils.HIVE_METASTORE_VERSION.key -> "2.3.103") +: + (HiveUtils.HIVE_METASTORE_VERSION.key -> "2.3.10") +: (HiveUtils.HIVE_METASTORE_JARS.key -> "maven") +: filteredSettings } @@ -726,7 +726,7 @@ object SPARK_9757 extends QueryTest { val hiveWarehouseLocation = Utils.createTempDir() val sparkContext = new SparkContext( new SparkConf() - .set(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.103") + .set(HiveUtils.HIVE_METASTORE_VERSION.key, "2.3.10") .set(HiveUtils.HIVE_METASTORE_JARS.key, "maven") .set(UI_ENABLED, false) .set(WAREHOUSE_PATH.key, hiveWarehouseLocation.toString)) From e191b99f3c85c74021ca703fb29f89d298297e09 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sun, 21 Apr 2024 12:18:44 +0800 Subject: [PATCH 08/15] guava --- core/pom.xml | 1 + dev/deps/spark-deps-hadoop-3-hive-2.3 | 4 ++-- pom.xml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index ef9f74e51edde..332c89e414901 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -542,6 +542,7 @@ org.eclipse.jetty:jetty-util org.eclipse.jetty:jetty-server com.google.guava:guava + com.google.guava:failureaccess com.google.protobuf:* diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index ba79e7d08113e..6a33c58a11575 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -70,7 +70,7 @@ flatbuffers-java/23.5.26//flatbuffers-java-23.5.26.jar gcs-connector/hadoop3-2.2.21/shaded/gcs-connector-hadoop3-2.2.21-shaded.jar gmetric4j/1.0.10//gmetric4j-1.0.10.jar gson/2.2.4//gson-2.2.4.jar -guava/33.0.0-jre//guava-33.0.0-jre.jar +guava/33.1.0-jre//guava-33.1.0-jre.jar hadoop-aliyun/3.4.0//hadoop-aliyun-3.4.0.jar hadoop-annotations/3.4.0//hadoop-annotations-3.4.0.jar hadoop-aws/3.4.0//hadoop-aws-3.4.0.jar @@ -105,7 +105,7 @@ icu4j/72.1//icu4j-72.1.jar ini4j/0.5.4//ini4j-0.5.4.jar istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar ivy/2.5.2//ivy-2.5.2.jar -j2objc-annotations/2.8//j2objc-annotations-2.8.jar +j2objc-annotations/3.0.0//j2objc-annotations-3.0.0.jar jackson-annotations/2.17.0//jackson-annotations-2.17.0.jar jackson-core/2.17.0//jackson-core-2.17.0.jar jackson-databind/2.17.0//jackson-databind-2.17.0.jar diff --git a/pom.xml b/pom.xml index 32d9bcea1f5ad..f49929a24ca30 100644 --- a/pom.xml +++ b/pom.xml @@ -197,7 +197,7 @@ 2.12.0 4.1.17 - 33.0.0-jre + 33.1.0-jre 3.1.9 3.0.12 2.12.7 From 05c511ebeb92e4ed77b3a4bf455035aa15f4b721 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sun, 21 Apr 2024 13:00:42 +0800 Subject: [PATCH 09/15] Remove codehaus.jackson.version --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index f49929a24ca30..d0719a7f1a7d3 100644 --- a/pom.xml +++ b/pom.xml @@ -180,7 +180,6 @@ true true true - 1.9.13 2.17.0 2.17.0 2.3.1 From d7619bb41982d1312bed59884a0d2012c43da287 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sun, 21 Apr 2024 12:29:21 +0800 Subject: [PATCH 10/15] Remove commons-lang definition --- connector/kafka-0-10-assembly/pom.xml | 5 ----- connector/kinesis-asl-assembly/pom.xml | 5 ----- pom.xml | 13 ------------- 3 files changed, 23 deletions(-) diff --git a/connector/kafka-0-10-assembly/pom.xml b/connector/kafka-0-10-assembly/pom.xml index b2fcbdf8eca7d..bd311b3a98047 100644 --- a/connector/kafka-0-10-assembly/pom.xml +++ b/connector/kafka-0-10-assembly/pom.xml @@ -54,11 +54,6 @@ commons-codec provided - - commons-lang - commons-lang - provided - com.google.protobuf protobuf-java diff --git a/connector/kinesis-asl-assembly/pom.xml b/connector/kinesis-asl-assembly/pom.xml index 577ec21530837..0e93526fce721 100644 --- a/connector/kinesis-asl-assembly/pom.xml +++ b/connector/kinesis-asl-assembly/pom.xml @@ -54,11 +54,6 @@ jackson-databind provided - - commons-lang - commons-lang - provided - org.glassfish.jersey.core jersey-client diff --git a/pom.xml b/pom.xml index d0719a7f1a7d3..226ceeb049304 100644 --- a/pom.xml +++ b/pom.xml @@ -189,8 +189,6 @@ 1.17.0 1.26.1 2.16.1 - - 2.6 3.14.0 @@ -617,11 +615,6 @@ commons-text 1.12.0 - - commons-lang - commons-lang - ${commons-lang2.version} - commons-io commons-io @@ -2890,12 +2883,6 @@ hive-storage-api ${hive.storage.version} ${hive.storage.scope} - - - commons-lang - commons-lang - - commons-cli From 32d481fa67b66583606149dfe3a681dd37953466 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sun, 21 Apr 2024 12:33:40 +0800 Subject: [PATCH 11/15] aggdesigner-algorithm --- .../main/scala/org/apache/spark/sql/hive/client/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala index 81d75649733a2..d172af21a9170 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala @@ -64,7 +64,7 @@ package object client { "org.apache.calcite:calcite-druid", "org.apache.calcite.avatica:avatica", "org.apache.curator:*", - "org.pentaho:pentaho-aggdesigner-algorithm", + "net.hydromatic:aggdesigner-algorithm", "org.apache.hive:hive-vector-code-gen")) // Since Hive 3.0, HookUtils uses org.apache.logging.log4j.util.Strings From 613eb85fd3bf1faeb0b6318300e6cdc433494305 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sun, 21 Apr 2024 12:36:29 +0800 Subject: [PATCH 12/15] guava --- assembly/pom.xml | 2 +- project/SparkBuild.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 60cbe23b4f1c9..3fdf46bc37f39 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -78,7 +78,7 @@ com.google.guava diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 9c2cfcbb3ba1c..cab015bf9c62f 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1055,7 +1055,7 @@ object KubernetesIntegrationTests { * Overrides to work around sbt's dependency resolution being different from Maven's. */ object DependencyOverrides { - lazy val guavaVersion = sys.props.get("guava.version").getOrElse("33.0.0-jre") + lazy val guavaVersion = sys.props.get("guava.version").getOrElse("33.1.0-jre") lazy val settings = Seq( dependencyOverrides += "com.google.guava" % "guava" % guavaVersion, dependencyOverrides += "xerces" % "xercesImpl" % "2.12.2", From 71e8f936b357b5c9f3a434105715cb40bd322319 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 4 Mar 2024 05:28:03 +0800 Subject: [PATCH 13/15] ADD JAR --- .../src/main/scala/org/apache/spark/util/MavenUtils.scala | 7 +++++++ .../test/scala/org/apache/spark/util/MavenUtilsSuite.scala | 3 ++- .../apache/spark/sql/hive/execution/HiveQuerySuite.scala | 6 ++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala index 08291859a32cc..0835976d56891 100644 --- a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala +++ b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala @@ -192,6 +192,13 @@ private[spark] object MavenUtils extends Logging { sys.env.getOrElse("DEFAULT_ARTIFACT_REPOSITORY", "https://repos.spark-packages.org/")) sp.setName("spark-packages") cr.add(sp) + + val staging: IBiblioResolver = new IBiblioResolver + staging.setM2compatible(true) + staging.setUsepoms(true) + staging.setRoot("https://repository.apache.org/content/repositories/orgapachehive-1128/") + staging.setName("hive-staging-repo") + cr.add(staging) cr } diff --git a/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala b/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala index d30422ca8dd54..d622f2aaabcc7 100644 --- a/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala +++ b/common/utils/src/test/scala/org/apache/spark/util/MavenUtilsSuite.scala @@ -78,11 +78,12 @@ class MavenUtilsSuite val settings = new IvySettings val res1 = MavenUtils.createRepoResolvers(settings.getDefaultIvyUserDir) // should have central and spark-packages by default - assert(res1.getResolvers.size() === 4) + assert(res1.getResolvers.size() === 5) assert(res1.getResolvers.get(0).asInstanceOf[IBiblioResolver].getName === "local-m2-cache") assert(res1.getResolvers.get(1).asInstanceOf[FileSystemResolver].getName === "local-ivy-cache") assert(res1.getResolvers.get(2).asInstanceOf[IBiblioResolver].getName === "central") assert(res1.getResolvers.get(3).asInstanceOf[IBiblioResolver].getName === "spark-packages") + assert(res1.getResolvers.get(4).asInstanceOf[IBiblioResolver].getName === "hive-staging-repo") } test("create additional resolvers") { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 397da6c18b50a..5e58959ca4f7d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -1627,10 +1627,8 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd test("SPARK-33084: Add jar support Ivy URI in SQL") { val testData = TestHive.getHiveFile("data/files/sample.json").toURI withTable("t") { - // hive-catalog-core has some transitive dependencies which dont exist on maven central - // and hence cannot be found in the test environment or are non-jar (.pom) which cause - // failures in tests. Use transitive=false as it should be good enough to test the Ivy - // support in Hive ADD JAR + // Use transitive=false as it should be good enough to test the Ivy support + // in Hive ADD JAR sql(s"ADD JAR ivy://org.apache.hive.hcatalog:hive-hcatalog-core:$hiveVersion" + "?transitive=false") sql( From a31b06048c011dddd18563479d775c8462593f2c Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 6 May 2024 14:07:15 +0800 Subject: [PATCH 14/15] Apache Hive 2.3.10 RC1 --- .../utils/src/main/scala/org/apache/spark/util/MavenUtils.scala | 2 +- pom.xml | 2 +- project/SparkBuild.scala | 2 +- .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala index 0835976d56891..7cc441c68d65d 100644 --- a/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala +++ b/common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala @@ -196,7 +196,7 @@ private[spark] object MavenUtils extends Logging { val staging: IBiblioResolver = new IBiblioResolver staging.setM2compatible(true) staging.setUsepoms(true) - staging.setRoot("https://repository.apache.org/content/repositories/orgapachehive-1128/") + staging.setRoot("https://repository.apache.org/content/repositories/orgapachehive-1129/") staging.setName("hive-staging-repo") cr.add(staging) cr diff --git a/pom.xml b/pom.xml index 226ceeb049304..dd67058e9e598 100644 --- a/pom.xml +++ b/pom.xml @@ -356,7 +356,7 @@ hive-staging-repo - https://repository.apache.org/content/repositories/orgapachehive-1128/ + https://repository.apache.org/content/repositories/orgapachehive-1129/ true always diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index cab015bf9c62f..ab3194381cab6 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -277,7 +277,7 @@ object SparkBuild extends PomBuild { Resolver.mavenLocal, Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ) ++ Seq( - "hive-staging-repo" at "https://repository.apache.org/content/repositories/orgapachehive-1128/", + "hive-staging-repo" at "https://repository.apache.org/content/repositories/orgapachehive-1129/", Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) ), externalResolvers := resolvers.value, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index f6dc095fa4310..2e0e8ca1ce9c6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -4156,7 +4156,7 @@ object SQLConf { .createWithDefault( sys.env.getOrElse("DEFAULT_ARTIFACT_REPOSITORY", // scalastyle:off line.size.limit - "https://repository.apache.org/content/repositories/orgapachehive-1128/," + + "https://repository.apache.org/content/repositories/orgapachehive-1129/," + // scalastyle:on line.size.limit "https://maven-central.storage-download.googleapis.com/maven2/")) From 0a149942e709f03cd7e6bce55058de11e7663dfa Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 6 May 2024 19:59:39 +0800 Subject: [PATCH 15/15] fix --- dev/deps/spark-deps-hadoop-3-hive-2.3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 6a33c58a11575..e10a3db136574 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -63,7 +63,7 @@ derby/10.16.1.1//derby-10.16.1.1.jar derbyshared/10.16.1.1//derbyshared-10.16.1.1.jar derbytools/10.16.1.1//derbytools-10.16.1.1.jar dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar -error_prone_annotations/2.23.0//error_prone_annotations-2.23.0.jar +error_prone_annotations/2.26.1//error_prone_annotations-2.26.1.jar esdk-obs-java/3.20.4.2//esdk-obs-java-3.20.4.2.jar failureaccess/1.0.2//failureaccess-1.0.2.jar flatbuffers-java/23.5.26//flatbuffers-java-23.5.26.jar