diff --git a/core/pom.xml b/core/pom.xml index 3eedc69c9593b..7ff409043c46c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -26,12 +26,15 @@ spark-core_2.12 - - core - jar Spark Project Core http://spark.apache.org/ + + + core + src/main/scala-${scala.binary.version} + + com.thoughtworks.paranamer @@ -516,6 +519,24 @@ + + org.codehaus.mojo + build-helper-maven-plugin + + + add-sources + generate-sources + + add-source + + + + ${extra.source.dir} + + + + + diff --git a/core/src/main/scala-2.12/org/apache/spark/util/OrderingUtil.scala b/core/src/main/scala-2.12/org/apache/spark/util/OrderingUtil.scala new file mode 100644 index 0000000000000..193ae36f3c089 --- /dev/null +++ b/core/src/main/scala-2.12/org/apache/spark/util/OrderingUtil.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.util + +/** + * This class only exists to bridge the difference between Scala 2.12 and Scala 2.13's + * support for floating-point ordering. It is implemented separately for both as there + * is no method that exists in both for comparison. + * + * It functions like Ordering.Double in Scala 2.12. + */ +private[spark] object OrderingUtil { + + def compareDouble(x: Double, y: Double): Int = Ordering.Double.compare(x, y) + + def compareFloat(x: Float, y: Float): Int = Ordering.Float.compare(x, y) + +} diff --git a/core/src/main/scala-2.13/org/apache/spark/util/OrderingUtil.scala b/core/src/main/scala-2.13/org/apache/spark/util/OrderingUtil.scala new file mode 100644 index 0000000000000..e861506672706 --- /dev/null +++ b/core/src/main/scala-2.13/org/apache/spark/util/OrderingUtil.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.util + +/** + * This class only exists to bridge the difference between Scala 2.12 and Scala 2.13's + * support for floating-point ordering. It is implemented separately for both as there + * is no method that exists in both for comparison. + * + * It functions like Ordering.Double.TotalOrdering in Scala 2.13, which matches java.lang.Double + * rather than Scala 2.12's Ordering.Double in handling of NaN. + */ +private[spark] object OrderingUtil { + + def compareDouble(x: Double, y: Double): Int = Ordering.Double.TotalOrdering.compare(x, y) + + def compareFloat(x: Float, y: Float): Int = Ordering.Float.TotalOrdering.compare(x, y) + +} diff --git a/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala index d1603b85a8e94..bd1921d7b770e 100644 --- a/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/collection/SorterSuite.scala @@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit import org.apache.spark.SparkFunSuite import org.apache.spark.internal.Logging +import org.apache.spark.util.OrderingUtil import org.apache.spark.util.Utils.timeIt import org.apache.spark.util.random.XORShiftRandom @@ -59,7 +60,7 @@ class SorterSuite extends SparkFunSuite with Logging { Arrays.sort(keys) new Sorter(new KVArraySortDataFormat[Double, Number]) - .sort(keyValueArray, 0, keys.length, Ordering.Double) + .sort(keyValueArray, 0, keys.length, OrderingUtil.compareDouble) keys.zipWithIndex.foreach { case (k, i) => assert(k === keyValueArray(2 * i)) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala index 518255ecc42cc..71fed3e69b651 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala @@ -21,7 +21,7 @@ import scala.math.Numeric._ import scala.math.Ordering import org.apache.spark.sql.types.Decimal.DecimalIsConflicted - +import org.apache.spark.util.OrderingUtil object ByteExactNumeric extends ByteIsIntegral with Ordering.ByteOrdering { private def checkOverflow(res: Int, x: Byte, y: Byte, op: String): Unit = { @@ -118,7 +118,7 @@ object LongExactNumeric extends LongIsIntegral with Ordering.LongOrdering { } } -object FloatExactNumeric extends FloatIsFractional with Ordering.FloatOrdering { +object FloatExactNumeric extends FloatIsFractional { private def overflowException(x: Float, dataType: String) = throw new ArithmeticException(s"Casting $x to $dataType causes overflow") @@ -148,9 +148,11 @@ object FloatExactNumeric extends FloatIsFractional with Ordering.FloatOrdering { overflowException(x, "int") } } + + override def compare(x: Float, y: Float): Int = OrderingUtil.compareFloat(x, y) } -object DoubleExactNumeric extends DoubleIsFractional with Ordering.DoubleOrdering { +object DoubleExactNumeric extends DoubleIsFractional { private def overflowException(x: Double, dataType: String) = throw new ArithmeticException(s"Casting $x to $dataType causes overflow") @@ -174,6 +176,8 @@ object DoubleExactNumeric extends DoubleIsFractional with Ordering.DoubleOrderin overflowException(x, "long") } } + + override def compare(x: Double, y: Double): Int = OrderingUtil.compareDouble(x, y) } object DecimalExactNumeric extends DecimalIsConflicted {