From 581f075a934586162130893876d00a42a0155957 Mon Sep 17 00:00:00 2001 From: Darek Date: Fri, 22 Oct 2021 11:18:01 -0400 Subject: [PATCH 1/5] Upgrading SystemRequirements to include Java <= 17 --- R/pkg/DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION index 126d08b4862f8..084645395925a 100644 --- a/R/pkg/DESCRIPTION +++ b/R/pkg/DESCRIPTION @@ -13,7 +13,7 @@ Authors@R: c(person("Shivaram", "Venkataraman", role = "aut", License: Apache License (== 2.0) URL: https://www.apache.org https://spark.apache.org BugReports: https://spark.apache.org/contributing.html -SystemRequirements: Java (>= 8, < 12) +SystemRequirements: Java (>= 8, <= 17) Depends: R (>= 3.5), methods From 3b3c4261e76cea55c719831fe39e164e4b98adb2 Mon Sep 17 00:00:00 2001 From: RabbidHY Date: Sat, 23 Oct 2021 11:18:14 +0900 Subject: [PATCH 2/5] [SPARK-37084][SQL] Set spark.sql.files.openCostInBytes to bytesConf ### What changes were proposed in this pull request? Set `spark.sql.files.openCostInBytes` to bytesConf. ### Why are the changes needed? The name is _*InBytes_, but it actually only accepts **long type**. This is confusing for users. After the changes, it can accept **String** as input which is more flexible to users. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing test. Closes #34353 from RabbidHY/SPARK-37084. Authored-by: RabbidHY Signed-off-by: Hyukjin Kwon --- .../org/apache/spark/internal/config/ConfigEntrySuite.scala | 2 ++ .../main/scala/org/apache/spark/sql/internal/SQLConf.scala | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/internal/config/ConfigEntrySuite.scala b/core/src/test/scala/org/apache/spark/internal/config/ConfigEntrySuite.scala index 6bd63fa5ffa4a..1c9a9a1203451 100644 --- a/core/src/test/scala/org/apache/spark/internal/config/ConfigEntrySuite.scala +++ b/core/src/test/scala/org/apache/spark/internal/config/ConfigEntrySuite.scala @@ -94,6 +94,8 @@ class ConfigEntrySuite extends SparkFunSuite { assert(conf.get(bytes) === 1024L) conf.set(bytes.key, "1k") assert(conf.get(bytes) === 1L) + conf.set(bytes.key, "2048") + assert(conf.get(bytes) === 2048) } test("conf entry: regex") { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index eed74da5e8f58..5023b4a616555 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1419,8 +1419,8 @@ object SQLConf { " bigger files (which is scheduled first). This configuration is effective only when using" + " file-based sources such as Parquet, JSON and ORC.") .version("2.0.0") - .longConf - .createWithDefault(4 * 1024 * 1024) + .bytesConf(ByteUnit.BYTE) + .createWithDefaultString("4MB") val FILES_MIN_PARTITION_NUM = buildConf("spark.sql.files.minPartitionNum") .doc("The suggested (not guaranteed) minimum number of split file partitions. " + From 700b8c055bde30d250911b05b6e678277fb3f1c9 Mon Sep 17 00:00:00 2001 From: Darek Date: Fri, 22 Oct 2021 11:18:01 -0400 Subject: [PATCH 3/5] Upgrading SystemRequirements to include Java <= 17 --- R/pkg/DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION index 126d08b4862f8..084645395925a 100644 --- a/R/pkg/DESCRIPTION +++ b/R/pkg/DESCRIPTION @@ -13,7 +13,7 @@ Authors@R: c(person("Shivaram", "Venkataraman", role = "aut", License: Apache License (== 2.0) URL: https://www.apache.org https://spark.apache.org BugReports: https://spark.apache.org/contributing.html -SystemRequirements: Java (>= 8, < 12) +SystemRequirements: Java (>= 8, <= 17) Depends: R (>= 3.5), methods From da7f81bce1923fb8d658509ab88986a890891769 Mon Sep 17 00:00:00 2001 From: Darek Date: Sat, 23 Oct 2021 08:18:15 -0400 Subject: [PATCH 4/5] [SPARK-37091][R] SystemRequirements to be Java < 18 --- R/pkg/DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION index 084645395925a..6b85bb758a081 100644 --- a/R/pkg/DESCRIPTION +++ b/R/pkg/DESCRIPTION @@ -13,7 +13,7 @@ Authors@R: c(person("Shivaram", "Venkataraman", role = "aut", License: Apache License (== 2.0) URL: https://www.apache.org https://spark.apache.org BugReports: https://spark.apache.org/contributing.html -SystemRequirements: Java (>= 8, <= 17) +SystemRequirements: Java (>= 8, < 18) Depends: R (>= 3.5), methods From 0ef0e932bd8d9d52cc25712d821f657f9519a577 Mon Sep 17 00:00:00 2001 From: Darek Date: Sun, 24 Oct 2021 08:14:26 -0400 Subject: [PATCH 5/5] Revert "[SPARK-37084][SQL] Set spark.sql.files.openCostInBytes to bytesConf" This reverts commit 3b3c4261e76cea55c719831fe39e164e4b98adb2. --- .../org/apache/spark/internal/config/ConfigEntrySuite.scala | 2 -- .../main/scala/org/apache/spark/sql/internal/SQLConf.scala | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/internal/config/ConfigEntrySuite.scala b/core/src/test/scala/org/apache/spark/internal/config/ConfigEntrySuite.scala index 1c9a9a1203451..6bd63fa5ffa4a 100644 --- a/core/src/test/scala/org/apache/spark/internal/config/ConfigEntrySuite.scala +++ b/core/src/test/scala/org/apache/spark/internal/config/ConfigEntrySuite.scala @@ -94,8 +94,6 @@ class ConfigEntrySuite extends SparkFunSuite { assert(conf.get(bytes) === 1024L) conf.set(bytes.key, "1k") assert(conf.get(bytes) === 1L) - conf.set(bytes.key, "2048") - assert(conf.get(bytes) === 2048) } test("conf entry: regex") { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 5023b4a616555..eed74da5e8f58 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1419,8 +1419,8 @@ object SQLConf { " bigger files (which is scheduled first). This configuration is effective only when using" + " file-based sources such as Parquet, JSON and ORC.") .version("2.0.0") - .bytesConf(ByteUnit.BYTE) - .createWithDefaultString("4MB") + .longConf + .createWithDefault(4 * 1024 * 1024) val FILES_MIN_PARTITION_NUM = buildConf("spark.sql.files.minPartitionNum") .doc("The suggested (not guaranteed) minimum number of split file partitions. " +