From 3b305d067424a83026ad388fb7d50018099070e7 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 5 Sep 2017 00:37:46 -0700 Subject: [PATCH 1/3] [SPARK-13656][SQL] Delete spark.sql.parquet.cacheMetadata from SQLConf and docs --- docs/sql-programming-guide.md | 7 ------- .../main/scala/org/apache/spark/sql/internal/SQLConf.scala | 7 ------- 2 files changed, 14 deletions(-) diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index ee231a934a3af..17a0a4d8271de 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -924,13 +924,6 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession flag tells Spark SQL to interpret INT96 data as a timestamp to provide compatibility with these systems. - - spark.sql.parquet.cacheMetadata - true - - Turns on caching of Parquet schema metadata. Can speed up querying of static data. - - spark.sql.parquet.compression.codec snappy diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index c407874381ac2..f588abd463bb9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -284,11 +284,6 @@ object SQLConf { .booleanConf .createWithDefault(false) - val PARQUET_CACHE_METADATA = buildConf("spark.sql.parquet.cacheMetadata") - .doc("Turns on caching of Parquet schema metadata. Can speed up querying of static data.") - .booleanConf - .createWithDefault(true) - val PARQUET_COMPRESSION = buildConf("spark.sql.parquet.compression.codec") .doc("Sets the compression codec use when writing Parquet files. Acceptable values include: " + "uncompressed, snappy, gzip, lzo.") @@ -1010,8 +1005,6 @@ class SQLConf extends Serializable with Logging { def parquetCompressionCodec: String = getConf(PARQUET_COMPRESSION) - def parquetCacheMetadata: Boolean = getConf(PARQUET_CACHE_METADATA) - def parquetVectorizedReaderEnabled: Boolean = getConf(PARQUET_VECTORIZED_READER_ENABLED) def columnBatchSize: Int = getConf(COLUMN_BATCH_SIZE) From 40ed9ffc42c976f2efebc417e1258bc73df18236 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 7 Sep 2017 10:58:51 -0700 Subject: [PATCH 2/3] Add to migraion guide --- docs/sql-programming-guide.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index 17a0a4d8271de..6528d59d05150 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -1580,6 +1580,10 @@ options. Note that this is different from the Hive behavior. - As a result, `DROP TABLE` statements on those tables will not remove the data. + - From Spark 2.0.1, `spark.sql.parquet.cacheMetadata` is no longer used. See + [SPARK-16321](https://issues.apache.org/jira/browse/SPARK-16321) and + [SPARK-15639](https://issues.apache.org/jira/browse/SPARK-15639) for details. + ## Upgrading From Spark SQL 1.5 to 1.6 - From Spark 1.6, by default the Thrift server runs in multi-session mode. Which means each JDBC/ODBC From 8e3d8fe26c6bbf15e17a4b80ff8357fe870f2d46 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 7 Sep 2017 11:41:05 -0700 Subject: [PATCH 3/3] Fix JIRA pointer. --- docs/sql-programming-guide.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md index 6528d59d05150..52d6b9d735676 100644 --- a/docs/sql-programming-guide.md +++ b/docs/sql-programming-guide.md @@ -1580,9 +1580,8 @@ options. Note that this is different from the Hive behavior. - As a result, `DROP TABLE` statements on those tables will not remove the data. - - From Spark 2.0.1, `spark.sql.parquet.cacheMetadata` is no longer used. See - [SPARK-16321](https://issues.apache.org/jira/browse/SPARK-16321) and - [SPARK-15639](https://issues.apache.org/jira/browse/SPARK-15639) for details. + - `spark.sql.parquet.cacheMetadata` is no longer used. + See [SPARK-13664](https://issues.apache.org/jira/browse/SPARK-13664) for details. ## Upgrading From Spark SQL 1.5 to 1.6