From 5ba55f8cb031f2a93930144121f36cb3868b2cfc Mon Sep 17 00:00:00 2001 From: ayushbilala Date: Fri, 8 May 2026 15:39:06 +0530 Subject: [PATCH 1/4] [SPARK-53840][SQL] Add AS JSON output support for SHOW TABLES and SHOW TABLE EXTENDED --- .../resources/error/error-conditions.json | 5 + .../sql/catalyst/parser/SqlBaseParser.g4 | 4 +- .../spark/sql/errors/CompilationErrors.scala | 6 + .../sql/catalyst/parser/AstBuilder.scala | 12 +- .../catalyst/plans/logical/v2Commands.scala | 7 + .../analysis/ResolveSessionCatalog.scala | 12 +- .../apache/spark/sql/classic/Catalog.scala | 2 +- .../spark/sql/execution/command/tables.scala | 55 ++++- .../datasources/v2/DataSourceV2Strategy.scala | 17 +- .../datasources/v2/ShowTablesExec.scala | 12 +- .../datasources/v2/ShowTablesJsonExec.scala | 115 +++++++++++ .../command/ShowTablesParserSuite.scala | 37 ++++ .../command/ShowTablesSuiteBase.scala | 191 ++++++++++++++++++ 13 files changed, 456 insertions(+), 19 deletions(-) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesJsonExec.scala diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 2775e34808153..b29fe05ad1f51 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -8295,6 +8295,11 @@ " is a VARIABLE and cannot be updated using the SET statement. Use SET VARIABLE = ... instead." ] }, + "SHOW_TABLE_EXTENDED_JSON_WITH_PARTITION" : { + "message" : [ + "SHOW TABLE EXTENDED with PARTITION does not support AS JSON output." + ] + }, "SQL_CURSOR" : { "message" : [ "SQL cursor operations (DECLARE CURSOR, OPEN, FETCH, CLOSE) are not supported." diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index 744c472b20179..152ea07641677 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 @@ -364,9 +364,9 @@ statement | EXPLAIN (LOGICAL | FORMATTED | EXTENDED | CODEGEN | COST)? (statement|setResetStatement) #explain | SHOW TABLES ((FROM | IN) identifierReference)? - (LIKE? pattern=stringLit)? #showTables + (LIKE? pattern=stringLit)? (AS JSON)? #showTables | SHOW TABLE EXTENDED ((FROM | IN) ns=identifierReference)? - LIKE pattern=stringLit partitionSpec? #showTableExtended + LIKE pattern=stringLit partitionSpec? (AS JSON)? #showTableExtended | SHOW TBLPROPERTIES table=identifierReference (LEFT_PAREN key=propertyKeyOrStringLit RIGHT_PAREN)? #showTblProperties | SHOW COLUMNS (FROM | IN) table=identifierReference diff --git a/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala b/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala index 6a275b9ad0c16..8ddf82225ca54 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala @@ -53,6 +53,12 @@ private[sql] trait CompilationErrors extends DataTypeErrorsBase { messageParameters = Map.empty) } + def showTableExtendedJsonWithPartitionError(): AnalysisException = { + new AnalysisException( + errorClass = "UNSUPPORTED_FEATURE.SHOW_TABLE_EXTENDED_JSON_WITH_PARTITION", + messageParameters = Map.empty) + } + def cannotFindDescriptorFileError(filePath: String, cause: Throwable): AnalysisException = { new AnalysisException( errorClass = "PROTOBUF_DESCRIPTOR_FILE_NOT_FOUND", diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index ccca53f351e8b..ab03621756ae1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -5847,7 +5847,10 @@ class AstBuilder extends DataTypeAstBuilder } else { CurrentNamespace } - ShowTables(ns, Option(ctx.pattern).map(x => string(visitStringLit(x)))) + val asJson = ctx.JSON != null + val pattern = Option(ctx.pattern).map(x => string(visitStringLit(x))) + val output = if (asJson) ShowTables.getJsonOutputAttrs else ShowTables.getOutputAttrs + ShowTables(ns, pattern, asJson, output) } /** @@ -5855,6 +5858,10 @@ class AstBuilder extends DataTypeAstBuilder */ override def visitShowTableExtended( ctx: ShowTableExtendedContext): LogicalPlan = withOrigin(ctx) { + val asJson = ctx.JSON != null + if (asJson && ctx.partitionSpec != null) { + throw QueryCompilationErrors.showTableExtendedJsonWithPartitionError() + } Option(ctx.partitionSpec).map { spec => val table = withOrigin(ctx.pattern) { if (ctx.identifierReference() != null) { @@ -5874,7 +5881,8 @@ class AstBuilder extends DataTypeAstBuilder } else { CurrentNamespace } - ShowTablesExtended(ns, string(visitStringLit(ctx.pattern))) + val output = if (asJson) ShowTables.getJsonOutputAttrs else ShowTablesUtils.getOutputAttrs + ShowTablesExtended(ns, string(visitStringLit(ctx.pattern)), asJson, output) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala index 40cf5009b97dc..55e3c4029a7f9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala @@ -1403,10 +1403,12 @@ case class RenameTable( case class ShowTables( namespace: LogicalPlan, pattern: Option[String], + asJson: Boolean = false, override val output: Seq[Attribute] = ShowTables.getOutputAttrs) extends UnaryCommand { override def child: LogicalPlan = namespace override protected def withNewChildInternal(newChild: LogicalPlan): ShowTables = copy(namespace = newChild) + override protected def stringArgs: Iterator[Any] = Iterator(pattern, output) } object ShowTables { @@ -1414,6 +1416,9 @@ object ShowTables { AttributeReference("namespace", StringType, nullable = false)(), AttributeReference("tableName", StringType, nullable = false)(), AttributeReference("isTemporary", BooleanType, nullable = false)()) + + def getJsonOutputAttrs: Seq[Attribute] = Seq( + AttributeReference("json_metadata", StringType, nullable = false)()) } /** @@ -1422,10 +1427,12 @@ object ShowTables { case class ShowTablesExtended( namespace: LogicalPlan, pattern: String, + asJson: Boolean = false, override val output: Seq[Attribute] = ShowTablesUtils.getOutputAttrs) extends UnaryCommand { override def child: LogicalPlan = namespace override protected def withNewChildInternal(newChild: LogicalPlan): ShowTablesExtended = copy(namespace = newChild) + override protected def stringArgs: Iterator[Any] = Iterator(pattern, output) } object ShowTablesUtils { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala index cfd52707bbc2c..ff861332a7703 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala @@ -367,19 +367,23 @@ class ResolveSessionCatalog(val catalogManager: CatalogManager) case d @ DropNamespace(ResolvedV1Database(db), _, _) if conf.useV1Command => DropDatabaseCommand(db, d.ifExists, d.cascade) - case ShowTables(ResolvedV1Database(db), pattern, output) if conf.useV1Command => - ShowTablesCommand(Some(db), pattern, output) + case ShowTables(ResolvedV1Database(db), pattern, asJson, output) if conf.useV1Command => + ShowTablesCommand(Some(db), pattern, output, asJson = asJson) case ShowTablesExtended( ResolvedV1Database(db), pattern, + asJson, output) => - val newOutput = if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) { + val newOutput = if (asJson) { + output + } else if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) { output.head.withName("database") +: output.tail } else { output } - ShowTablesCommand(Some(db), Some(pattern), newOutput, isExtended = true) + ShowTablesCommand(Some(db), Some(pattern), newOutput, + isExtended = true, asJson = asJson) case ShowTablePartition( ResolvedTable(catalog, _, table: V1Table, _), diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala index 40c40f6ea78aa..939b12a404fd2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala @@ -195,7 +195,7 @@ class Catalog(sparkSession: SparkSession) extends catalog.Catalog with Logging { private def makeTablesDataset(plan: ShowTables): Dataset[Table] = { val qe = sparkSession.sessionState.executePlan(plan) val catalog = qe.analyzed.collectFirst { - case ShowTables(r: ResolvedNamespace, _, _) => r.catalog + case ShowTables(r: ResolvedNamespace, _, _, _) => r.catalog case _: ShowTablesCommand => sparkSession.sessionState.catalogManager.v2SessionCatalog }.get diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index ca534706635a1..319f3284c7c66 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -25,6 +25,8 @@ import scala.util.control.NonFatal import org.apache.hadoop.fs.{FileContext, FsConstants, Path} import org.apache.hadoop.fs.permission.{AclEntry, AclEntryScope, AclEntryType, FsAction, FsPermission} +import org.json4s._ +import org.json4s.jackson.JsonMethods._ import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.{SQLConfHelper, TableIdentifier} @@ -927,9 +929,21 @@ case class ShowTablesCommand( tableIdentifierPattern: Option[String], override val output: Seq[Attribute], isExtended: Boolean = false, - partitionSpec: Option[TablePartitionSpec] = None) extends LeafRunnableCommand { + partitionSpec: Option[TablePartitionSpec] = None, + asJson: Boolean = false) extends LeafRunnableCommand { + + override protected def stringArgs: Iterator[Any] = + Iterator(databaseName, tableIdentifierPattern, output, isExtended, partitionSpec) override def run(sparkSession: SparkSession): Seq[Row] = { + if (asJson) { + runAsJson(sparkSession) + } else { + runAsText(sparkSession) + } + } + + private def runAsText(sparkSession: SparkSession): Seq[Row] = { // Since we need to return a Seq of rows, we will call getTables directly // instead of calling tables in sparkSession. val catalog = sparkSession.sessionState.catalog @@ -972,6 +986,45 @@ case class ShowTablesCommand( Seq(Row(database, tableName, isTemp, s"$information\n")) } } + + private def runAsJson(sparkSession: SparkSession): Seq[Row] = { + val catalog = sparkSession.sessionState.catalog + val db = databaseName.getOrElse(catalog.getCurrentDatabase) + val tables = + tableIdentifierPattern.map(catalog.listTables(db, _)).getOrElse(catalog.listTables(db)) + + val jsonTables = tables.map { tableIdent => + val isTemp = catalog.isTempView(tableIdent) + val ns = tableIdent.database.toList + + if (isExtended) { + val tableType = if (isTemp) { + "VIEW" + } else { + val meta = catalog.getTempViewOrPermanentTableMetadata(tableIdent) + if (meta.tableType == CatalogTableType.VIEW) "VIEW" else "TABLE" + } + + JObject( + "name" -> JString(tableIdent.table), + "catalog" -> JString( + sparkSession.sessionState.catalogManager.v2SessionCatalog.name()), + "namespace" -> JArray(ns.map(JString(_))), + "type" -> JString(tableType), + "isTemporary" -> JBool(isTemp) + ) + } else { + JObject( + "name" -> JString(tableIdent.table), + "namespace" -> JArray(ns.map(JString(_))), + "isTemporary" -> JBool(isTemp) + ) + } + }.toList + + val jsonOutput = JObject("tables" -> JArray(jsonTables)) + Seq(Row(compact(render(jsonOutput)))) + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala index 4fd7d993cc3d0..1309c844a0fa7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala @@ -664,8 +664,13 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat case DropNamespace(ResolvedNamespace(catalog, ns, _), ifExists, cascade) => DropNamespaceExec(catalog, ns, ifExists, cascade) :: Nil - case ShowTables(ResolvedNamespace(catalog, ns, _), pattern, output) => - ShowTablesExec(output, catalog.asTableCatalog, ns, pattern) :: Nil + case ShowTables(ResolvedNamespace(catalog, ns, _), pattern, asJson, output) => + if (asJson) { + ShowTablesJsonExec( + output, catalog.asTableCatalog, ns, pattern.getOrElse("*"), isExtended = false) :: Nil + } else { + ShowTablesExec(output, catalog.asTableCatalog, ns, pattern) :: Nil + } // SHOW VIEWS on a v2 ViewCatalog. `ResolveSessionCatalog` rewrites the SHOW VIEWS plan to // v1 `ShowViewsCommand` only when the catalog is NOT a `ViewCatalog`; non-`ViewCatalog` @@ -679,8 +684,14 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat case ShowTablesExtended( ResolvedNamespace(catalog, ns, _), pattern, + asJson, output) => - ShowTablesExtendedExec(output, catalog.asTableCatalog, ns, pattern) :: Nil + if (asJson) { + ShowTablesJsonExec( + output, catalog.asTableCatalog, ns, pattern, isExtended = true) :: Nil + } else { + ShowTablesExtendedExec(output, catalog.asTableCatalog, ns, pattern) :: Nil + } case ShowTablePartition(r: ResolvedTable, part, output) => ShowTablePartitionExec(output, r.catalog, r.identifier, diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala index 8680785e0815f..c74d9f8fa748a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala @@ -41,19 +41,19 @@ case class ShowTablesExec( namespace: Seq[String], pattern: Option[String]) extends V2CommandExec with LeafExecNode { override protected def run(): Seq[InternalRow] = { - val rows = new ArrayBuffer[InternalRow]() - val identifiers: Array[Identifier] = catalog match { case mc: TableViewCatalog => mc.listTableAndViewSummaries(namespace.toArray).map(_.identifier()) case _ => catalog.listTables(namespace.toArray) } - identifiers.foreach { ident => - if (pattern.map(StringUtils.filterPattern(Seq(ident.name()), _).nonEmpty).getOrElse(true)) { - rows += toCatalystRow(ident.namespace().quoted, ident.name(), isTempView(ident, catalog)) - } + val filteredIdents = identifiers.filter { ident => + pattern.map(StringUtils.filterPattern(Seq(ident.name()), _).nonEmpty).getOrElse(true) } + val rows = new ArrayBuffer[InternalRow]() + filteredIdents.foreach { ident => + rows += toCatalystRow(ident.namespace().quoted, ident.name(), isTempView(ident, catalog)) + } rows.toSeq } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesJsonExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesJsonExec.scala new file mode 100644 index 0000000000000..8af65d76c344b --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesJsonExec.scala @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.v2 + +import scala.collection.mutable.ArrayBuffer + +import org.json4s._ +import org.json4s.jackson.JsonMethods._ + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.util.StringUtils +import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, TableCatalog, TableViewCatalog} +import org.apache.spark.sql.execution.LeafExecNode + +/** + * Physical plan node for `SHOW TABLES AS JSON` and `SHOW TABLE EXTENDED AS JSON`. + * + * For a [[TableViewCatalog]] (non-extended only), listing is done via + * [[TableViewCatalog#listTableAndViewSummaries]] so that views appear alongside tables, + * matching the v1 `SHOW TABLES` semantics. + */ +case class ShowTablesJsonExec( + output: Seq[Attribute], + catalog: TableCatalog, + namespace: Seq[String], + pattern: String, + isExtended: Boolean) extends V2CommandExec with LeafExecNode { + + override protected def run(): Seq[InternalRow] = { + val identifiers: Array[Identifier] = if (!isExtended) { + catalog match { + case mc: TableViewCatalog => + mc.listTableAndViewSummaries(namespace.toArray).map(_.identifier()) + case _ => catalog.listTables(namespace.toArray) + } + } else { + catalog.listTables(namespace.toArray) + } + + val filteredIdents = identifiers.filter { ident => + StringUtils.filterPattern(Seq(ident.name()), pattern).nonEmpty + } + + val jsonRows = new ArrayBuffer[JObject]() + filteredIdents.foreach { ident => + jsonRows += toJsonEntry(ident.name(), ident.namespace(), isTempView(ident)) + } + + // For non-session V2 catalogs that don't surface temp views via listTables() or + // listTableAndViewSummaries(), fetch them separately. For V2SessionCatalog, + // listTables() already includes local temp views, so we skip this to avoid duplicates. + // For TableViewCatalog (non-extended path), views come from listTableAndViewSummaries(). + if (!CatalogV2Util.isSessionCatalog(catalog) && + (isExtended || !catalog.isInstanceOf[TableViewCatalog])) { + val sessionCatalog = session.sessionState.catalog + val db = namespace match { + case Seq(db) => db + case _ => "" + } + sessionCatalog.listTempViews(db, pattern).foreach { tempView => + jsonRows += toJsonEntry( + tempView.identifier.table, + tempView.identifier.database.toArray, + isTemporary = true) + } + } + + val jsonOutput = JObject("tables" -> JArray(jsonRows.toList)) + Seq(toCatalystRow(compact(render(jsonOutput)))) + } + + private def toJsonEntry( + name: String, + namespace: Array[String], + isTemporary: Boolean): JObject = { + val nsArray = JArray(namespace.map(JString(_)).toList) + if (isExtended) { + JObject( + "name" -> JString(name), + "catalog" -> JString(catalog.name()), + "namespace" -> nsArray, + "type" -> JString(if (isTemporary) "VIEW" else "TABLE"), + "isTemporary" -> JBool(isTemporary) + ) + } else { + JObject( + "name" -> JString(name), + "namespace" -> nsArray, + "isTemporary" -> JBool(isTemporary) + ) + } + } + + private def isTempView(ident: Identifier): Boolean = { + if (CatalogV2Util.isSessionCatalog(catalog)) { + session.sessionState.catalog.isTempView((ident.namespace() :+ ident.name()).toSeq) + } else false + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala index 2a4a49c75ad92..fd883c67389a9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.execution.command +import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, CurrentNamespace, UnresolvedNamespace, UnresolvedPartitionSpec, UnresolvedTable} import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan import org.apache.spark.sql.catalyst.plans.logical.{ShowTablePartition, ShowTables, ShowTablesExtended} @@ -49,6 +50,21 @@ class ShowTablesParserSuite extends AnalysisTest with SharedSparkSession { ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*"))) } + test("show tables as json") { + comparePlans( + parsePlan("SHOW TABLES AS JSON"), + ShowTables(CurrentNamespace, None, asJson = true, + output = ShowTables.getJsonOutputAttrs)) + comparePlans( + parsePlan("SHOW TABLES IN ns1 AS JSON"), + ShowTables(UnresolvedNamespace(Seq("ns1")), None, asJson = true, + output = ShowTables.getJsonOutputAttrs)) + comparePlans( + parsePlan("SHOW TABLES IN ns1 LIKE '*test*' AS JSON"), + ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*"), asJson = true, + output = ShowTables.getJsonOutputAttrs)) + } + test("show table extended") { comparePlans( parsePlan("SHOW TABLE EXTENDED LIKE '*test*'"), @@ -80,4 +96,25 @@ class ShowTablesParserSuite extends AnalysisTest with SharedSparkSession { "SHOW TABLE EXTENDED ... PARTITION ..."), UnresolvedPartitionSpec(Map("ds" -> "2008-04-09")))) } + + test("show table extended as json") { + comparePlans( + parsePlan("SHOW TABLE EXTENDED LIKE '*test*' AS JSON"), + ShowTablesExtended(CurrentNamespace, "*test*", asJson = true, + output = ShowTables.getJsonOutputAttrs)) + comparePlans( + parsePlan(s"SHOW TABLE EXTENDED IN $catalog.ns1.ns2 LIKE '*test*' AS JSON"), + ShowTablesExtended(UnresolvedNamespace(Seq(catalog, "ns1", "ns2")), "*test*", + asJson = true, output = ShowTables.getJsonOutputAttrs)) + } + + test("show table extended as json with partition should fail") { + checkError( + exception = intercept[AnalysisException] { + parsePlan("SHOW TABLE EXTENDED LIKE '*test*' PARTITION(ds='2008-04-09') AS JSON") + }, + condition = "UNSUPPORTED_FEATURE.SHOW_TABLE_EXTENDED_JSON_WITH_PARTITION", + parameters = Map.empty + ) + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala index dbeb67c253208..92816793604bd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala @@ -17,6 +17,9 @@ package org.apache.spark.sql.execution.command +import org.json4s._ +import org.json4s.jackson.JsonMethods._ + import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ import org.apache.spark.sql.internal.SQLConf @@ -32,6 +35,8 @@ import org.apache.spark.sql.internal.SQLConf * - V1 Hive External catalog: `org.apache.spark.sql.hive.execution.command.ShowTablesSuite` */ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { + implicit val formats: Formats = DefaultFormats + override val command = "SHOW TABLES" protected def defaultNamespace: Seq[String] @@ -461,4 +466,190 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { } } } + + test("SHOW TABLES AS JSON returns single row with json_metadata column") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT, data STRING) $defaultUsing") + val df = sql(s"SHOW TABLES IN $catalog.ns AS JSON") + assert(df.schema.fieldNames === Seq("json_metadata")) + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.length == 1, s"Expected 1 entry, got: $tables") + val tblEntry = tables.find(t => (t \ "name").extract[String] == "tbl") + assert(tblEntry.isDefined) + assert((tblEntry.get \ "isTemporary").extract[Boolean] == false) + assert((tblEntry.get \ "namespace").isInstanceOf[JArray]) + } + } + + test("SHOW TABLES AS JSON with empty database") { + withNamespace(s"$catalog.ns_empty") { + sql(s"CREATE NAMESPACE $catalog.ns_empty") + val df = sql(s"SHOW TABLES IN $catalog.ns_empty AS JSON") + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.isEmpty) + } + } + + test("SHOW TABLE EXTENDED AS JSON returns single row with json_metadata column") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT, data STRING) $defaultUsing") + val df = sql(s"SHOW TABLE EXTENDED IN $catalog.ns LIKE 'tbl' AS JSON") + assert(df.schema.fieldNames === Seq("json_metadata")) + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.length == 1) + val entry = tables.head + assert((entry \ "name").extract[String] == "tbl") + assert((entry \ "type").extract[String] == "TABLE") + assert((entry \ "isTemporary").extract[Boolean] == false) + assert((entry \ "catalog").isInstanceOf[JString]) + assert((entry \ "namespace").isInstanceOf[JArray]) + } + } + + test("SHOW TABLES AS JSON includes temp views") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + withTempView("tv") { + sql("CREATE TEMP VIEW tv AS SELECT 1 AS id") + val df = sql(s"SHOW TABLES IN $catalog.ns AS JSON") + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + // Verify no duplicate entries + val names = tables.map(t => (t \ "name").extract[String]) + assert(names.distinct.length == names.length, s"Duplicate entries found: $names") + val tempView = tables.find(t => (t \ "name").extract[String] == "tv") + assert(tempView.isDefined) + assert((tempView.get \ "isTemporary").extract[Boolean] == true) + } + } + } + + test("SHOW TABLE EXTENDED AS JSON with local temp view") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + val localTmpViewName = "tbl_local_tmp" + withTempView(localTmpViewName) { + sql(s"CREATE TEMPORARY VIEW $localTmpViewName AS SELECT id FROM $t") + + val df = sql(s"SHOW TABLE EXTENDED IN $catalog.ns LIKE 'tbl*' AS JSON") + assert(df.schema.fieldNames === Seq("json_metadata")) + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.length == 2, s"Expected 2 entries (tbl + $localTmpViewName), got: $tables") + + // Verify no duplicate entries + val names = tables.map(e => (e \ "name").extract[String]) + assert(names.distinct.length == names.length, s"Duplicate entries found: $names") + + val tblEntry = tables.find(e => (e \ "name").extract[String] == "tbl") + assert(tblEntry.isDefined) + assert((tblEntry.get \ "isTemporary").extract[Boolean] == false) + assert((tblEntry.get \ "type").extract[String] == "TABLE") + + val tempViewEntry = tables.find(e => (e \ "name").extract[String] == localTmpViewName) + assert(tempViewEntry.isDefined) + assert((tempViewEntry.get \ "isTemporary").extract[Boolean] == true) + assert((tempViewEntry.get \ "type").extract[String] == "VIEW") + } + } + } + + test("SHOW TABLE EXTENDED AS JSON with global temp view") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + val globalTmpViewName = "ext_json_gtv" + val globalNamespace = "global_temp" + withView(s"$globalNamespace.$globalTmpViewName") { + sql(s"CREATE OR REPLACE GLOBAL TEMP VIEW $globalTmpViewName AS SELECT id FROM $t") + + val df = sql(s"SHOW TABLE EXTENDED IN $globalNamespace LIKE 'ext_json*' AS JSON") + assert(df.schema.fieldNames === Seq("json_metadata")) + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.length == 1, s"Expected 1 entry, got: $tables") + + val globalTempViewEntry = + tables.find(e => (e \ "name").extract[String] == globalTmpViewName) + assert(globalTempViewEntry.isDefined) + assert((globalTempViewEntry.get \ "isTemporary").extract[Boolean] == true) + assert((globalTempViewEntry.get \ "type").extract[String] == "VIEW") + } + } + } + + test("SHOW TABLES AS JSON with global temp view") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + val globalTmpViewName = "show_json_gtv" + val globalNamespace = "global_temp" + withView(s"$globalNamespace.$globalTmpViewName") { + sql(s"CREATE OR REPLACE GLOBAL TEMP VIEW $globalTmpViewName AS SELECT id FROM $t") + + val df = sql(s"SHOW TABLES IN $globalNamespace AS JSON") + assert(df.schema.fieldNames === Seq("json_metadata")) + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + assert(tables.length == 1, s"Expected 1 entry, got: $tables") + + val globalTempViewEntry = + tables.find(e => (e \ "name").extract[String] == globalTmpViewName) + assert(globalTempViewEntry.isDefined) + assert((globalTempViewEntry.get \ "isTemporary").extract[Boolean] == true) + } + } + } + + test("SHOW TABLE EXTENDED AS JSON with both local and global temp views") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + val localTmpViewName = "both_json_ltv" + val globalTmpViewName = "both_json_gtv" + val globalNamespace = "global_temp" + withView(localTmpViewName, s"$globalNamespace.$globalTmpViewName") { + sql(s"CREATE OR REPLACE TEMP VIEW $localTmpViewName AS SELECT id FROM $t") + sql(s"CREATE OR REPLACE GLOBAL TEMP VIEW $globalTmpViewName AS SELECT id FROM $t") + + val df = sql(s"SHOW TABLE EXTENDED IN $globalNamespace LIKE 'both_json*' AS JSON") + assert(df.count() == 1) + + val jsonStr = df.collect()(0).getString(0) + val json = parse(jsonStr) + val tables = (json \ "tables").asInstanceOf[JArray].arr + + assert(tables.length == 2) + + val globalTempViewEntry = + tables.find(e => (e \ "name").extract[String] == globalTmpViewName) + assert(globalTempViewEntry.isDefined) + assert((globalTempViewEntry.get \ "isTemporary").extract[Boolean] == true) + + val localTempViewEntry = + tables.find(e => (e \ "name").extract[String] == localTmpViewName) + assert(localTempViewEntry.isDefined) + assert((localTempViewEntry.get \ "isTemporary").extract[Boolean] == true) + } + } + } } From d8ea2eb6a6ca9ba06b334bf454bb8ed7ddd60b7d Mon Sep 17 00:00:00 2001 From: ayushbilala Date: Mon, 11 May 2026 10:08:17 +0530 Subject: [PATCH 2/4] [SPARK-53840][SQL] Refactor SHOW TABLES AS JSON to use unified ShowTablesJsonCommand --- .../sql/catalyst/parser/AstBuilder.scala | 11 +- .../catalyst/plans/logical/v2Commands.scala | 7 - .../analysis/ResolveSessionCatalog.scala | 17 +- .../apache/spark/sql/classic/Catalog.scala | 2 +- .../spark/sql/execution/SparkSqlParser.scala | 32 ++++ .../command/ShowTablesJsonCommand.scala | 152 ++++++++++++++++++ .../spark/sql/execution/command/tables.scala | 56 +------ .../datasources/v2/DataSourceV2Strategy.scala | 22 +-- .../datasources/v2/ShowTablesJsonExec.scala | 115 ------------- .../command/ShowTablesParserSuite.scala | 32 ++-- .../command/ShowTablesSuiteBase.scala | 51 +++++- 11 files changed, 262 insertions(+), 235 deletions(-) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala delete mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesJsonExec.scala diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index ab03621756ae1..27ec55ac27a8a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -5847,10 +5847,8 @@ class AstBuilder extends DataTypeAstBuilder } else { CurrentNamespace } - val asJson = ctx.JSON != null val pattern = Option(ctx.pattern).map(x => string(visitStringLit(x))) - val output = if (asJson) ShowTables.getJsonOutputAttrs else ShowTables.getOutputAttrs - ShowTables(ns, pattern, asJson, output) + ShowTables(ns, pattern) } /** @@ -5858,10 +5856,6 @@ class AstBuilder extends DataTypeAstBuilder */ override def visitShowTableExtended( ctx: ShowTableExtendedContext): LogicalPlan = withOrigin(ctx) { - val asJson = ctx.JSON != null - if (asJson && ctx.partitionSpec != null) { - throw QueryCompilationErrors.showTableExtendedJsonWithPartitionError() - } Option(ctx.partitionSpec).map { spec => val table = withOrigin(ctx.pattern) { if (ctx.identifierReference() != null) { @@ -5881,8 +5875,7 @@ class AstBuilder extends DataTypeAstBuilder } else { CurrentNamespace } - val output = if (asJson) ShowTables.getJsonOutputAttrs else ShowTablesUtils.getOutputAttrs - ShowTablesExtended(ns, string(visitStringLit(ctx.pattern)), asJson, output) + ShowTablesExtended(ns, string(visitStringLit(ctx.pattern))) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala index 55e3c4029a7f9..40cf5009b97dc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala @@ -1403,12 +1403,10 @@ case class RenameTable( case class ShowTables( namespace: LogicalPlan, pattern: Option[String], - asJson: Boolean = false, override val output: Seq[Attribute] = ShowTables.getOutputAttrs) extends UnaryCommand { override def child: LogicalPlan = namespace override protected def withNewChildInternal(newChild: LogicalPlan): ShowTables = copy(namespace = newChild) - override protected def stringArgs: Iterator[Any] = Iterator(pattern, output) } object ShowTables { @@ -1416,9 +1414,6 @@ object ShowTables { AttributeReference("namespace", StringType, nullable = false)(), AttributeReference("tableName", StringType, nullable = false)(), AttributeReference("isTemporary", BooleanType, nullable = false)()) - - def getJsonOutputAttrs: Seq[Attribute] = Seq( - AttributeReference("json_metadata", StringType, nullable = false)()) } /** @@ -1427,12 +1422,10 @@ object ShowTables { case class ShowTablesExtended( namespace: LogicalPlan, pattern: String, - asJson: Boolean = false, override val output: Seq[Attribute] = ShowTablesUtils.getOutputAttrs) extends UnaryCommand { override def child: LogicalPlan = namespace override protected def withNewChildInternal(newChild: LogicalPlan): ShowTablesExtended = copy(namespace = newChild) - override protected def stringArgs: Iterator[Any] = Iterator(pattern, output) } object ShowTablesUtils { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala index ff861332a7703..90f4aa353a7eb 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala @@ -367,23 +367,16 @@ class ResolveSessionCatalog(val catalogManager: CatalogManager) case d @ DropNamespace(ResolvedV1Database(db), _, _) if conf.useV1Command => DropDatabaseCommand(db, d.ifExists, d.cascade) - case ShowTables(ResolvedV1Database(db), pattern, asJson, output) if conf.useV1Command => - ShowTablesCommand(Some(db), pattern, output, asJson = asJson) + case ShowTables(ResolvedV1Database(db), pattern, output) if conf.useV1Command => + ShowTablesCommand(Some(db), pattern, output) - case ShowTablesExtended( - ResolvedV1Database(db), - pattern, - asJson, - output) => - val newOutput = if (asJson) { - output - } else if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) { + case ShowTablesExtended(ResolvedV1Database(db), pattern, output) => + val newOutput = if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) { output.head.withName("database") +: output.tail } else { output } - ShowTablesCommand(Some(db), Some(pattern), newOutput, - isExtended = true, asJson = asJson) + ShowTablesCommand(Some(db), Some(pattern), newOutput, isExtended = true) case ShowTablePartition( ResolvedTable(catalog, _, table: V1Table, _), diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala index 939b12a404fd2..40c40f6ea78aa 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala @@ -195,7 +195,7 @@ class Catalog(sparkSession: SparkSession) extends catalog.Catalog with Logging { private def makeTablesDataset(plan: ShowTables): Dataset[Table] = { val qe = sparkSession.sessionState.executePlan(plan) val catalog = qe.analyzed.collectFirst { - case ShowTables(r: ResolvedNamespace, _, _, _) => r.catalog + case ShowTables(r: ResolvedNamespace, _, _) => r.catalog case _: ShowTablesCommand => sparkSession.sessionState.catalogManager.v2SessionCatalog }.get diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 5993e86f5cce5..bce51f7c23fae 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -1496,6 +1496,38 @@ class SparkSqlAstBuilder extends AstBuilder { Option(ctx.pattern).map(x => string(visitStringLit(x)))) } + /** + * Create a [[ShowTablesJsonCommand]] or [[ShowTables]] command. + */ + override def visitShowTables(ctx: ShowTablesContext): LogicalPlan = withOrigin(ctx) { + if (ctx.JSON == null) return super.visitShowTables(ctx) + val ns = if (ctx.identifierReference() != null) { + withIdentClause(ctx.identifierReference, UnresolvedNamespace(_)) + } else { + CurrentNamespace + } + val pattern = Option(ctx.pattern).map(x => string(visitStringLit(x))) + ShowTablesJsonCommand(ns, pattern, isExtended = false) + } + + /** + * Create a [[ShowTablesJsonCommand]], [[ShowTablesExtended]], or [[ShowTablePartition]] command. + */ + override def visitShowTableExtended( + ctx: ShowTableExtendedContext): LogicalPlan = withOrigin(ctx) { + val asJson = ctx.JSON != null + if (asJson && ctx.partitionSpec != null) { + throw QueryCompilationErrors.showTableExtendedJsonWithPartitionError() + } + if (!asJson || ctx.partitionSpec != null) return super.visitShowTableExtended(ctx) + val ns = if (ctx.identifierReference() != null) { + withIdentClause(ctx.identifierReference, UnresolvedNamespace(_)) + } else { + CurrentNamespace + } + ShowTablesJsonCommand(ns, Some(string(visitStringLit(ctx.pattern))), isExtended = true) + } + override def visitDescribeProcedure( ctx: DescribeProcedureContext): LogicalPlan = withOrigin(ctx) { withIdentClause(ctx.identifierReference(), procIdentifier => diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala new file mode 100644 index 0000000000000..b925dc08279f2 --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import scala.collection.mutable.ArrayBuffer + +import org.json4s._ +import org.json4s.jackson.JsonMethods._ + +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.catalyst.analysis.ResolvedNamespace +import org.apache.spark.sql.catalyst.catalog.CatalogTableType +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.util.StringUtils +import org.apache.spark.sql.connector.catalog.{CatalogV2Util, TableCatalog, TableViewCatalog} +import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ +import org.apache.spark.sql.types.{MetadataBuilder, StringType} + +/** + * The command for `SHOW TABLES AS JSON` and `SHOW TABLE EXTENDED AS JSON`. + */ +case class ShowTablesJsonCommand( + child: LogicalPlan, + pattern: Option[String], + isExtended: Boolean, + override val output: Seq[Attribute] = Seq( + AttributeReference( + "json_metadata", + StringType, + nullable = false, + new MetadataBuilder() + .putString("comment", "JSON metadata of the tables") + .build())() + )) extends UnaryRunnableCommand { + + override def run(sparkSession: SparkSession): Seq[Row] = { + val jsonOutput = child match { + case ResolvedNamespace(catalog, ns, _) => + if (CatalogV2Util.isSessionCatalog(catalog)) { + runForSessionCatalog(sparkSession, ns) + } else { + runForV2Catalog(sparkSession, catalog.asTableCatalog, ns) + } + } + Seq(Row(compact(render(jsonOutput)))) + } + + private def runForSessionCatalog( + sparkSession: SparkSession, + ns: Seq[String]): JObject = { + val sessionCatalog = sparkSession.sessionState.catalog + val db = ns.headOption.getOrElse(sessionCatalog.getCurrentDatabase) + val tables = pattern + .map(p => sessionCatalog.listTables(db, p)) + .getOrElse(sessionCatalog.listTables(db)) + + val jsonTables = tables.map { tableIdent => + val isTemp = sessionCatalog.isTempView(tableIdent) + val namespace = tableIdent.database.toList + + if (isExtended) { + val tableType = if (isTemp) { + "VIEW" + } else { + val meta = sessionCatalog.getTempViewOrPermanentTableMetadata(tableIdent) + if (meta.tableType == CatalogTableType.VIEW) "VIEW" else "TABLE" + } + JObject( + "name" -> JString(tableIdent.table), + "catalog" -> JString( + sparkSession.sessionState.catalogManager.v2SessionCatalog.name()), + "namespace" -> JArray(namespace.map(JString(_))), + "type" -> JString(tableType), + "isTemporary" -> JBool(isTemp) + ) + } else { + JObject( + "name" -> JString(tableIdent.table), + "namespace" -> JArray(namespace.map(JString(_))), + "isTemporary" -> JBool(isTemp) + ) + } + }.toList + + JObject("tables" -> JArray(jsonTables)) + } + + private def runForV2Catalog( + sparkSession: SparkSession, + catalog: TableCatalog, + ns: Seq[String]): JObject = { + val identifiers = if (!isExtended) { + catalog match { + case mc: TableViewCatalog => + mc.listTableAndViewSummaries(ns.toArray).map(_.identifier()) + case _ => catalog.listTables(ns.toArray) + } + } else { + catalog.listTables(ns.toArray) + } + + val pat = pattern.getOrElse("*") + val filteredIdents = identifiers.filter { ident => + StringUtils.filterPattern(Seq(ident.name()), pat).nonEmpty + } + + val jsonRows = new ArrayBuffer[JObject]() + filteredIdents.foreach { ident => + val nsArray = JArray(ident.namespace().map(JString(_)).toList) + // Non-session V2 catalogs do not surface session temp views. + val isTemp = false + + val entry = if (isExtended) { + JObject( + "name" -> JString(ident.name()), + "catalog" -> JString(catalog.name()), + "namespace" -> nsArray, + "type" -> JString(if (isTemp) "VIEW" else "TABLE"), + "isTemporary" -> JBool(isTemp) + ) + } else { + JObject( + "name" -> JString(ident.name()), + "namespace" -> nsArray, + "isTemporary" -> JBool(isTemp) + ) + } + jsonRows += entry + } + + JObject("tables" -> JArray(jsonRows.toList)) + } + + override protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = + copy(child = newChild) +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 319f3284c7c66..78e9649add456 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -25,9 +25,6 @@ import scala.util.control.NonFatal import org.apache.hadoop.fs.{FileContext, FsConstants, Path} import org.apache.hadoop.fs.permission.{AclEntry, AclEntryScope, AclEntryType, FsAction, FsPermission} -import org.json4s._ -import org.json4s.jackson.JsonMethods._ - import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.{SQLConfHelper, TableIdentifier} import org.apache.spark.sql.catalyst.analysis.{ResolvedPersistentView, ResolvedTable, ResolvedTempView, UnresolvedAttribute} @@ -929,21 +926,9 @@ case class ShowTablesCommand( tableIdentifierPattern: Option[String], override val output: Seq[Attribute], isExtended: Boolean = false, - partitionSpec: Option[TablePartitionSpec] = None, - asJson: Boolean = false) extends LeafRunnableCommand { - - override protected def stringArgs: Iterator[Any] = - Iterator(databaseName, tableIdentifierPattern, output, isExtended, partitionSpec) + partitionSpec: Option[TablePartitionSpec] = None) extends LeafRunnableCommand { override def run(sparkSession: SparkSession): Seq[Row] = { - if (asJson) { - runAsJson(sparkSession) - } else { - runAsText(sparkSession) - } - } - - private def runAsText(sparkSession: SparkSession): Seq[Row] = { // Since we need to return a Seq of rows, we will call getTables directly // instead of calling tables in sparkSession. val catalog = sparkSession.sessionState.catalog @@ -986,45 +971,6 @@ case class ShowTablesCommand( Seq(Row(database, tableName, isTemp, s"$information\n")) } } - - private def runAsJson(sparkSession: SparkSession): Seq[Row] = { - val catalog = sparkSession.sessionState.catalog - val db = databaseName.getOrElse(catalog.getCurrentDatabase) - val tables = - tableIdentifierPattern.map(catalog.listTables(db, _)).getOrElse(catalog.listTables(db)) - - val jsonTables = tables.map { tableIdent => - val isTemp = catalog.isTempView(tableIdent) - val ns = tableIdent.database.toList - - if (isExtended) { - val tableType = if (isTemp) { - "VIEW" - } else { - val meta = catalog.getTempViewOrPermanentTableMetadata(tableIdent) - if (meta.tableType == CatalogTableType.VIEW) "VIEW" else "TABLE" - } - - JObject( - "name" -> JString(tableIdent.table), - "catalog" -> JString( - sparkSession.sessionState.catalogManager.v2SessionCatalog.name()), - "namespace" -> JArray(ns.map(JString(_))), - "type" -> JString(tableType), - "isTemporary" -> JBool(isTemp) - ) - } else { - JObject( - "name" -> JString(tableIdent.table), - "namespace" -> JArray(ns.map(JString(_))), - "isTemporary" -> JBool(isTemp) - ) - } - }.toList - - val jsonOutput = JObject("tables" -> JArray(jsonTables)) - Seq(Row(compact(render(jsonOutput)))) - } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala index 1309c844a0fa7..89330f08d1808 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala @@ -664,13 +664,8 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat case DropNamespace(ResolvedNamespace(catalog, ns, _), ifExists, cascade) => DropNamespaceExec(catalog, ns, ifExists, cascade) :: Nil - case ShowTables(ResolvedNamespace(catalog, ns, _), pattern, asJson, output) => - if (asJson) { - ShowTablesJsonExec( - output, catalog.asTableCatalog, ns, pattern.getOrElse("*"), isExtended = false) :: Nil - } else { - ShowTablesExec(output, catalog.asTableCatalog, ns, pattern) :: Nil - } + case ShowTables(ResolvedNamespace(catalog, ns, _), pattern, output) => + ShowTablesExec(output, catalog.asTableCatalog, ns, pattern) :: Nil // SHOW VIEWS on a v2 ViewCatalog. `ResolveSessionCatalog` rewrites the SHOW VIEWS plan to // v1 `ShowViewsCommand` only when the catalog is NOT a `ViewCatalog`; non-`ViewCatalog` @@ -681,17 +676,8 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat case ShowViews(ResolvedNamespace(catalog: ViewCatalog, ns, _), pattern, output) => ShowViewsExec(output, catalog, ns, pattern) :: Nil - case ShowTablesExtended( - ResolvedNamespace(catalog, ns, _), - pattern, - asJson, - output) => - if (asJson) { - ShowTablesJsonExec( - output, catalog.asTableCatalog, ns, pattern, isExtended = true) :: Nil - } else { - ShowTablesExtendedExec(output, catalog.asTableCatalog, ns, pattern) :: Nil - } + case ShowTablesExtended(ResolvedNamespace(catalog, ns, _), pattern, output) => + ShowTablesExtendedExec(output, catalog.asTableCatalog, ns, pattern) :: Nil case ShowTablePartition(r: ResolvedTable, part, output) => ShowTablePartitionExec(output, r.catalog, r.identifier, diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesJsonExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesJsonExec.scala deleted file mode 100644 index 8af65d76c344b..0000000000000 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesJsonExec.scala +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources.v2 - -import scala.collection.mutable.ArrayBuffer - -import org.json4s._ -import org.json4s.jackson.JsonMethods._ - -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.util.StringUtils -import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Identifier, TableCatalog, TableViewCatalog} -import org.apache.spark.sql.execution.LeafExecNode - -/** - * Physical plan node for `SHOW TABLES AS JSON` and `SHOW TABLE EXTENDED AS JSON`. - * - * For a [[TableViewCatalog]] (non-extended only), listing is done via - * [[TableViewCatalog#listTableAndViewSummaries]] so that views appear alongside tables, - * matching the v1 `SHOW TABLES` semantics. - */ -case class ShowTablesJsonExec( - output: Seq[Attribute], - catalog: TableCatalog, - namespace: Seq[String], - pattern: String, - isExtended: Boolean) extends V2CommandExec with LeafExecNode { - - override protected def run(): Seq[InternalRow] = { - val identifiers: Array[Identifier] = if (!isExtended) { - catalog match { - case mc: TableViewCatalog => - mc.listTableAndViewSummaries(namespace.toArray).map(_.identifier()) - case _ => catalog.listTables(namespace.toArray) - } - } else { - catalog.listTables(namespace.toArray) - } - - val filteredIdents = identifiers.filter { ident => - StringUtils.filterPattern(Seq(ident.name()), pattern).nonEmpty - } - - val jsonRows = new ArrayBuffer[JObject]() - filteredIdents.foreach { ident => - jsonRows += toJsonEntry(ident.name(), ident.namespace(), isTempView(ident)) - } - - // For non-session V2 catalogs that don't surface temp views via listTables() or - // listTableAndViewSummaries(), fetch them separately. For V2SessionCatalog, - // listTables() already includes local temp views, so we skip this to avoid duplicates. - // For TableViewCatalog (non-extended path), views come from listTableAndViewSummaries(). - if (!CatalogV2Util.isSessionCatalog(catalog) && - (isExtended || !catalog.isInstanceOf[TableViewCatalog])) { - val sessionCatalog = session.sessionState.catalog - val db = namespace match { - case Seq(db) => db - case _ => "" - } - sessionCatalog.listTempViews(db, pattern).foreach { tempView => - jsonRows += toJsonEntry( - tempView.identifier.table, - tempView.identifier.database.toArray, - isTemporary = true) - } - } - - val jsonOutput = JObject("tables" -> JArray(jsonRows.toList)) - Seq(toCatalystRow(compact(render(jsonOutput)))) - } - - private def toJsonEntry( - name: String, - namespace: Array[String], - isTemporary: Boolean): JObject = { - val nsArray = JArray(namespace.map(JString(_)).toList) - if (isExtended) { - JObject( - "name" -> JString(name), - "catalog" -> JString(catalog.name()), - "namespace" -> nsArray, - "type" -> JString(if (isTemporary) "VIEW" else "TABLE"), - "isTemporary" -> JBool(isTemporary) - ) - } else { - JObject( - "name" -> JString(name), - "namespace" -> nsArray, - "isTemporary" -> JBool(isTemporary) - ) - } - } - - private def isTempView(ident: Identifier): Boolean = { - if (CatalogV2Util.isSessionCatalog(catalog)) { - session.sessionState.catalog.isTempView((ident.namespace() :+ ident.name()).toSeq) - } else false - } -} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala index fd883c67389a9..06ac05b48fa28 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesParserSuite.scala @@ -21,6 +21,7 @@ import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, CurrentNamespace, UnresolvedNamespace, UnresolvedPartitionSpec, UnresolvedTable} import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan import org.apache.spark.sql.catalyst.plans.logical.{ShowTablePartition, ShowTables, ShowTablesExtended} +import org.apache.spark.sql.execution.command.ShowTablesJsonCommand import org.apache.spark.sql.test.SharedSparkSession class ShowTablesParserSuite extends AnalysisTest with SharedSparkSession { @@ -51,18 +52,16 @@ class ShowTablesParserSuite extends AnalysisTest with SharedSparkSession { } test("show tables as json") { + val parse = spark.sessionState.sqlParser.parsePlan _ comparePlans( - parsePlan("SHOW TABLES AS JSON"), - ShowTables(CurrentNamespace, None, asJson = true, - output = ShowTables.getJsonOutputAttrs)) + parse("SHOW TABLES AS JSON"), + ShowTablesJsonCommand(CurrentNamespace, None, isExtended = false)) comparePlans( - parsePlan("SHOW TABLES IN ns1 AS JSON"), - ShowTables(UnresolvedNamespace(Seq("ns1")), None, asJson = true, - output = ShowTables.getJsonOutputAttrs)) + parse("SHOW TABLES IN ns1 AS JSON"), + ShowTablesJsonCommand(UnresolvedNamespace(Seq("ns1")), None, isExtended = false)) comparePlans( - parsePlan("SHOW TABLES IN ns1 LIKE '*test*' AS JSON"), - ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*"), asJson = true, - output = ShowTables.getJsonOutputAttrs)) + parse("SHOW TABLES IN ns1 LIKE '*test*' AS JSON"), + ShowTablesJsonCommand(UnresolvedNamespace(Seq("ns1")), Some("*test*"), isExtended = false)) } test("show table extended") { @@ -98,20 +97,21 @@ class ShowTablesParserSuite extends AnalysisTest with SharedSparkSession { } test("show table extended as json") { + val parse = spark.sessionState.sqlParser.parsePlan _ comparePlans( - parsePlan("SHOW TABLE EXTENDED LIKE '*test*' AS JSON"), - ShowTablesExtended(CurrentNamespace, "*test*", asJson = true, - output = ShowTables.getJsonOutputAttrs)) + parse("SHOW TABLE EXTENDED LIKE '*test*' AS JSON"), + ShowTablesJsonCommand(CurrentNamespace, Some("*test*"), isExtended = true)) comparePlans( - parsePlan(s"SHOW TABLE EXTENDED IN $catalog.ns1.ns2 LIKE '*test*' AS JSON"), - ShowTablesExtended(UnresolvedNamespace(Seq(catalog, "ns1", "ns2")), "*test*", - asJson = true, output = ShowTables.getJsonOutputAttrs)) + parse(s"SHOW TABLE EXTENDED IN $catalog.ns1.ns2 LIKE '*test*' AS JSON"), + ShowTablesJsonCommand( + UnresolvedNamespace(Seq(catalog, "ns1", "ns2")), Some("*test*"), isExtended = true)) } test("show table extended as json with partition should fail") { checkError( exception = intercept[AnalysisException] { - parsePlan("SHOW TABLE EXTENDED LIKE '*test*' PARTITION(ds='2008-04-09') AS JSON") + spark.sessionState.sqlParser.parsePlan( + "SHOW TABLE EXTENDED LIKE '*test*' PARTITION(ds='2008-04-09') AS JSON") }, condition = "UNSUPPORTED_FEATURE.SHOW_TABLE_EXTENDED_JSON_WITH_PARTITION", parameters = Map.empty diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala index 92816793604bd..73c3fb4164631 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala @@ -527,7 +527,6 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { val jsonStr = df.collect()(0).getString(0) val json = parse(jsonStr) val tables = (json \ "tables").asInstanceOf[JArray].arr - // Verify no duplicate entries val names = tables.map(t => (t \ "name").extract[String]) assert(names.distinct.length == names.length, s"Duplicate entries found: $names") val tempView = tables.find(t => (t \ "name").extract[String] == "tv") @@ -553,7 +552,6 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { val tables = (json \ "tables").asInstanceOf[JArray].arr assert(tables.length == 2, s"Expected 2 entries (tbl + $localTmpViewName), got: $tables") - // Verify no duplicate entries val names = tables.map(e => (e \ "name").extract[String]) assert(names.distinct.length == names.length, s"Duplicate entries found: $names") @@ -652,4 +650,53 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { } } } + + test("SHOW TABLES AS JSON - LIKE pattern filters results") { + withNamespaceAndTable("ns", "tab_matched") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + withTable(s"$catalog.ns.other") { + sql(s"CREATE TABLE $catalog.ns.other (id INT) $defaultUsing") + val jsonStr = sql(s"SHOW TABLES IN $catalog.ns LIKE 'tab_matched' AS JSON") + .collect()(0).getString(0) + val tables = (parse(jsonStr) \ "tables").asInstanceOf[JArray].arr + assert(tables.exists(e => (e \ "name").extract[String] == "tab_matched")) + assert(!tables.exists(e => (e \ "name").extract[String] == "other")) + } + } + } + + test("SHOW TABLES AS JSON - without IN clause uses current namespace") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + sql(s"USE $catalog.ns") + val jsonStr = sql("SHOW TABLES AS JSON").collect()(0).getString(0) + val tables = (parse(jsonStr) \ "tables").asInstanceOf[JArray].arr + assert(tables.exists(e => (e \ "name").extract[String] == "tbl")) + } + } + + test("SHOW TABLES AS JSON - output wraps table entries in a tables key") { + withNamespace(s"$catalog.ns") { + sql(s"CREATE NAMESPACE $catalog.ns") + val jsonStr = sql(s"SHOW TABLES IN $catalog.ns AS JSON").collect()(0).getString(0) + val json = parse(jsonStr) + assert(json \ "tables" != JNothing) + } + } + + test("SHOW TABLES AS JSON - same result regardless of useV1Command") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t (id INT) $defaultUsing") + val query = s"SHOW TABLES IN $catalog.ns AS JSON" + val resultDefault = sql(query).collect()(0).getString(0) + withSQLConf(SQLConf.LEGACY_USE_V1_COMMAND.key -> "true") { + val resultV1 = sql(query).collect()(0).getString(0) + val namesDefault = (parse(resultDefault) \ "tables").asInstanceOf[JArray].arr + .map(e => (e \ "name").extract[String]).sorted + val namesV1 = (parse(resultV1) \ "tables").asInstanceOf[JArray].arr + .map(e => (e \ "name").extract[String]).sorted + assert(namesDefault === namesV1) + } + } + } } From 99aec51741b5ce676f96e896bad72f3cfe13fb15 Mon Sep 17 00:00:00 2001 From: ayushbilala Date: Fri, 5 Jun 2026 18:37:49 +0530 Subject: [PATCH 3/4] [SPARK-53840][SQL] Address review: use granular tableType, revert cosmetic diffs --- .../spark/sql/catalyst/parser/AstBuilder.scala | 3 +-- .../analysis/ResolveSessionCatalog.scala | 5 ++++- .../command/ShowTablesJsonCommand.scala | 17 ++++++++--------- .../spark/sql/execution/command/tables.scala | 1 + .../datasources/v2/DataSourceV2Strategy.scala | 5 ++++- .../datasources/v2/ShowTablesExec.scala | 12 ++++++------ .../execution/command/ShowTablesSuiteBase.scala | 13 +++++++------ .../execution/command/v1/ShowTablesSuite.scala | 1 + 8 files changed, 32 insertions(+), 25 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 27ec55ac27a8a..ccca53f351e8b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -5847,8 +5847,7 @@ class AstBuilder extends DataTypeAstBuilder } else { CurrentNamespace } - val pattern = Option(ctx.pattern).map(x => string(visitStringLit(x))) - ShowTables(ns, pattern) + ShowTables(ns, Option(ctx.pattern).map(x => string(visitStringLit(x)))) } /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala index 90f4aa353a7eb..cfd52707bbc2c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala @@ -370,7 +370,10 @@ class ResolveSessionCatalog(val catalogManager: CatalogManager) case ShowTables(ResolvedV1Database(db), pattern, output) if conf.useV1Command => ShowTablesCommand(Some(db), pattern, output) - case ShowTablesExtended(ResolvedV1Database(db), pattern, output) => + case ShowTablesExtended( + ResolvedV1Database(db), + pattern, + output) => val newOutput = if (conf.getConf(SQLConf.LEGACY_KEEP_COMMAND_OUTPUT_SCHEMA)) { output.head.withName("database") +: output.tail } else { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala index b925dc08279f2..7c3ed1564e447 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ShowTablesJsonCommand.scala @@ -22,9 +22,9 @@ import scala.collection.mutable.ArrayBuffer import org.json4s._ import org.json4s.jackson.JsonMethods._ +import org.apache.spark.SparkException import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.analysis.ResolvedNamespace -import org.apache.spark.sql.catalyst.catalog.CatalogTableType import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.util.StringUtils @@ -57,6 +57,9 @@ case class ShowTablesJsonCommand( } else { runForV2Catalog(sparkSession, catalog.asTableCatalog, ns) } + case other => + throw SparkException.internalError( + s"Unexpected child in ShowTablesJsonCommand: ${other.getClass.getSimpleName}") } Seq(Row(compact(render(jsonOutput)))) } @@ -78,8 +81,7 @@ case class ShowTablesJsonCommand( val tableType = if (isTemp) { "VIEW" } else { - val meta = sessionCatalog.getTempViewOrPermanentTableMetadata(tableIdent) - if (meta.tableType == CatalogTableType.VIEW) "VIEW" else "TABLE" + sessionCatalog.getTempViewOrPermanentTableMetadata(tableIdent).tableType.name } JObject( "name" -> JString(tableIdent.table), @@ -123,22 +125,19 @@ case class ShowTablesJsonCommand( val jsonRows = new ArrayBuffer[JObject]() filteredIdents.foreach { ident => val nsArray = JArray(ident.namespace().map(JString(_)).toList) - // Non-session V2 catalogs do not surface session temp views. - val isTemp = false - val entry = if (isExtended) { JObject( "name" -> JString(ident.name()), "catalog" -> JString(catalog.name()), "namespace" -> nsArray, - "type" -> JString(if (isTemp) "VIEW" else "TABLE"), - "isTemporary" -> JBool(isTemp) + "type" -> JString("TABLE"), + "isTemporary" -> JBool(false) ) } else { JObject( "name" -> JString(ident.name()), "namespace" -> nsArray, - "isTemporary" -> JBool(isTemp) + "isTemporary" -> JBool(false) ) } jsonRows += entry diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 78e9649add456..ca534706635a1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -25,6 +25,7 @@ import scala.util.control.NonFatal import org.apache.hadoop.fs.{FileContext, FsConstants, Path} import org.apache.hadoop.fs.permission.{AclEntry, AclEntryScope, AclEntryType, FsAction, FsPermission} + import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.{SQLConfHelper, TableIdentifier} import org.apache.spark.sql.catalyst.analysis.{ResolvedPersistentView, ResolvedTable, ResolvedTempView, UnresolvedAttribute} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala index 89330f08d1808..4fd7d993cc3d0 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala @@ -676,7 +676,10 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat case ShowViews(ResolvedNamespace(catalog: ViewCatalog, ns, _), pattern, output) => ShowViewsExec(output, catalog, ns, pattern) :: Nil - case ShowTablesExtended(ResolvedNamespace(catalog, ns, _), pattern, output) => + case ShowTablesExtended( + ResolvedNamespace(catalog, ns, _), + pattern, + output) => ShowTablesExtendedExec(output, catalog.asTableCatalog, ns, pattern) :: Nil case ShowTablePartition(r: ResolvedTable, part, output) => diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala index c74d9f8fa748a..8680785e0815f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala @@ -41,19 +41,19 @@ case class ShowTablesExec( namespace: Seq[String], pattern: Option[String]) extends V2CommandExec with LeafExecNode { override protected def run(): Seq[InternalRow] = { + val rows = new ArrayBuffer[InternalRow]() + val identifiers: Array[Identifier] = catalog match { case mc: TableViewCatalog => mc.listTableAndViewSummaries(namespace.toArray).map(_.identifier()) case _ => catalog.listTables(namespace.toArray) } - val filteredIdents = identifiers.filter { ident => - pattern.map(StringUtils.filterPattern(Seq(ident.name()), _).nonEmpty).getOrElse(true) + identifiers.foreach { ident => + if (pattern.map(StringUtils.filterPattern(Seq(ident.name()), _).nonEmpty).getOrElse(true)) { + rows += toCatalystRow(ident.namespace().quoted, ident.name(), isTempView(ident, catalog)) + } } - val rows = new ArrayBuffer[InternalRow]() - filteredIdents.foreach { ident => - rows += toCatalystRow(ident.namespace().quoted, ident.name(), isTempView(ident, catalog)) - } rows.toSeq } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala index 73c3fb4164631..091d69a46df03 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala @@ -39,6 +39,7 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { override val command = "SHOW TABLES" protected def defaultNamespace: Seq[String] + protected def expectedTableTypeInJson: String = "TABLE" protected def runShowTablesSql(sqlText: String, expected: Seq[Row]): Unit = { val df = spark.sql(sqlText) @@ -511,7 +512,7 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { assert(tables.length == 1) val entry = tables.head assert((entry \ "name").extract[String] == "tbl") - assert((entry \ "type").extract[String] == "TABLE") + assert((entry \ "type").extract[String] == expectedTableTypeInJson) assert((entry \ "isTemporary").extract[Boolean] == false) assert((entry \ "catalog").isInstanceOf[JString]) assert((entry \ "namespace").isInstanceOf[JArray]) @@ -558,7 +559,7 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { val tblEntry = tables.find(e => (e \ "name").extract[String] == "tbl") assert(tblEntry.isDefined) assert((tblEntry.get \ "isTemporary").extract[Boolean] == false) - assert((tblEntry.get \ "type").extract[String] == "TABLE") + assert((tblEntry.get \ "type").extract[String] == expectedTableTypeInJson) val tempViewEntry = tables.find(e => (e \ "name").extract[String] == localTmpViewName) assert(tempViewEntry.isDefined) @@ -651,7 +652,7 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { } } - test("SHOW TABLES AS JSON - LIKE pattern filters results") { + test("SHOW TABLES AS JSON with LIKE pattern") { withNamespaceAndTable("ns", "tab_matched") { t => sql(s"CREATE TABLE $t (id INT) $defaultUsing") withTable(s"$catalog.ns.other") { @@ -665,7 +666,7 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { } } - test("SHOW TABLES AS JSON - without IN clause uses current namespace") { + test("SHOW TABLES AS JSON without IN clause") { withNamespaceAndTable("ns", "tbl") { t => sql(s"CREATE TABLE $t (id INT) $defaultUsing") sql(s"USE $catalog.ns") @@ -675,7 +676,7 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { } } - test("SHOW TABLES AS JSON - output wraps table entries in a tables key") { + test("SHOW TABLES AS JSON wraps entries in tables key") { withNamespace(s"$catalog.ns") { sql(s"CREATE NAMESPACE $catalog.ns") val jsonStr = sql(s"SHOW TABLES IN $catalog.ns AS JSON").collect()(0).getString(0) @@ -684,7 +685,7 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { } } - test("SHOW TABLES AS JSON - same result regardless of useV1Command") { + test("SHOW TABLES AS JSON with useV1Command") { withNamespaceAndTable("ns", "tbl") { t => sql(s"CREATE TABLE $t (id INT) $defaultUsing") val query = s"SHOW TABLES IN $catalog.ns AS JSON" diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala index 9d353fde898f2..3928f63fe2463 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.internal.SQLConf */ trait ShowTablesSuiteBase extends command.ShowTablesSuiteBase with command.TestsV1AndV2Commands { override def defaultNamespace: Seq[String] = Seq("default") + override def expectedTableTypeInJson: String = "MANAGED" private def withSourceViews(f: => Unit): Unit = { withTable("source", "source2") { From 543033a5d748e9174687a80d05907c280ab4f11c Mon Sep 17 00:00:00 2001 From: ayushbilala Date: Sat, 6 Jun 2026 23:51:19 +0530 Subject: [PATCH 4/4] [SPARK-53840][SQL] Fix V2 temp-view test failures, rename parity test --- .../command/ShowTablesSuiteBase.scala | 29 ++++++++++++------- .../command/v2/ShowTablesSuite.scala | 1 + 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala index 091d69a46df03..5d40421045b3d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuiteBase.scala @@ -40,6 +40,7 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { override val command = "SHOW TABLES" protected def defaultNamespace: Seq[String] protected def expectedTableTypeInJson: String = "TABLE" + protected def expectsSessionTempViews: Boolean = true protected def runShowTablesSql(sqlText: String, expected: Seq[Row]): Unit = { val df = spark.sql(sqlText) @@ -531,8 +532,12 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { val names = tables.map(t => (t \ "name").extract[String]) assert(names.distinct.length == names.length, s"Duplicate entries found: $names") val tempView = tables.find(t => (t \ "name").extract[String] == "tv") - assert(tempView.isDefined) - assert((tempView.get \ "isTemporary").extract[Boolean] == true) + if (expectsSessionTempViews) { + assert(tempView.isDefined) + assert((tempView.get \ "isTemporary").extract[Boolean] == true) + } else { + assert(tempView.isEmpty) + } } } } @@ -551,20 +556,22 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { val jsonStr = df.collect()(0).getString(0) val json = parse(jsonStr) val tables = (json \ "tables").asInstanceOf[JArray].arr - assert(tables.length == 2, s"Expected 2 entries (tbl + $localTmpViewName), got: $tables") - - val names = tables.map(e => (e \ "name").extract[String]) - assert(names.distinct.length == names.length, s"Duplicate entries found: $names") val tblEntry = tables.find(e => (e \ "name").extract[String] == "tbl") assert(tblEntry.isDefined) assert((tblEntry.get \ "isTemporary").extract[Boolean] == false) assert((tblEntry.get \ "type").extract[String] == expectedTableTypeInJson) - val tempViewEntry = tables.find(e => (e \ "name").extract[String] == localTmpViewName) - assert(tempViewEntry.isDefined) - assert((tempViewEntry.get \ "isTemporary").extract[Boolean] == true) - assert((tempViewEntry.get \ "type").extract[String] == "VIEW") + if (expectsSessionTempViews) { + assert(tables.length == 2) + val tempViewEntry = + tables.find(e => (e \ "name").extract[String] == localTmpViewName) + assert(tempViewEntry.isDefined) + assert((tempViewEntry.get \ "isTemporary").extract[Boolean] == true) + assert((tempViewEntry.get \ "type").extract[String] == "VIEW") + } else { + assert(tables.length == 1) + } } } } @@ -685,7 +692,7 @@ trait ShowTablesSuiteBase extends QueryTest with DDLCommandTestUtils { } } - test("SHOW TABLES AS JSON with useV1Command") { + test("SHOW TABLES AS JSON produces same result for useV1Command true and false") { withNamespaceAndTable("ns", "tbl") { t => sql(s"CREATE TABLE $t (id INT) $defaultUsing") val query = s"SHOW TABLES IN $catalog.ns AS JSON" diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala index 5719fbee370a8..96e61014980e1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala @@ -26,6 +26,7 @@ import org.apache.spark.util.Utils */ class ShowTablesSuite extends command.ShowTablesSuiteBase with CommandSuiteBase { override def defaultNamespace: Seq[String] = Nil + override def expectsSessionTempViews: Boolean = false // The test fails for V1 catalog with the error: // org.apache.spark.sql.AnalysisException: