From 9af03e9140fb94ab34a8fa95a5ec3954ab355936 Mon Sep 17 00:00:00 2001 From: allisonwang-db Date: Fri, 23 Jul 2021 10:56:16 -0700 Subject: [PATCH 1/2] nested fields --- .../spark/sql/catalyst/analysis/Analyzer.scala | 5 +++-- .../sql/catalyst/analysis/AnalysisSuite.scala | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 91f6a06390135..1359d051920a9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -2553,8 +2553,9 @@ class Analyzer(override val catalogManager: CatalogManager) // a table `t` has two columns `c1` and `c2`, for query `SELECT ... FROM t // GROUP BY c1 HAVING c2 = 0`, even though we can resolve column `c2` here, we // should undo it later and fail with "Column c2 not found". - agg.child.resolve(u.nameParts, resolver).map(TempResolvedColumn(_, u.nameParts)) - .getOrElse(u) + agg.child.resolve(u.nameParts, resolver).map(_.transformUp { + case a: Attribute => TempResolvedColumn(a, u.nameParts) + }).getOrElse(u) } catch { case _: AnalysisException => u } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala index 5cef24310e355..2e2e2db8119c7 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala @@ -1115,4 +1115,22 @@ class AnalysisSuite extends AnalysisTest with Matchers { Seq("grouping_id() can only be used with GroupingSets/Cube/Rollup"), false) } + + test("SPARK-36275: Resolve aggregate functions should work with nested fields") { + assertAnalysisSuccess(parsePlan( + """ + |SELECT c.x, SUM(c.y) + |FROM VALUES NAMED_STRUCT('x', 'A', 'y', 1), NAMED_STRUCT('x', 'A', 'y', 2) AS t(c) + |GROUP BY c.x + |HAVING c.x > 1 + |""".stripMargin)) + + assertAnalysisSuccess(parsePlan( + """ + |SELECT c.x, SUM(c.y) + |FROM VALUES NAMED_STRUCT('x', 'A', 'y', 1), NAMED_STRUCT('x', 'A', 'y', 2) AS t(c) + |GROUP BY c.x + |ORDER BY c.x + |""".stripMargin)) + } } From fb3931c6149d0d1d5962c5475f3d67b8430db07e Mon Sep 17 00:00:00 2001 From: allisonwang-db Date: Mon, 26 Jul 2021 12:55:48 -0700 Subject: [PATCH 2/2] update --- .../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 5 +++-- .../spark/sql/catalyst/analysis/AnalysisSuite.scala | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 1359d051920a9..3e048ed9b6933 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -2553,8 +2553,9 @@ class Analyzer(override val catalogManager: CatalogManager) // a table `t` has two columns `c1` and `c2`, for query `SELECT ... FROM t // GROUP BY c1 HAVING c2 = 0`, even though we can resolve column `c2` here, we // should undo it later and fail with "Column c2 not found". - agg.child.resolve(u.nameParts, resolver).map(_.transformUp { - case a: Attribute => TempResolvedColumn(a, u.nameParts) + agg.child.resolve(u.nameParts, resolver).map({ + case a: Alias => TempResolvedColumn(a.child, u.nameParts) + case o => TempResolvedColumn(o, u.nameParts) }).getOrElse(u) } catch { case _: AnalysisException => u diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala index 2e2e2db8119c7..6ddc6b7eb716b 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala @@ -1132,5 +1132,13 @@ class AnalysisSuite extends AnalysisTest with Matchers { |GROUP BY c.x |ORDER BY c.x |""".stripMargin)) + + assertAnalysisError(parsePlan( + """ + |SELECT c.x + |FROM VALUES NAMED_STRUCT('x', 'A', 'y', 1), NAMED_STRUCT('x', 'A', 'y', 2) AS t(c) + |GROUP BY c.x + |ORDER BY c.x + c.y + |""".stripMargin), "cannot resolve 'c.y' given input columns: [x]" :: Nil) } }