From c7c55637bfc523237f5cc5c5b61837b1e3d5fdfc Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Fri, 19 Feb 2016 12:22:22 -0800
Subject: [PATCH] [SPARK-13384][SQL] Keep attribute qualifiers after dedup in
Analyzer
JIRA: https://issues.apache.org/jira/browse/SPARK-13384
## What changes were proposed in this pull request?
When we de-duplicate attributes in Analyzer, we create new attributes. However, we don't keep original qualifiers. Some plans will be failed to analysed. We should keep original qualifiers in new attributes.
## How was the this patch tested?
Unit test is added.
Author: Liang-Chi Hsieh <viirya@gmail.com>
Closes #11261 from viirya/keep-attr-qualifiers.
---
.../spark/sql/catalyst/analysis/Analyzer.scala | 3 ++-
.../sql/catalyst/analysis/AnalysisSuite.scala | 14 ++++++++++++++
2 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 004c1caaff..8368657b2b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -432,7 +432,8 @@ class Analyzer(
case r if r == oldRelation => newRelation
} transformUp {
case other => other transformExpressions {
- case a: Attribute => attributeRewrites.get(a).getOrElse(a)
+ case a: Attribute =>
+ attributeRewrites.get(a).getOrElse(a).withQualifiers(a.qualifiers)
}
}
newRight
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index f85ae24e04..92db02944c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.Inner
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.types._
@@ -336,4 +337,17 @@ class AnalysisSuite extends AnalysisTest {
val plan = relation.select(CaseWhen(Seq((Literal(true), 'a.attr)), 'b).as("val"))
assertAnalysisSuccess(plan)
}
+
+ test("Keep attribute qualifiers after dedup") {
+ val input = LocalRelation('key.int, 'value.string)
+
+ val query =
+ Project(Seq($"x.key", $"y.key"),
+ Join(
+ Project(Seq($"x.key"), Subquery("x", input)),
+ Project(Seq($"y.key"), Subquery("y", input)),
+ Inner, None))
+
+ assertAnalysisSuccess(query)
+ }
}
--
GitLab