diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
index dbb12d56f9497994dfe9ca212f576ed7cbf4eed9..dba69659afc804ac9aed5fa81c32b37fb609e8fe 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala
@@ -105,7 +105,7 @@ abstract class LogicalPlan extends QueryPlan[LogicalPlan] with Logging {
}
/**
- * Optionally resolves the given string to a [[NamedExpression]] using the input from all child
+ * Optionally resolves the given strings to a [[NamedExpression]] using the input from all child
* nodes of this LogicalPlan. The attribute is expressed as
* as string in the following form: `[scope].AttributeName.[nested].[fields]...`.
*/
@@ -116,7 +116,7 @@ abstract class LogicalPlan extends QueryPlan[LogicalPlan] with Logging {
resolve(nameParts, children.flatMap(_.output), resolver, throwErrors)
/**
- * Optionally resolves the given string to a [[NamedExpression]] based on the output of this
+ * Optionally resolves the given strings to a [[NamedExpression]] based on the output of this
* LogicalPlan. The attribute is expressed as string in the following form:
* `[scope].AttributeName.[nested].[fields]...`.
*/
@@ -126,6 +126,57 @@ abstract class LogicalPlan extends QueryPlan[LogicalPlan] with Logging {
throwErrors: Boolean = false): Option[NamedExpression] =
resolve(nameParts, output, resolver, throwErrors)
+ /**
+ * Given an attribute name, split it to name parts by dot, but
+ * don't split the name parts quoted by backticks, for example,
+ * `ab.cd`.`efg` should be split into two parts "ab.cd" and "efg".
+ */
+ def resolveQuoted(
+ name: String,
+ resolver: Resolver): Option[NamedExpression] = {
+ resolve(parseAttributeName(name), resolver, true)
+ }
+
+ /**
+ * Internal method, used to split attribute name by dot with backticks rule.
+ * Backticks must appear in pairs, and the quoted string must be a complete name part,
+ * which means `ab..c`e.f is not allowed.
+ * Escape character is not supported now, so we can't use backtick inside name part.
+ */
+ private def parseAttributeName(name: String): Seq[String] = {
+ val e = new AnalysisException(s"syntax error in attribute name: $name")
+ val nameParts = scala.collection.mutable.ArrayBuffer.empty[String]
+ val tmp = scala.collection.mutable.ArrayBuffer.empty[Char]
+ var inBacktick = false
+ var i = 0
+ while (i < name.length) {
+ val char = name(i)
+ if (inBacktick) {
+ if (char == '`') {
+ inBacktick = false
+ if (i + 1 < name.length && name(i + 1) != '.') throw e
+ } else {
+ tmp += char
+ }
+ } else {
+ if (char == '`') {
+ if (tmp.nonEmpty) throw e
+ inBacktick = true
+ } else if (char == '.') {
+ if (tmp.isEmpty) throw e
+ nameParts += tmp.mkString
+ tmp.clear()
+ } else {
+ tmp += char
+ }
+ }
+ i += 1
+ }
+ if (tmp.isEmpty || inBacktick) throw e
+ nameParts += tmp.mkString
+ nameParts.toSeq
+ }
+
/**
* Resolve the given `name` string against the given attribute, returning either 0 or 1 match.
*
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index c820a673575ff2cae1897671721e7790dbdac9ed..4fd5105c2744376bbf122a7fd346f157e321508e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -160,7 +160,7 @@ class DataFrame private[sql](
}
protected[sql] def resolve(colName: String): NamedExpression = {
- queryExecution.analyzed.resolve(colName.split("\\."), sqlContext.analyzer.resolver).getOrElse {
+ queryExecution.analyzed.resolveQuoted(colName, sqlContext.analyzer.resolver).getOrElse {
throw new AnalysisException(
s"""Cannot resolve column name "$colName" among (${schema.fieldNames.mkString(", ")})""")
}
@@ -168,7 +168,7 @@ class DataFrame private[sql](
protected[sql] def numericColumns: Seq[Expression] = {
schema.fields.filter(_.dataType.isInstanceOf[NumericType]).map { n =>
- queryExecution.analyzed.resolve(n.name.split("\\."), sqlContext.analyzer.resolver).get
+ queryExecution.analyzed.resolveQuoted(n.name, sqlContext.analyzer.resolver).get
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 52aa1f6558f80766cd7ada20c89839cd6cafc338..1d5f6b3aad6fd703c19356adffd41a3d94e3f1cb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -459,6 +459,33 @@ class DataFrameSuite extends QueryTest {
assert(complexData.filter(complexData("m")(complexData("s")("value")) === 1).count() == 1)
}
+ test("SPARK-7551: support backticks for DataFrame attribute resolution") {
+ val df = TestSQLContext.jsonRDD(TestSQLContext.sparkContext.makeRDD(
+ """{"a.b": {"c": {"d..e": {"f": 1}}}}""" :: Nil))
+ checkAnswer(
+ df.select(df("`a.b`.c.`d..e`.`f`")),
+ Row(1)
+ )
+
+ val df2 = TestSQLContext.jsonRDD(TestSQLContext.sparkContext.makeRDD(
+ """{"a b": {"c": {"d e": {"f": 1}}}}""" :: Nil))
+ checkAnswer(
+ df2.select(df2("`a b`.c.d e.f")),
+ Row(1)
+ )
+
+ def checkError(testFun: => Unit): Unit = {
+ val e = intercept[org.apache.spark.sql.AnalysisException] {
+ testFun
+ }
+ assert(e.getMessage.contains("syntax error in attribute name:"))
+ }
+ checkError(df("`abc.`c`"))
+ checkError(df("`abc`..d"))
+ checkError(df("`a`.b."))
+ checkError(df("`a.b`.c.`d"))
+ }
+
test("SPARK-7324 dropDuplicates") {
val testData = TestSQLContext.sparkContext.parallelize(
(2, 1, 2) :: (1, 1, 1) ::