Skip to content
Snippets Groups Projects
Commit 22271f96 authored by Josh Rosen's avatar Josh Rosen Committed by Reynold Xin
Browse files

[SPARK-5462] [SQL] Use analyzed query plan in DataFrame.apply()

This patch changes DataFrame's `apply()` method to use an analyzed query plan when resolving column names.  This fixes a bug where `apply` would throw "invalid call to qualifiers on unresolved object" errors when called on DataFrames constructed via `SQLContext.sql()`.

Author: Josh Rosen <joshrosen@databricks.com>

Closes #4282 from JoshRosen/SPARK-5462 and squashes the following commits:

b9e6da2 [Josh Rosen] [SPARK-5462] Use analyzed query plan in DataFrame.apply().
parent 5c746eed
No related branches found
No related tags found
No related merge requests found
......@@ -111,14 +111,16 @@ class DataFrame protected[sql](
/** Returns the list of numeric columns, useful for doing aggregation. */
protected[sql] def numericColumns: Seq[Expression] = {
schema.fields.filter(_.dataType.isInstanceOf[NumericType]).map { n =>
logicalPlan.resolve(n.name, sqlContext.analyzer.resolver).get
queryExecution.analyzed.resolve(n.name, sqlContext.analyzer.resolver).get
}
}
/** Resolves a column name into a Catalyst [[NamedExpression]]. */
protected[sql] def resolve(colName: String): NamedExpression = {
logicalPlan.resolve(colName, sqlContext.analyzer.resolver).getOrElse(throw new RuntimeException(
s"""Cannot resolve column name "$colName" among (${schema.fieldNames.mkString(", ")})"""))
queryExecution.analyzed.resolve(colName, sqlContext.analyzer.resolver).getOrElse {
throw new RuntimeException(
s"""Cannot resolve column name "$colName" among (${schema.fieldNames.mkString(", ")})""")
}
}
/** Left here for compatibility reasons. */
......
......@@ -276,5 +276,9 @@ class DataFrameSuite extends QueryTest {
)
}
test("apply on query results (SPARK-5462)") {
val df = testData.sqlContext.sql("select key from testData")
checkAnswer(df("key"), testData.select('key).collect().toSeq)
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment