From ef89b82d831d1d35dfaa6387ff2077ea2f2073cc Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Sun, 1 Feb 2015 17:52:18 -0800
Subject: [PATCH] [Minor][SQL] Little refactor DataFrame related codes
Simplify some codes related to DataFrame.
* Calling `toAttributes` instead of a `map`.
* Original `createDataFrame` creates the `StructType` and its attributes in a redundant way. Refactored it to create `StructType` and call `toAttributes` on it directly.
Author: Liang-Chi Hsieh <viirya@gmail.com>
Closes #4298 from viirya/refactor_df and squashes the following commits:
1d61c64 [Liang-Chi Hsieh] Revert it.
f36efb5 [Liang-Chi Hsieh] Relax the constraint of toDataFrame.
2c9f370 [Liang-Chi Hsieh] Just refactor DataFrame codes.
---
.../scala/org/apache/spark/sql/catalyst/ScalaReflection.scala | 2 +-
sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 90646fd25b..e0db587efb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -98,7 +98,7 @@ trait ScalaReflection {
/** Returns a Sequence of attributes for the given case class type. */
def attributesFor[T: TypeTag]: Seq[Attribute] = schemaFor[T] match {
case Schema(s: StructType, _) =>
- s.fields.map(f => AttributeReference(f.name, f.dataType, f.nullable, f.metadata)())
+ s.toAttributes
}
/** Returns a catalyst DataType and its nullability for the given Scala Type using reflection. */
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index f87fde4ed8..84933dd944 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -168,8 +168,8 @@ class SQLContext(@transient val sparkContext: SparkContext)
*/
implicit def createDataFrame[A <: Product: TypeTag](rdd: RDD[A]): DataFrame = {
SparkPlan.currentContext.set(self)
- val attributeSeq = ScalaReflection.attributesFor[A]
- val schema = StructType.fromAttributes(attributeSeq)
+ val schema = ScalaReflection.schemaFor[A].dataType.asInstanceOf[StructType]
+ val attributeSeq = schema.toAttributes
val rowRDD = RDDConversions.productToRowRdd(rdd, schema)
new DataFrame(this, LogicalRDD(attributeSeq, rowRDD)(self))
}
--
GitLab