From 3ae0cda83c5106136e90d59c20e61db345a5085f Mon Sep 17 00:00:00 2001
From: wangfei <wangfei1@huawei.com>
Date: Tue, 2 Dec 2014 14:30:44 -0800
Subject: [PATCH] [SPARK-4695][SQL] Get result using executeCollect
Using ```executeCollect``` to collect the result, because executeCollect is a custom implementation of collect in spark sql which better than rdd's collect
Author: wangfei <wangfei1@huawei.com>
Closes #3547 from scwf/executeCollect and squashes the following commits:
a5ab68e [wangfei] Revert "adding debug info"
a60d680 [wangfei] fix test failure
0db7ce8 [wangfei] adding debug info
184c594 [wangfei] using executeCollect instead collect
---
.../main/scala/org/apache/spark/sql/hive/HiveContext.scala | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index 304b9a73ee..34fc21e61f 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -377,7 +377,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) {
command.executeCollect().map(_.head.toString)
case other =>
- val result: Seq[Seq[Any]] = toRdd.map(_.copy()).collect().toSeq
+ val result: Seq[Seq[Any]] = other.executeCollect().toSeq
// We need the types so we can output struct field names
val types = analyzed.output.map(_.dataType)
// Reformat to match hive tab delimited output.
@@ -416,6 +416,8 @@ object HiveContext {
case (bin: Array[Byte], BinaryType) => new String(bin, "UTF-8")
case (decimal: Decimal, DecimalType()) => // Hive strips trailing zeros so use its toString
HiveShim.createDecimal(decimal.toBigDecimal.underlying()).toString
+ case (decimal: BigDecimal, DecimalType()) =>
+ HiveShim.createDecimal(decimal.underlying()).toString
case (other, tpe) if primitiveTypes contains tpe => other.toString
}
--
GitLab