From f5da592fc63b8d3bc09d49c196d6c5d98cd2a013 Mon Sep 17 00:00:00 2001
From: hyukjinkwon <gurwls223@gmail.com>
Date: Wed, 27 Apr 2016 17:41:05 -0700
Subject: [PATCH] [SPARK-12143][SQL] Binary type support for Hive thrift server
## What changes were proposed in this pull request?
https://issues.apache.org/jira/browse/SPARK-12143
This PR adds the support for conversion between `SparkRow` in Spark and `RowSet` in Hive for `BinaryType` as `Array[Byte]` (JDBC)
## How was this patch tested?
Unittests in `HiveThriftBinaryServerSuite` (regression test)
Closes #10139
Author: hyukjinkwon <gurwls223@gmail.com>
Closes #12733 from HyukjinKwon/SPARK-12143.
---
.../SparkExecuteStatementOperation.scala | 6 ++++--
.../HiveThriftServer2Suites.scala | 19 +++++++++++++++++++
2 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index 18b78ab506..40dc81e02d 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -96,8 +96,10 @@ private[hive] class SparkExecuteStatementOperation(
case DateType =>
to += from.getAs[Date](ordinal)
case TimestampType =>
- to += from.getAs[Timestamp](ordinal)
- case BinaryType | _: ArrayType | _: StructType | _: MapType =>
+ to += from.getAs[Timestamp](ordinal)
+ case BinaryType =>
+ to += from.getAs[Array[Byte]](ordinal)
+ case _: ArrayType | _: StructType | _: MapType =>
val hiveString = HiveUtils.toHiveString((from.get(ordinal), dataTypes(ordinal)))
to += hiveString
}
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index f15f5b01e2..55a93ea06b 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -202,6 +202,25 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
}
}
+ test("SPARK-12143 regression: Binary type support") {
+ withJdbcStatement { statement =>
+ val queries = Seq(
+ "DROP TABLE IF EXISTS test_binary",
+ "CREATE TABLE test_binary(key INT, value STRING)",
+ s"LOAD DATA LOCAL INPATH '${TestData.smallKv}' OVERWRITE INTO TABLE test_binary")
+
+ queries.foreach(statement.execute)
+
+ val expected: Array[Byte] = "val_238".getBytes
+ assertResult(expected) {
+ val resultSet = statement.executeQuery(
+ "SELECT CAST(value as BINARY) FROM test_date LIMIT 1")
+ resultSet.next()
+ resultSet.getObject(1)
+ }
+ }
+ }
+
test("test multiple session") {
import org.apache.spark.sql.internal.SQLConf
var defaultV1: String = null
--
GitLab