From 5ab6e9f0c06b763b06a0e1396bdb08a823146c32 Mon Sep 17 00:00:00 2001
From: Daoyuan Wang <daoyuan.wang@intel.com>
Date: Wed, 25 Mar 2015 18:43:26 -0700
Subject: [PATCH] [SPARK-6202] [SQL] enable variable substitution on test
framework
Author: Daoyuan Wang <daoyuan.wang@intel.com>
Closes #4930 from adrian-wang/testvs and squashes the following commits:
2ce590f [Daoyuan Wang] add explicit function types
b1d68bf [Daoyuan Wang] only substitute for parseSql
9c4a950 [Daoyuan Wang] add a comment explaining
18fb481 [Daoyuan Wang] enable variable substitute on test framework
---
.../scala/org/apache/spark/sql/hive/test/TestHive.scala | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
index dc61e9d2e3..a3497eadd6 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -23,6 +23,7 @@ import java.util.{Set => JavaSet}
import org.apache.hadoop.hive.ql.exec.FunctionRegistry
import org.apache.hadoop.hive.ql.io.avro.{AvroContainerInputFormat, AvroContainerOutputFormat}
import org.apache.hadoop.hive.ql.metadata.Table
+import org.apache.hadoop.hive.ql.parse.VariableSubstitution
import org.apache.hadoop.hive.ql.processors._
import org.apache.hadoop.hive.serde2.RegexSerDe
import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
@@ -153,8 +154,13 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) {
val describedTable = "DESCRIBE (\\w+)".r
+ val vs = new VariableSubstitution()
+
+ // we should substitute variables in hql to pass the text to parseSql() as a parameter.
+ // Hive parser need substituted text. HiveContext.sql() does this but return a DataFrame,
+ // while we need a logicalPlan so we cannot reuse that.
protected[hive] class HiveQLQueryExecution(hql: String)
- extends this.QueryExecution(HiveQl.parseSql(hql)) {
+ extends this.QueryExecution(HiveQl.parseSql(vs.substitute(hiveconf, hql))) {
def hiveExec(): Seq[String] = runSqlHive(hql)
override def toString: String = hql + "\n" + super.toString
}
--
GitLab