From 198b0426e07f3d4b1fbbef21d39daa32a75da36c Mon Sep 17 00:00:00 2001
From: Liwei Lin <lwlin7@gmail.com>
Date: Sun, 24 Jul 2016 08:35:57 +0100
Subject: [PATCH] [SPARK-16515][SQL][FOLLOW-UP] Fix test `script` on OS
X/Windows...
The current `sed` in `test_script.sh` is missing a `$`, leading to the failure of `script` test on OS X:
```
== Results ==
!== Correct Answer - 2 == == Spark Answer - 2 ==
![x1_y1] [x1]
![x2_y2] [x2]
```
In addition, this `script` test would also fail on systems like Windows where we couldn't be able to invoke `bash` or `echo | sed`.
This patch
- fixes `sed` in `test_script.sh`
- adds command guards so that the `script` test would pass on systems like Windows
- Jenkins
- Manually verified tests pass on OS X
Author: Liwei Lin <lwlin7@gmail.com>
Closes #14280 from lw-lin/osx-sed.
(cherry picked from commit d6795c7a254b83d4ae4785f3add74981e5273c91)
Signed-off-by: Sean Owen <sowen@cloudera.com>
---
sql/hive/src/test/resources/test_script.sh | 2 +-
.../sql/hive/execution/SQLQuerySuite.scala | 26 +++++++++++++------
2 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/sql/hive/src/test/resources/test_script.sh b/sql/hive/src/test/resources/test_script.sh
index ab998c41b2..eb0c50e982 100755
--- a/sql/hive/src/test/resources/test_script.sh
+++ b/sql/hive/src/test/resources/test_script.sh
@@ -19,5 +19,5 @@
while read line
do
- echo "$line" | sed 's/\t/_/'
+ echo "$line" | sed $'s/\t/_/'
done < /dev/stdin
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 9a36fd6c99..c4d9e0aee9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -19,6 +19,9 @@ package org.apache.spark.sql.hive.execution
import java.sql.{Date, Timestamp}
+import scala.sys.process.Process
+import scala.util.Try
+
import org.apache.hadoop.fs.Path
import org.apache.spark.sql._
@@ -64,14 +67,17 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
import spark.implicits._
test("script") {
- val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
- df.createOrReplaceTempView("script_table")
- val query1 = sql(
- """
- |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
- |REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS
- |(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
- checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
+ if (testCommandAvailable("bash") && testCommandAvailable("echo | sed")) {
+ val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
+ df.createOrReplaceTempView("script_table")
+ val query1 = sql(
+ """
+ |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
+ |REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS
+ |(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
+ checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
+ }
+ // else skip this test
}
test("UDTF") {
@@ -1677,4 +1683,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
)
}
}
+
+ def testCommandAvailable(command: String): Boolean = {
+ Try(Process(command) !!).isSuccess
+ }
}
--
GitLab