Skip to content
Snippets Groups Projects
Commit 198b0426 authored by Liwei Lin's avatar Liwei Lin Committed by Sean Owen
Browse files

[SPARK-16515][SQL][FOLLOW-UP] Fix test `script` on OS X/Windows...


The current `sed` in `test_script.sh` is missing a `$`, leading to the failure of `script` test on OS X:
```
== Results ==
!== Correct Answer - 2 ==   == Spark Answer - 2 ==
![x1_y1]                    [x1]
![x2_y2]                    [x2]
```

In addition, this `script` test would also fail on systems like Windows where we couldn't be able to invoke `bash` or `echo | sed`.

This patch
- fixes `sed` in `test_script.sh`
- adds command guards so that the `script` test would pass on systems like Windows

- Jenkins
- Manually verified tests pass on OS X

Author: Liwei Lin <lwlin7@gmail.com>

Closes #14280 from lw-lin/osx-sed.

(cherry picked from commit d6795c7a)
Signed-off-by: default avatarSean Owen <sowen@cloudera.com>
parent 31c3bcb4
No related branches found
No related tags found
No related merge requests found
...@@ -19,5 +19,5 @@ ...@@ -19,5 +19,5 @@
while read line while read line
do do
echo "$line" | sed 's/\t/_/' echo "$line" | sed $'s/\t/_/'
done < /dev/stdin done < /dev/stdin
...@@ -19,6 +19,9 @@ package org.apache.spark.sql.hive.execution ...@@ -19,6 +19,9 @@ package org.apache.spark.sql.hive.execution
import java.sql.{Date, Timestamp} import java.sql.{Date, Timestamp}
import scala.sys.process.Process
import scala.util.Try
import org.apache.hadoop.fs.Path import org.apache.hadoop.fs.Path
import org.apache.spark.sql._ import org.apache.spark.sql._
...@@ -64,14 +67,17 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { ...@@ -64,14 +67,17 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
import spark.implicits._ import spark.implicits._
test("script") { test("script") {
val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3") if (testCommandAvailable("bash") && testCommandAvailable("echo | sed")) {
df.createOrReplaceTempView("script_table") val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
val query1 = sql( df.createOrReplaceTempView("script_table")
""" val query1 = sql(
|SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table """
|REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
|(col1 STRING, col2 STRING)) script_test_table""".stripMargin) |REDUCE c1, c2, c3 USING 'bash src/test/resources/test_script.sh' AS
checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil) |(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
}
// else skip this test
} }
test("UDTF") { test("UDTF") {
...@@ -1677,4 +1683,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { ...@@ -1677,4 +1683,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
) )
} }
} }
def testCommandAvailable(command: String): Boolean = {
Try(Process(command) !!).isSuccess
}
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment