From bd86cb1738800a0aa4c88b9afdba2f97ac6cbf25 Mon Sep 17 00:00:00 2001
From: RongGu <gurongwalker@gmail.com>
Date: Sun, 9 Nov 2014 23:48:15 -0800
Subject: [PATCH] [SPARK-2703][Core]Make Tachyon related unit tests execute
without deploying a Tachyon system locally.
Make Tachyon related unit tests execute without deploying a Tachyon system locally.
Author: RongGu <gurongwalker@gmail.com>
Closes #3030 from RongGu/SPARK-2703 and squashes the following commits:
ad08827 [RongGu] Make Tachyon related unit tests execute without deploying a Tachyon system locally
---
core/pom.xml | 7 +++++++
.../org/apache/spark/storage/BlockManagerSuite.scala | 11 +++++++++--
project/SparkBuild.scala | 2 ++
3 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 41296e0eca..92e9f1fc46 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -204,6 +204,13 @@
<artifactId>derby</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.tachyonproject</groupId>
+ <artifactId>tachyon</artifactId>
+ <version>0.5.0</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.tachyonproject</groupId>
<artifactId>tachyon-client</artifactId>
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
index 9529502bc8..86503c9a02 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
@@ -36,6 +36,7 @@ import org.mockito.Mockito.{mock, when}
import org.scalatest.{BeforeAndAfter, FunSuite, Matchers, PrivateMethodTester}
import org.scalatest.concurrent.Eventually._
import org.scalatest.concurrent.Timeouts._
+import tachyon.master.LocalTachyonCluster
import org.apache.spark.{MapOutputTrackerMaster, SparkConf, SparkContext, SecurityManager}
import org.apache.spark.executor.DataReadMethod
@@ -536,9 +537,14 @@ class BlockManagerSuite extends FunSuite with Matchers with BeforeAndAfter
}
test("tachyon storage") {
- // TODO Make the spark.test.tachyon.enable true after using tachyon 0.5.0 testing jar.
- val tachyonUnitTestEnabled = conf.getBoolean("spark.test.tachyon.enable", false)
+ val tachyonUnitTestEnabled = conf.getBoolean("spark.test.tachyon.enable", true)
if (tachyonUnitTestEnabled) {
+ val tachyonCluster = new LocalTachyonCluster(30000000)
+ tachyonCluster.start()
+ val tachyonURL = tachyon.Constants.HEADER +
+ tachyonCluster.getMasterHostname() + ":" + tachyonCluster.getMasterPort()
+ conf.set("spark.tachyonStore.url", tachyonURL)
+ conf.set("spark.tachyonStore.folderName", "app-test")
store = makeBlockManager(1200)
val a1 = new Array[Byte](400)
val a2 = new Array[Byte](400)
@@ -549,6 +555,7 @@ class BlockManagerSuite extends FunSuite with Matchers with BeforeAndAfter
assert(store.getSingle("a3").isDefined, "a3 was in store")
assert(store.getSingle("a2").isDefined, "a2 was in store")
assert(store.getSingle("a1").isDefined, "a1 was in store")
+ tachyonCluster.stop()
} else {
info("tachyon storage test disabled.")
}
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 657e4b4432..351e57a4b5 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -360,6 +360,8 @@ object TestSettings {
testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
// Enable Junit testing.
libraryDependencies += "com.novocode" % "junit-interface" % "0.9" % "test",
+ // Enable Tachyon local testing.
+ libraryDependencies += "org.tachyonproject" % "tachyon" % "0.5.0" % "test" classifier "tests",
// Only allow one test at a time, even across projects, since they run in the same JVM
parallelExecution in Test := false,
concurrentRestrictions in Global += Tags.limit(Tags.Test, 1),
--
GitLab