Skip to content
Snippets Groups Projects
Commit 7e15044d authored by Yucai Yu's avatar Yucai Yu Committed by Sean Owen
Browse files

[SPARK-12582][TEST] IndexShuffleBlockResolverSuite fails in windows

[SPARK-12582][Test] IndexShuffleBlockResolverSuite fails in windows

* IndexShuffleBlockResolverSuite fails in windows due to file is not closed.
* mv IndexShuffleBlockResolverSuite.scala from "test/java" to "test/scala".

https://issues.apache.org/jira/browse/SPARK-12582

Author: Yucai Yu <yucai.yu@intel.com>

Closes #10526 from yucai/master.
parent 9f0995bb
No related branches found
No related tags found
No related merge requests found
...@@ -19,18 +19,18 @@ package org.apache.spark.shuffle.sort ...@@ -19,18 +19,18 @@ package org.apache.spark.shuffle.sort
import java.io.{File, FileInputStream, FileOutputStream} import java.io.{File, FileInputStream, FileOutputStream}
import org.mockito.{Mock, MockitoAnnotations}
import org.mockito.Answers.RETURNS_SMART_NULLS import org.mockito.Answers.RETURNS_SMART_NULLS
import org.mockito.Matchers._ import org.mockito.Matchers._
import org.mockito.Mockito._ import org.mockito.Mockito._
import org.mockito.invocation.InvocationOnMock import org.mockito.invocation.InvocationOnMock
import org.mockito.stubbing.Answer import org.mockito.stubbing.Answer
import org.mockito.{Mock, MockitoAnnotations}
import org.scalatest.BeforeAndAfterEach import org.scalatest.BeforeAndAfterEach
import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.shuffle.IndexShuffleBlockResolver import org.apache.spark.shuffle.IndexShuffleBlockResolver
import org.apache.spark.storage._ import org.apache.spark.storage._
import org.apache.spark.util.Utils import org.apache.spark.util.Utils
import org.apache.spark.{SparkConf, SparkFunSuite}
class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEach { class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEach {
...@@ -64,12 +64,15 @@ class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEa ...@@ -64,12 +64,15 @@ class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEa
} }
test("commit shuffle files multiple times") { test("commit shuffle files multiple times") {
val lengths = Array[Long](10, 0, 20)
val resolver = new IndexShuffleBlockResolver(conf, blockManager) val resolver = new IndexShuffleBlockResolver(conf, blockManager)
val lengths = Array[Long](10, 0, 20)
val dataTmp = File.createTempFile("shuffle", null, tempDir) val dataTmp = File.createTempFile("shuffle", null, tempDir)
val out = new FileOutputStream(dataTmp) val out = new FileOutputStream(dataTmp)
out.write(new Array[Byte](30)) Utils.tryWithSafeFinally {
out.close() out.write(new Array[Byte](30))
} {
out.close()
}
resolver.writeIndexFileAndCommit(1, 2, lengths, dataTmp) resolver.writeIndexFileAndCommit(1, 2, lengths, dataTmp)
val dataFile = resolver.getDataFile(1, 2) val dataFile = resolver.getDataFile(1, 2)
...@@ -77,12 +80,15 @@ class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEa ...@@ -77,12 +80,15 @@ class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEa
assert(dataFile.length() === 30) assert(dataFile.length() === 30)
assert(!dataTmp.exists()) assert(!dataTmp.exists())
val lengths2 = new Array[Long](3)
val dataTmp2 = File.createTempFile("shuffle", null, tempDir) val dataTmp2 = File.createTempFile("shuffle", null, tempDir)
val out2 = new FileOutputStream(dataTmp2) val out2 = new FileOutputStream(dataTmp2)
val lengths2 = new Array[Long](3) Utils.tryWithSafeFinally {
out2.write(Array[Byte](1)) out2.write(Array[Byte](1))
out2.write(new Array[Byte](29)) out2.write(new Array[Byte](29))
out2.close() } {
out2.close()
}
resolver.writeIndexFileAndCommit(1, 2, lengths2, dataTmp2) resolver.writeIndexFileAndCommit(1, 2, lengths2, dataTmp2)
assert(lengths2.toSeq === lengths.toSeq) assert(lengths2.toSeq === lengths.toSeq)
assert(dataFile.exists()) assert(dataFile.exists())
...@@ -90,20 +96,27 @@ class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEa ...@@ -90,20 +96,27 @@ class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEa
assert(!dataTmp2.exists()) assert(!dataTmp2.exists())
// The dataFile should be the previous one // The dataFile should be the previous one
val in = new FileInputStream(dataFile)
val firstByte = new Array[Byte](1) val firstByte = new Array[Byte](1)
in.read(firstByte) val in = new FileInputStream(dataFile)
Utils.tryWithSafeFinally {
in.read(firstByte)
} {
in.close()
}
assert(firstByte(0) === 0) assert(firstByte(0) === 0)
// remove data file // remove data file
dataFile.delete() dataFile.delete()
val lengths3 = Array[Long](10, 10, 15)
val dataTmp3 = File.createTempFile("shuffle", null, tempDir) val dataTmp3 = File.createTempFile("shuffle", null, tempDir)
val out3 = new FileOutputStream(dataTmp3) val out3 = new FileOutputStream(dataTmp3)
val lengths3 = Array[Long](10, 10, 15) Utils.tryWithSafeFinally {
out3.write(Array[Byte](2)) out3.write(Array[Byte](2))
out3.write(new Array[Byte](34)) out3.write(new Array[Byte](34))
out3.close() } {
out3.close()
}
resolver.writeIndexFileAndCommit(1, 2, lengths3, dataTmp3) resolver.writeIndexFileAndCommit(1, 2, lengths3, dataTmp3)
assert(lengths3.toSeq != lengths.toSeq) assert(lengths3.toSeq != lengths.toSeq)
assert(dataFile.exists()) assert(dataFile.exists())
...@@ -111,9 +124,13 @@ class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEa ...@@ -111,9 +124,13 @@ class IndexShuffleBlockResolverSuite extends SparkFunSuite with BeforeAndAfterEa
assert(!dataTmp2.exists()) assert(!dataTmp2.exists())
// The dataFile should be the previous one // The dataFile should be the previous one
val in2 = new FileInputStream(dataFile)
val firstByte2 = new Array[Byte](1) val firstByte2 = new Array[Byte](1)
in2.read(firstByte2) val in2 = new FileInputStream(dataFile)
Utils.tryWithSafeFinally {
in2.read(firstByte2)
} {
in2.close()
}
assert(firstByte2(0) === 2) assert(firstByte2(0) === 2)
} }
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment