Skip to content
Snippets Groups Projects
Commit 7453f311 authored by Aaron Davidson's avatar Aaron Davidson
Browse files

Address minor comments

parent 84991a1b
No related branches found
No related tags found
No related merge requests found
......@@ -34,7 +34,8 @@ import org.apache.spark.util.Utils
*
* @param rootDirs The directories to use for storing block files. Data will be hashed among these.
*/
private[spark] class DiskBlockManager(shuffleManager: ShuffleBlockManager, rootDirs: String) extends PathResolver with Logging {
private[spark] class DiskBlockManager(shuffleManager: ShuffleBlockManager, rootDirs: String)
extends PathResolver with Logging {
private val MAX_DIR_CREATION_ATTEMPTS: Int = 10
private val subDirsPerLocalDir = System.getProperty("spark.diskStore.subDirectories", "64").toInt
......
......@@ -27,7 +27,8 @@ import scala.collection.mutable
import org.apache.spark.Logging
import org.apache.spark.serializer.Serializer
import org.apache.spark.util.{MetadataCleanerType, MetadataCleaner, AGodDamnPrimitiveVector, TimeStampedHashMap}
import org.apache.spark.util.{MetadataCleanerType, MetadataCleaner, TimeStampedHashMap}
import org.apache.spark.util.collection.PrimitiveVector
private[spark]
class ShuffleWriterGroup(
......@@ -203,7 +204,7 @@ class ShuffleBlockManager(blockManager: BlockManager) extends Logging {
*/
private[spark]
class ShuffleFileGroup(val shuffleId: Int, val fileId: Int, val files: Array[ShuffleFile]) {
private val mapIds = new AGodDamnPrimitiveVector[Int]()
private val mapIds = new PrimitiveVector[Int]()
files.foreach(_.setShuffleFileGroup(this))
......@@ -238,7 +239,7 @@ class ShuffleFile(val file: File) {
* Consecutive offsets of blocks into the file, ordered by position in the file.
* This ordering allows us to compute block lengths by examining the following block offset.
*/
val blockOffsets = new AGodDamnPrimitiveVector[Long]()
val blockOffsets = new PrimitiveVector[Long]()
/** Back pointer to whichever ShuffleFileGroup this file is a part of. */
private var shuffleFileGroup : ShuffleFileGroup = _
......
......@@ -15,12 +15,11 @@
* limitations under the License.
*/
package org.apache.spark.util
package org.apache.spark.util.collection
/** Provides a simple, non-threadsafe, array-backed vector that can store primitives. */
class AGodDamnPrimitiveVector[@specialized(Long, Int, Double) V: ClassManifest]
(initialSize: Int = 64)
{
private[spark]
class PrimitiveVector[@specialized(Long, Int, Double) V: ClassManifest](initialSize: Int = 64) {
private var numElements = 0
private var array = new Array[V](initialSize)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment