diff --git a/core/src/main/scala/org/apache/spark/Accumulators.scala b/core/src/main/scala/org/apache/spark/Accumulators.scala
index bcf832467f00b3a899c6d9d584ff195086bc6501..330df1d59a9b1aaed83793c27f7f6a1392e4c338 100644
--- a/core/src/main/scala/org/apache/spark/Accumulators.scala
+++ b/core/src/main/scala/org/apache/spark/Accumulators.scala
@@ -18,8 +18,6 @@
package org.apache.spark
import java.io.{ObjectInputStream, Serializable}
-import java.util.concurrent.atomic.AtomicLong
-import java.lang.ThreadLocal
import scala.collection.generic.Growable
import scala.collection.mutable.Map
@@ -109,7 +107,7 @@ class Accumulable[R, T] (
* The typical use of this method is to directly mutate the local value, eg., to add
* an element to a Set.
*/
- def localValue = value_
+ def localValue: R = value_
/**
* Set the accumulator's value; only allowed on master.
@@ -137,7 +135,7 @@ class Accumulable[R, T] (
Accumulators.register(this, false)
}
- override def toString = if (value_ == null) "null" else value_.toString
+ override def toString: String = if (value_ == null) "null" else value_.toString
}
/**
@@ -257,22 +255,22 @@ object AccumulatorParam {
implicit object DoubleAccumulatorParam extends AccumulatorParam[Double] {
def addInPlace(t1: Double, t2: Double): Double = t1 + t2
- def zero(initialValue: Double) = 0.0
+ def zero(initialValue: Double): Double = 0.0
}
implicit object IntAccumulatorParam extends AccumulatorParam[Int] {
def addInPlace(t1: Int, t2: Int): Int = t1 + t2
- def zero(initialValue: Int) = 0
+ def zero(initialValue: Int): Int = 0
}
implicit object LongAccumulatorParam extends AccumulatorParam[Long] {
- def addInPlace(t1: Long, t2: Long) = t1 + t2
- def zero(initialValue: Long) = 0L
+ def addInPlace(t1: Long, t2: Long): Long = t1 + t2
+ def zero(initialValue: Long): Long = 0L
}
implicit object FloatAccumulatorParam extends AccumulatorParam[Float] {
- def addInPlace(t1: Float, t2: Float) = t1 + t2
- def zero(initialValue: Float) = 0f
+ def addInPlace(t1: Float, t2: Float): Float = t1 + t2
+ def zero(initialValue: Float): Float = 0f
}
// TODO: Add AccumulatorParams for other types, e.g. lists and strings
@@ -351,6 +349,7 @@ private[spark] object Accumulators extends Logging {
}
}
- def stringifyPartialValue(partialValue: Any) = "%s".format(partialValue)
- def stringifyValue(value: Any) = "%s".format(value)
+ def stringifyPartialValue(partialValue: Any): String = "%s".format(partialValue)
+
+ def stringifyValue(value: Any): String = "%s".format(value)
}
diff --git a/core/src/main/scala/org/apache/spark/Dependency.scala b/core/src/main/scala/org/apache/spark/Dependency.scala
index 9a7cd4523e5abd0309977efa9930cf2ba2983801..fc8cdde9348ee2550b909a6fe7f8ed4013d62808 100644
--- a/core/src/main/scala/org/apache/spark/Dependency.scala
+++ b/core/src/main/scala/org/apache/spark/Dependency.scala
@@ -74,7 +74,7 @@ class ShuffleDependency[K, V, C](
val mapSideCombine: Boolean = false)
extends Dependency[Product2[K, V]] {
- override def rdd = _rdd.asInstanceOf[RDD[Product2[K, V]]]
+ override def rdd: RDD[Product2[K, V]] = _rdd.asInstanceOf[RDD[Product2[K, V]]]
val shuffleId: Int = _rdd.context.newShuffleId()
@@ -91,7 +91,7 @@ class ShuffleDependency[K, V, C](
*/
@DeveloperApi
class OneToOneDependency[T](rdd: RDD[T]) extends NarrowDependency[T](rdd) {
- override def getParents(partitionId: Int) = List(partitionId)
+ override def getParents(partitionId: Int): List[Int] = List(partitionId)
}
@@ -107,7 +107,7 @@ class OneToOneDependency[T](rdd: RDD[T]) extends NarrowDependency[T](rdd) {
class RangeDependency[T](rdd: RDD[T], inStart: Int, outStart: Int, length: Int)
extends NarrowDependency[T](rdd) {
- override def getParents(partitionId: Int) = {
+ override def getParents(partitionId: Int): List[Int] = {
if (partitionId >= outStart && partitionId < outStart + length) {
List(partitionId - outStart + inStart)
} else {
diff --git a/core/src/main/scala/org/apache/spark/FutureAction.scala b/core/src/main/scala/org/apache/spark/FutureAction.scala
index e97a7375a267b6356925611b1520579f6b111d39..91f9ef8ce71859c6950eb39a54084e8606aef843 100644
--- a/core/src/main/scala/org/apache/spark/FutureAction.scala
+++ b/core/src/main/scala/org/apache/spark/FutureAction.scala
@@ -168,7 +168,7 @@ class SimpleFutureAction[T] private[spark](jobWaiter: JobWaiter[_], resultFunc:
}
}
- def jobIds = Seq(jobWaiter.jobId)
+ def jobIds: Seq[Int] = Seq(jobWaiter.jobId)
}
@@ -276,7 +276,7 @@ class ComplexFutureAction[T] extends FutureAction[T] {
override def value: Option[Try[T]] = p.future.value
- def jobIds = jobs
+ def jobIds: Seq[Int] = jobs
}
diff --git a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
index 69178da1a77730e186105af079532b137eb1d34f..715f292f034699e49afa08b1559baaac39147233 100644
--- a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
+++ b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
@@ -65,7 +65,7 @@ private[spark] class HeartbeatReceiver(sc: SparkContext, scheduler: TaskSchedule
super.preStart()
}
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case Heartbeat(executorId, taskMetrics, blockManagerId) =>
val unknownExecutor = !scheduler.executorHeartbeatReceived(
executorId, taskMetrics, blockManagerId)
diff --git a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
index 6e4edc7c80d7ae788f1af622e68ce27b51034321..c9426c5de23a24adbfa58572d3930a20bc543cbf 100644
--- a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
+++ b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
@@ -43,7 +43,7 @@ private[spark] class MapOutputTrackerMasterActor(tracker: MapOutputTrackerMaster
extends Actor with ActorLogReceive with Logging {
val maxAkkaFrameSize = AkkaUtils.maxFrameSizeBytes(conf)
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case GetMapOutputStatuses(shuffleId: Int) =>
val hostPort = sender.path.address.hostPort
logInfo("Asked to send map output locations for shuffle " + shuffleId + " to " + hostPort)
diff --git a/core/src/main/scala/org/apache/spark/Partitioner.scala b/core/src/main/scala/org/apache/spark/Partitioner.scala
index e53a78ead2c0ed811fe5a6899d7156c67fe586be..b8d244408bc5b8df3abd405ed84538caf8ad2e8c 100644
--- a/core/src/main/scala/org/apache/spark/Partitioner.scala
+++ b/core/src/main/scala/org/apache/spark/Partitioner.scala
@@ -76,7 +76,7 @@ object Partitioner {
* produce an unexpected or incorrect result.
*/
class HashPartitioner(partitions: Int) extends Partitioner {
- def numPartitions = partitions
+ def numPartitions: Int = partitions
def getPartition(key: Any): Int = key match {
case null => 0
@@ -154,7 +154,7 @@ class RangePartitioner[K : Ordering : ClassTag, V](
}
}
- def numPartitions = rangeBounds.length + 1
+ def numPartitions: Int = rangeBounds.length + 1
private var binarySearch: ((Array[K], K) => Int) = CollectionsUtils.makeBinarySearch[K]
diff --git a/core/src/main/scala/org/apache/spark/SerializableWritable.scala b/core/src/main/scala/org/apache/spark/SerializableWritable.scala
index 55cb25946c2addb35c81e286ee8713fe86590034..cb2cae185256a747178bc3ac8a8608c503e40c50 100644
--- a/core/src/main/scala/org/apache/spark/SerializableWritable.scala
+++ b/core/src/main/scala/org/apache/spark/SerializableWritable.scala
@@ -28,8 +28,10 @@ import org.apache.spark.util.Utils
@DeveloperApi
class SerializableWritable[T <: Writable](@transient var t: T) extends Serializable {
- def value = t
- override def toString = t.toString
+
+ def value: T = t
+
+ override def toString: String = t.toString
private def writeObject(out: ObjectOutputStream): Unit = Utils.tryOrIOException {
out.defaultWriteObject()
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index 2ca19f53d2f07846300dcbc6eaf03e8076c35ad6..0c123c96b8d7bd9a19e892a56218f036ad391792 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -133,7 +133,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging {
}
/** Set multiple parameters together */
- def setAll(settings: Traversable[(String, String)]) = {
+ def setAll(settings: Traversable[(String, String)]): SparkConf = {
this.settings.putAll(settings.toMap.asJava)
this
}
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 228ff715fe7cb165f6db91a211d5d103c8b85ab1..a70be16f77eeb3ff80d4db89241c27f904cde4c4 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -986,7 +986,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
union(Seq(first) ++ rest)
/** Get an RDD that has no partitions or elements. */
- def emptyRDD[T: ClassTag] = new EmptyRDD[T](this)
+ def emptyRDD[T: ClassTag]: EmptyRDD[T] = new EmptyRDD[T](this)
// Methods for creating shared variables
@@ -994,7 +994,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* Create an [[org.apache.spark.Accumulator]] variable of a given type, which tasks can "add"
* values to using the `+=` method. Only the driver can access the accumulator's `value`.
*/
- def accumulator[T](initialValue: T)(implicit param: AccumulatorParam[T]) =
+ def accumulator[T](initialValue: T)(implicit param: AccumulatorParam[T]): Accumulator[T] =
{
val acc = new Accumulator(initialValue, param)
cleaner.foreach(_.registerAccumulatorForCleanup(acc))
@@ -1006,7 +1006,8 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* in the Spark UI. Tasks can "add" values to the accumulator using the `+=` method. Only the
* driver can access the accumulator's `value`.
*/
- def accumulator[T](initialValue: T, name: String)(implicit param: AccumulatorParam[T]) = {
+ def accumulator[T](initialValue: T, name: String)(implicit param: AccumulatorParam[T])
+ : Accumulator[T] = {
val acc = new Accumulator(initialValue, param, Some(name))
cleaner.foreach(_.registerAccumulatorForCleanup(acc))
acc
@@ -1018,7 +1019,8 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* @tparam R accumulator result type
* @tparam T type that can be added to the accumulator
*/
- def accumulable[R, T](initialValue: R)(implicit param: AccumulableParam[R, T]) = {
+ def accumulable[R, T](initialValue: R)(implicit param: AccumulableParam[R, T])
+ : Accumulable[R, T] = {
val acc = new Accumulable(initialValue, param)
cleaner.foreach(_.registerAccumulatorForCleanup(acc))
acc
@@ -1031,7 +1033,8 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
* @tparam R accumulator result type
* @tparam T type that can be added to the accumulator
*/
- def accumulable[R, T](initialValue: R, name: String)(implicit param: AccumulableParam[R, T]) = {
+ def accumulable[R, T](initialValue: R, name: String)(implicit param: AccumulableParam[R, T])
+ : Accumulable[R, T] = {
val acc = new Accumulable(initialValue, param, Some(name))
cleaner.foreach(_.registerAccumulatorForCleanup(acc))
acc
@@ -1209,7 +1212,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
override def killExecutor(executorId: String): Boolean = super.killExecutor(executorId)
/** The version of Spark on which this application is running. */
- def version = SPARK_VERSION
+ def version: String = SPARK_VERSION
/**
* Return a map from the slave to the max memory available for caching and the remaining
@@ -1659,7 +1662,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
}
}
- def getCheckpointDir = checkpointDir
+ def getCheckpointDir: Option[String] = checkpointDir
/** Default level of parallelism to use when not given by user (e.g. parallelize and makeRDD). */
def defaultParallelism: Int = {
@@ -1900,28 +1903,28 @@ object SparkContext extends Logging {
"backward compatibility.", "1.3.0")
object DoubleAccumulatorParam extends AccumulatorParam[Double] {
def addInPlace(t1: Double, t2: Double): Double = t1 + t2
- def zero(initialValue: Double) = 0.0
+ def zero(initialValue: Double): Double = 0.0
}
@deprecated("Replaced by implicit objects in AccumulatorParam. This is kept here only for " +
"backward compatibility.", "1.3.0")
object IntAccumulatorParam extends AccumulatorParam[Int] {
def addInPlace(t1: Int, t2: Int): Int = t1 + t2
- def zero(initialValue: Int) = 0
+ def zero(initialValue: Int): Int = 0
}
@deprecated("Replaced by implicit objects in AccumulatorParam. This is kept here only for " +
"backward compatibility.", "1.3.0")
object LongAccumulatorParam extends AccumulatorParam[Long] {
- def addInPlace(t1: Long, t2: Long) = t1 + t2
- def zero(initialValue: Long) = 0L
+ def addInPlace(t1: Long, t2: Long): Long = t1 + t2
+ def zero(initialValue: Long): Long = 0L
}
@deprecated("Replaced by implicit objects in AccumulatorParam. This is kept here only for " +
"backward compatibility.", "1.3.0")
object FloatAccumulatorParam extends AccumulatorParam[Float] {
- def addInPlace(t1: Float, t2: Float) = t1 + t2
- def zero(initialValue: Float) = 0f
+ def addInPlace(t1: Float, t2: Float): Float = t1 + t2
+ def zero(initialValue: Float): Float = 0f
}
// The following deprecated functions have already been moved to `object RDD` to
@@ -1931,18 +1934,18 @@ object SparkContext extends Logging {
@deprecated("Replaced by implicit functions in the RDD companion object. This is " +
"kept here only for backward compatibility.", "1.3.0")
def rddToPairRDDFunctions[K, V](rdd: RDD[(K, V)])
- (implicit kt: ClassTag[K], vt: ClassTag[V], ord: Ordering[K] = null) = {
+ (implicit kt: ClassTag[K], vt: ClassTag[V], ord: Ordering[K] = null): PairRDDFunctions[K, V] =
RDD.rddToPairRDDFunctions(rdd)
- }
@deprecated("Replaced by implicit functions in the RDD companion object. This is " +
"kept here only for backward compatibility.", "1.3.0")
- def rddToAsyncRDDActions[T: ClassTag](rdd: RDD[T]) = RDD.rddToAsyncRDDActions(rdd)
+ def rddToAsyncRDDActions[T: ClassTag](rdd: RDD[T]): AsyncRDDActions[T] =
+ RDD.rddToAsyncRDDActions(rdd)
@deprecated("Replaced by implicit functions in the RDD companion object. This is " +
"kept here only for backward compatibility.", "1.3.0")
def rddToSequenceFileRDDFunctions[K <% Writable: ClassTag, V <% Writable: ClassTag](
- rdd: RDD[(K, V)]) = {
+ rdd: RDD[(K, V)]): SequenceFileRDDFunctions[K, V] = {
val kf = implicitly[K => Writable]
val vf = implicitly[V => Writable]
// Set the Writable class to null and `SequenceFileRDDFunctions` will use Reflection to get it
@@ -1954,16 +1957,17 @@ object SparkContext extends Logging {
@deprecated("Replaced by implicit functions in the RDD companion object. This is " +
"kept here only for backward compatibility.", "1.3.0")
def rddToOrderedRDDFunctions[K : Ordering : ClassTag, V: ClassTag](
- rdd: RDD[(K, V)]) =
+ rdd: RDD[(K, V)]): OrderedRDDFunctions[K, V, (K, V)] =
RDD.rddToOrderedRDDFunctions(rdd)
@deprecated("Replaced by implicit functions in the RDD companion object. This is " +
"kept here only for backward compatibility.", "1.3.0")
- def doubleRDDToDoubleRDDFunctions(rdd: RDD[Double]) = RDD.doubleRDDToDoubleRDDFunctions(rdd)
+ def doubleRDDToDoubleRDDFunctions(rdd: RDD[Double]): DoubleRDDFunctions =
+ RDD.doubleRDDToDoubleRDDFunctions(rdd)
@deprecated("Replaced by implicit functions in the RDD companion object. This is " +
"kept here only for backward compatibility.", "1.3.0")
- def numericRDDToDoubleRDDFunctions[T](rdd: RDD[T])(implicit num: Numeric[T]) =
+ def numericRDDToDoubleRDDFunctions[T](rdd: RDD[T])(implicit num: Numeric[T]): DoubleRDDFunctions =
RDD.numericRDDToDoubleRDDFunctions(rdd)
// The following deprecated functions have already been moved to `object WritableFactory` to
@@ -2134,7 +2138,7 @@ object SparkContext extends Logging {
(backend, scheduler)
case LOCAL_N_REGEX(threads) =>
- def localCpuCount = Runtime.getRuntime.availableProcessors()
+ def localCpuCount: Int = Runtime.getRuntime.availableProcessors()
// local[*] estimates the number of cores on the machine; local[N] uses exactly N threads.
val threadCount = if (threads == "*") localCpuCount else threads.toInt
if (threadCount <= 0) {
@@ -2146,7 +2150,7 @@ object SparkContext extends Logging {
(backend, scheduler)
case LOCAL_N_FAILURES_REGEX(threads, maxFailures) =>
- def localCpuCount = Runtime.getRuntime.availableProcessors()
+ def localCpuCount: Int = Runtime.getRuntime.availableProcessors()
// local[*, M] means the number of cores on the computer with M failures
// local[N, M] means exactly N threads with M failures
val threadCount = if (threads == "*") localCpuCount else threads.toInt
diff --git a/core/src/main/scala/org/apache/spark/TaskState.scala b/core/src/main/scala/org/apache/spark/TaskState.scala
index c415fe99b105ecdab89a44a9a231e24433fb6aa0..fe19f07e32d1b701271285f406884a4c1e54e88c 100644
--- a/core/src/main/scala/org/apache/spark/TaskState.scala
+++ b/core/src/main/scala/org/apache/spark/TaskState.scala
@@ -27,9 +27,9 @@ private[spark] object TaskState extends Enumeration {
type TaskState = Value
- def isFailed(state: TaskState) = (LOST == state) || (FAILED == state)
+ def isFailed(state: TaskState): Boolean = (LOST == state) || (FAILED == state)
- def isFinished(state: TaskState) = FINISHED_STATES.contains(state)
+ def isFinished(state: TaskState): Boolean = FINISHED_STATES.contains(state)
def toMesos(state: TaskState): MesosTaskState = state match {
case LAUNCHING => MesosTaskState.TASK_STARTING
diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala
index 35b324ba6f57329c6622ea71599d2ae68a49e52b..398ca41e161519a77b703ff1176e8d28975d0352 100644
--- a/core/src/main/scala/org/apache/spark/TestUtils.scala
+++ b/core/src/main/scala/org/apache/spark/TestUtils.scala
@@ -107,7 +107,7 @@ private[spark] object TestUtils {
private class JavaSourceFromString(val name: String, val code: String)
extends SimpleJavaFileObject(createURI(name), SOURCE) {
- override def getCharContent(ignoreEncodingErrors: Boolean) = code
+ override def getCharContent(ignoreEncodingErrors: Boolean): String = code
}
/** Creates a compiled class with the given name. Class file will be placed in destDir. */
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
index 3e9beb670f7ad350feaabce992db559b89b0abf8..18ccd625fc8d126d77d6282af2ce065a95d1675b 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
@@ -179,7 +179,7 @@ class JavaRDD[T](val rdd: RDD[T])(implicit val classTag: ClassTag[T])
def subtract(other: JavaRDD[T], p: Partitioner): JavaRDD[T] =
wrapRDD(rdd.subtract(other, p))
- override def toString = rdd.toString
+ override def toString: String = rdd.toString
/** Assign a name to this RDD */
def setName(name: String): JavaRDD[T] = {
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
index 6d6ed693be752f2055806b576b90bc7a6892d8fd..3be6783bba49d3712e1b66ffab1496da16fd7985 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
@@ -108,7 +108,7 @@ class JavaSparkContext(val sc: SparkContext)
private[spark] val env = sc.env
- def statusTracker = new JavaSparkStatusTracker(sc)
+ def statusTracker: JavaSparkStatusTracker = new JavaSparkStatusTracker(sc)
def isLocal: java.lang.Boolean = sc.isLocal
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala
index 71b26737b8c029d5540d12ebf481d41ff37b9eba..8f9647eea9e25f33e9bdb2b9a4b73f7a8b469940 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaUtils.scala
@@ -17,6 +17,8 @@
package org.apache.spark.api.java
+import java.util.Map.Entry
+
import com.google.common.base.Optional
import java.{util => ju}
@@ -30,8 +32,8 @@ private[spark] object JavaUtils {
}
// Workaround for SPARK-3926 / SI-8911
- def mapAsSerializableJavaMap[A, B](underlying: collection.Map[A, B]) =
- new SerializableMapWrapper(underlying)
+ def mapAsSerializableJavaMap[A, B](underlying: collection.Map[A, B]): SerializableMapWrapper[A, B]
+ = new SerializableMapWrapper(underlying)
// Implementation is copied from scala.collection.convert.Wrappers.MapWrapper,
// but implements java.io.Serializable. It can't just be subclassed to make it
@@ -40,36 +42,33 @@ private[spark] object JavaUtils {
class SerializableMapWrapper[A, B](underlying: collection.Map[A, B])
extends ju.AbstractMap[A, B] with java.io.Serializable { self =>
- override def size = underlying.size
+ override def size: Int = underlying.size
override def get(key: AnyRef): B = try {
- underlying get key.asInstanceOf[A] match {
- case None => null.asInstanceOf[B]
- case Some(v) => v
- }
+ underlying.getOrElse(key.asInstanceOf[A], null.asInstanceOf[B])
} catch {
case ex: ClassCastException => null.asInstanceOf[B]
}
override def entrySet: ju.Set[ju.Map.Entry[A, B]] = new ju.AbstractSet[ju.Map.Entry[A, B]] {
- def size = self.size
+ override def size: Int = self.size
- def iterator = new ju.Iterator[ju.Map.Entry[A, B]] {
+ override def iterator: ju.Iterator[ju.Map.Entry[A, B]] = new ju.Iterator[ju.Map.Entry[A, B]] {
val ui = underlying.iterator
var prev : Option[A] = None
- def hasNext = ui.hasNext
+ def hasNext: Boolean = ui.hasNext
- def next() = {
- val (k, v) = ui.next
+ def next(): Entry[A, B] = {
+ val (k, v) = ui.next()
prev = Some(k)
new ju.Map.Entry[A, B] {
import scala.util.hashing.byteswap32
- def getKey = k
- def getValue = v
- def setValue(v1 : B) = self.put(k, v1)
- override def hashCode = byteswap32(k.hashCode) + (byteswap32(v.hashCode) << 16)
- override def equals(other: Any) = other match {
+ override def getKey: A = k
+ override def getValue: B = v
+ override def setValue(v1 : B): B = self.put(k, v1)
+ override def hashCode: Int = byteswap32(k.hashCode) + (byteswap32(v.hashCode) << 16)
+ override def equals(other: Any): Boolean = other match {
case e: ju.Map.Entry[_, _] => k == e.getKey && v == e.getValue
case _ => false
}
@@ -81,7 +80,7 @@ private[spark] object JavaUtils {
case Some(k) =>
underlying match {
case mm: mutable.Map[A, _] =>
- mm remove k
+ mm.remove(k)
prev = None
case _ =>
throw new UnsupportedOperationException("remove")
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
index 4c71b69069eb33e116479cb1b0da8e80f47cd3aa..19f4c95fcad74a7850fa9db9962110e867f76c06 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
@@ -54,9 +54,11 @@ private[spark] class PythonRDD(
val bufferSize = conf.getInt("spark.buffer.size", 65536)
val reuse_worker = conf.getBoolean("spark.python.worker.reuse", true)
- override def getPartitions = firstParent.partitions
+ override def getPartitions: Array[Partition] = firstParent.partitions
- override val partitioner = if (preservePartitoning) firstParent.partitioner else None
+ override val partitioner: Option[Partitioner] = {
+ if (preservePartitoning) firstParent.partitioner else None
+ }
override def compute(split: Partition, context: TaskContext): Iterator[Array[Byte]] = {
val startTime = System.currentTimeMillis
@@ -92,7 +94,7 @@ private[spark] class PythonRDD(
// Return an iterator that read lines from the process's stdout
val stream = new DataInputStream(new BufferedInputStream(worker.getInputStream, bufferSize))
val stdoutIterator = new Iterator[Array[Byte]] {
- def next(): Array[Byte] = {
+ override def next(): Array[Byte] = {
val obj = _nextObj
if (hasNext) {
_nextObj = read()
@@ -175,7 +177,7 @@ private[spark] class PythonRDD(
var _nextObj = read()
- def hasNext = _nextObj != null
+ override def hasNext: Boolean = _nextObj != null
}
new InterruptibleIterator(context, stdoutIterator)
}
@@ -303,11 +305,10 @@ private class PythonException(msg: String, cause: Exception) extends RuntimeExce
* Form an RDD[(Array[Byte], Array[Byte])] from key-value pairs returned from Python.
* This is used by PySpark's shuffle operations.
*/
-private class PairwiseRDD(prev: RDD[Array[Byte]]) extends
- RDD[(Long, Array[Byte])](prev) {
- override def getPartitions = prev.partitions
- override val partitioner = prev.partitioner
- override def compute(split: Partition, context: TaskContext) =
+private class PairwiseRDD(prev: RDD[Array[Byte]]) extends RDD[(Long, Array[Byte])](prev) {
+ override def getPartitions: Array[Partition] = prev.partitions
+ override val partitioner: Option[Partitioner] = prev.partitioner
+ override def compute(split: Partition, context: TaskContext): Iterator[(Long, Array[Byte])] =
prev.iterator(split, context).grouped(2).map {
case Seq(a, b) => (Utils.deserializeLongValue(a), b)
case x => throw new SparkException("PairwiseRDD: unexpected value: " + x)
@@ -435,7 +436,7 @@ private[spark] object PythonRDD extends Logging {
keyConverterClass: String,
valueConverterClass: String,
minSplits: Int,
- batchSize: Int) = {
+ batchSize: Int): JavaRDD[Array[Byte]] = {
val keyClass = Option(keyClassMaybeNull).getOrElse("org.apache.hadoop.io.Text")
val valueClass = Option(valueClassMaybeNull).getOrElse("org.apache.hadoop.io.Text")
val kc = Utils.classForName(keyClass).asInstanceOf[Class[K]]
@@ -462,7 +463,7 @@ private[spark] object PythonRDD extends Logging {
keyConverterClass: String,
valueConverterClass: String,
confAsMap: java.util.HashMap[String, String],
- batchSize: Int) = {
+ batchSize: Int): JavaRDD[Array[Byte]] = {
val mergedConf = getMergedConf(confAsMap, sc.hadoopConfiguration())
val rdd =
newAPIHadoopRDDFromClassNames[K, V, F](sc,
@@ -488,7 +489,7 @@ private[spark] object PythonRDD extends Logging {
keyConverterClass: String,
valueConverterClass: String,
confAsMap: java.util.HashMap[String, String],
- batchSize: Int) = {
+ batchSize: Int): JavaRDD[Array[Byte]] = {
val conf = PythonHadoopUtil.mapToConf(confAsMap)
val rdd =
newAPIHadoopRDDFromClassNames[K, V, F](sc,
@@ -505,7 +506,7 @@ private[spark] object PythonRDD extends Logging {
inputFormatClass: String,
keyClass: String,
valueClass: String,
- conf: Configuration) = {
+ conf: Configuration): RDD[(K, V)] = {
val kc = Utils.classForName(keyClass).asInstanceOf[Class[K]]
val vc = Utils.classForName(valueClass).asInstanceOf[Class[V]]
val fc = Utils.classForName(inputFormatClass).asInstanceOf[Class[F]]
@@ -531,7 +532,7 @@ private[spark] object PythonRDD extends Logging {
keyConverterClass: String,
valueConverterClass: String,
confAsMap: java.util.HashMap[String, String],
- batchSize: Int) = {
+ batchSize: Int): JavaRDD[Array[Byte]] = {
val mergedConf = getMergedConf(confAsMap, sc.hadoopConfiguration())
val rdd =
hadoopRDDFromClassNames[K, V, F](sc,
@@ -557,7 +558,7 @@ private[spark] object PythonRDD extends Logging {
keyConverterClass: String,
valueConverterClass: String,
confAsMap: java.util.HashMap[String, String],
- batchSize: Int) = {
+ batchSize: Int): JavaRDD[Array[Byte]] = {
val conf = PythonHadoopUtil.mapToConf(confAsMap)
val rdd =
hadoopRDDFromClassNames[K, V, F](sc,
@@ -686,7 +687,7 @@ private[spark] object PythonRDD extends Logging {
pyRDD: JavaRDD[Array[Byte]],
batchSerialized: Boolean,
path: String,
- compressionCodecClass: String) = {
+ compressionCodecClass: String): Unit = {
saveAsHadoopFile(
pyRDD, batchSerialized, path, "org.apache.hadoop.mapred.SequenceFileOutputFormat",
null, null, null, null, new java.util.HashMap(), compressionCodecClass)
@@ -711,7 +712,7 @@ private[spark] object PythonRDD extends Logging {
keyConverterClass: String,
valueConverterClass: String,
confAsMap: java.util.HashMap[String, String],
- compressionCodecClass: String) = {
+ compressionCodecClass: String): Unit = {
val rdd = SerDeUtil.pythonToPairRDD(pyRDD, batchSerialized)
val (kc, vc) = getKeyValueTypes(keyClass, valueClass).getOrElse(
inferKeyValueTypes(rdd, keyConverterClass, valueConverterClass))
@@ -741,7 +742,7 @@ private[spark] object PythonRDD extends Logging {
valueClass: String,
keyConverterClass: String,
valueConverterClass: String,
- confAsMap: java.util.HashMap[String, String]) = {
+ confAsMap: java.util.HashMap[String, String]): Unit = {
val rdd = SerDeUtil.pythonToPairRDD(pyRDD, batchSerialized)
val (kc, vc) = getKeyValueTypes(keyClass, valueClass).getOrElse(
inferKeyValueTypes(rdd, keyConverterClass, valueConverterClass))
@@ -766,7 +767,7 @@ private[spark] object PythonRDD extends Logging {
confAsMap: java.util.HashMap[String, String],
keyConverterClass: String,
valueConverterClass: String,
- useNewAPI: Boolean) = {
+ useNewAPI: Boolean): Unit = {
val conf = PythonHadoopUtil.mapToConf(confAsMap)
val converted = convertRDD(SerDeUtil.pythonToPairRDD(pyRDD, batchSerialized),
keyConverterClass, valueConverterClass, new JavaToWritableConverter)
diff --git a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
index fb52a960e0765588a3775e9317ce812c4baa7a77..257491e90dd6610799bd5754cf913c22d9151c65 100644
--- a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
@@ -84,7 +84,7 @@ private[spark] object SerDeUtil extends Logging {
private var initialized = false
// This should be called before trying to unpickle array.array from Python
// In cluster mode, this should be put in closure
- def initialize() = {
+ def initialize(): Unit = {
synchronized{
if (!initialized) {
Unpickler.registerConstructor("array", "array", new ArrayConstructor())
diff --git a/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala b/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala
index cf289fb3ae39fc1d995c3e625f6dd44e99500bfc..8f30ff9202c838d1587494df7d591b75f8a987fc 100644
--- a/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala
@@ -18,38 +18,37 @@
package org.apache.spark.api.python
import java.io.{DataOutput, DataInput}
+import java.{util => ju}
import com.google.common.base.Charsets.UTF_8
import org.apache.hadoop.io._
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat
+
+import org.apache.spark.SparkException
import org.apache.spark.api.java.JavaSparkContext
-import org.apache.spark.{SparkContext, SparkException}
/**
* A class to test Pyrolite serialization on the Scala side, that will be deserialized
* in Python
- * @param str
- * @param int
- * @param double
*/
case class TestWritable(var str: String, var int: Int, var double: Double) extends Writable {
def this() = this("", 0, 0.0)
- def getStr = str
+ def getStr: String = str
def setStr(str: String) { this.str = str }
- def getInt = int
+ def getInt: Int = int
def setInt(int: Int) { this.int = int }
- def getDouble = double
+ def getDouble: Double = double
def setDouble(double: Double) { this.double = double }
- def write(out: DataOutput) = {
+ def write(out: DataOutput): Unit = {
out.writeUTF(str)
out.writeInt(int)
out.writeDouble(double)
}
- def readFields(in: DataInput) = {
+ def readFields(in: DataInput): Unit = {
str = in.readUTF()
int = in.readInt()
double = in.readDouble()
@@ -57,28 +56,28 @@ case class TestWritable(var str: String, var int: Int, var double: Double) exten
}
private[python] class TestInputKeyConverter extends Converter[Any, Any] {
- override def convert(obj: Any) = {
+ override def convert(obj: Any): Char = {
obj.asInstanceOf[IntWritable].get().toChar
}
}
private[python] class TestInputValueConverter extends Converter[Any, Any] {
import collection.JavaConversions._
- override def convert(obj: Any) = {
+ override def convert(obj: Any): ju.List[Double] = {
val m = obj.asInstanceOf[MapWritable]
seqAsJavaList(m.keySet.map(w => w.asInstanceOf[DoubleWritable].get()).toSeq)
}
}
private[python] class TestOutputKeyConverter extends Converter[Any, Any] {
- override def convert(obj: Any) = {
+ override def convert(obj: Any): Text = {
new Text(obj.asInstanceOf[Int].toString)
}
}
private[python] class TestOutputValueConverter extends Converter[Any, Any] {
import collection.JavaConversions._
- override def convert(obj: Any) = {
+ override def convert(obj: Any): DoubleWritable = {
new DoubleWritable(obj.asInstanceOf[java.util.Map[Double, _]].keySet().head)
}
}
@@ -86,7 +85,7 @@ private[python] class TestOutputValueConverter extends Converter[Any, Any] {
private[python] class DoubleArrayWritable extends ArrayWritable(classOf[DoubleWritable])
private[python] class DoubleArrayToWritableConverter extends Converter[Any, Writable] {
- override def convert(obj: Any) = obj match {
+ override def convert(obj: Any): DoubleArrayWritable = obj match {
case arr if arr.getClass.isArray && arr.getClass.getComponentType == classOf[Double] =>
val daw = new DoubleArrayWritable
daw.set(arr.asInstanceOf[Array[Double]].map(new DoubleWritable(_)))
diff --git a/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala
index a5ea478f231d717983ca4305c37a0bfe104f639c..12d79f6ed311b9c50b94d44eb5c3ccc3756ca609 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/Broadcast.scala
@@ -146,5 +146,5 @@ abstract class Broadcast[T: ClassTag](val id: Long) extends Serializable with Lo
}
}
- override def toString = "Broadcast(" + id + ")"
+ override def toString: String = "Broadcast(" + id + ")"
}
diff --git a/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala b/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala
index 8f8a0b11f9f2e34cd6619bc1e95c7254517fc576..685313ac009ba8cf417a3de3412af7c0e5635766 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/BroadcastManager.scala
@@ -58,7 +58,7 @@ private[spark] class BroadcastManager(
private val nextBroadcastId = new AtomicLong(0)
- def newBroadcast[T: ClassTag](value_ : T, isLocal: Boolean) = {
+ def newBroadcast[T: ClassTag](value_ : T, isLocal: Boolean): Broadcast[T] = {
broadcastFactory.newBroadcast[T](value_, isLocal, nextBroadcastId.getAndIncrement())
}
diff --git a/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala
index 1444c0dd3d2d6c3ccf8871de3233f0877a1d1064..74ccfa6d3c9a3b72bd921be87a7926cb80a96739 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala
@@ -160,7 +160,7 @@ private[broadcast] object HttpBroadcast extends Logging {
logInfo("Broadcast server started at " + serverUri)
}
- def getFile(id: Long) = new File(broadcastDir, BroadcastBlockId(id).name)
+ def getFile(id: Long): File = new File(broadcastDir, BroadcastBlockId(id).name)
private def write(id: Long, value: Any) {
val file = getFile(id)
@@ -222,7 +222,7 @@ private[broadcast] object HttpBroadcast extends Logging {
* If removeFromDriver is true, also remove these persisted blocks on the driver
* and delete the associated broadcast file.
*/
- def unpersist(id: Long, removeFromDriver: Boolean, blocking: Boolean) = synchronized {
+ def unpersist(id: Long, removeFromDriver: Boolean, blocking: Boolean): Unit = synchronized {
SparkEnv.get.blockManager.master.removeBroadcast(id, removeFromDriver, blocking)
if (removeFromDriver) {
val file = getFile(id)
diff --git a/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcastFactory.scala b/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcastFactory.scala
index c7ef02d572a19f952f9bdca4be0d8c8f95c109b3..cf3ae36f27949a7c79bf7bfe16e3f291191acfaf 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcastFactory.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcastFactory.scala
@@ -31,7 +31,7 @@ class HttpBroadcastFactory extends BroadcastFactory {
HttpBroadcast.initialize(isDriver, conf, securityMgr)
}
- override def newBroadcast[T: ClassTag](value_ : T, isLocal: Boolean, id: Long) =
+ override def newBroadcast[T: ClassTag](value_ : T, isLocal: Boolean, id: Long): Broadcast[T] =
new HttpBroadcast[T](value_, isLocal, id)
override def stop() { HttpBroadcast.stop() }
diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
index 94142d33369c773d6202a41a5e7148b471f41b18..23b02e60338fbd2c2e79f91771e6ae880ea97c44 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
@@ -222,7 +222,7 @@ private object TorrentBroadcast extends Logging {
* Remove all persisted blocks associated with this torrent broadcast on the executors.
* If removeFromDriver is true, also remove these persisted blocks on the driver.
*/
- def unpersist(id: Long, removeFromDriver: Boolean, blocking: Boolean) = {
+ def unpersist(id: Long, removeFromDriver: Boolean, blocking: Boolean): Unit = {
logDebug(s"Unpersisting TorrentBroadcast $id")
SparkEnv.get.blockManager.master.removeBroadcast(id, removeFromDriver, blocking)
}
diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala
index fb024c12094f2d8ac072d39de3a838a7c45eb1ee..96d8dd79908c8c4c71ff8c4bce5ca1ddfe868741 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcastFactory.scala
@@ -30,7 +30,7 @@ class TorrentBroadcastFactory extends BroadcastFactory {
override def initialize(isDriver: Boolean, conf: SparkConf, securityMgr: SecurityManager) { }
- override def newBroadcast[T: ClassTag](value_ : T, isLocal: Boolean, id: Long) = {
+ override def newBroadcast[T: ClassTag](value_ : T, isLocal: Boolean, id: Long): Broadcast[T] = {
new TorrentBroadcast[T](value_, id)
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/Client.scala b/core/src/main/scala/org/apache/spark/deploy/Client.scala
index 237d26fc6bd0ed0914eb6a7c8ce380e9df8126f2..65238af2caa24cd8a2660d304251558fe8bfd770 100644
--- a/core/src/main/scala/org/apache/spark/deploy/Client.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/Client.scala
@@ -38,7 +38,7 @@ private class ClientActor(driverArgs: ClientArguments, conf: SparkConf)
var masterActor: ActorSelection = _
val timeout = AkkaUtils.askTimeout(conf)
- override def preStart() = {
+ override def preStart(): Unit = {
masterActor = context.actorSelection(
Master.toAkkaUrl(driverArgs.master, AkkaUtils.protocol(context.system)))
@@ -118,7 +118,7 @@ private class ClientActor(driverArgs: ClientArguments, conf: SparkConf)
}
}
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case SubmitDriverResponse(success, driverId, message) =>
println(message)
diff --git a/core/src/main/scala/org/apache/spark/deploy/ClientArguments.scala b/core/src/main/scala/org/apache/spark/deploy/ClientArguments.scala
index 53bc62aff73950acd5595772eff67da37d4b3d75..5cbac787dceebf15991d608c25a696f8f727de9f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/ClientArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/ClientArguments.scala
@@ -42,7 +42,7 @@ private[deploy] class ClientArguments(args: Array[String]) {
var memory: Int = DEFAULT_MEMORY
var cores: Int = DEFAULT_CORES
private var _driverOptions = ListBuffer[String]()
- def driverOptions = _driverOptions.toSeq
+ def driverOptions: Seq[String] = _driverOptions.toSeq
// kill parameters
var driverId: String = ""
diff --git a/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala b/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala
index 7f600d89604a2929a35e4a0e2ac433f033e69b1e..0997507d016f5be3e126412683176085ae942368 100644
--- a/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/DeployMessage.scala
@@ -162,7 +162,7 @@ private[deploy] object DeployMessages {
Utils.checkHost(host, "Required hostname")
assert (port > 0)
- def uri = "spark://" + host + ":" + port
+ def uri: String = "spark://" + host + ":" + port
def restUri: Option[String] = restPort.map { p => "spark://" + host + ":" + p }
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala b/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala
index 5668b53fc6f4f1b388b9d5e2fbcc0c4c20e53817..a7c89276a045e2a8230de03ea66485161e35e747 100644
--- a/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/FaultToleranceTest.scala
@@ -426,7 +426,7 @@ private object SparkDocker {
}
private class DockerId(val id: String) {
- override def toString = id
+ override def toString: String = id
}
private object Docker extends Logging {
diff --git a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
index 458a7c3a455de84e909cd58725f39d09ebc477e4..dfc5b97e6a6c8d98c8c1a04b6508d509d7e9ac36 100644
--- a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
@@ -17,6 +17,7 @@
package org.apache.spark.deploy
+import org.json4s.JsonAST.JObject
import org.json4s.JsonDSL._
import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, WorkerStateResponse}
@@ -24,7 +25,7 @@ import org.apache.spark.deploy.master.{ApplicationInfo, DriverInfo, WorkerInfo}
import org.apache.spark.deploy.worker.ExecutorRunner
private[deploy] object JsonProtocol {
- def writeWorkerInfo(obj: WorkerInfo) = {
+ def writeWorkerInfo(obj: WorkerInfo): JObject = {
("id" -> obj.id) ~
("host" -> obj.host) ~
("port" -> obj.port) ~
@@ -39,7 +40,7 @@ private[deploy] object JsonProtocol {
("lastheartbeat" -> obj.lastHeartbeat)
}
- def writeApplicationInfo(obj: ApplicationInfo) = {
+ def writeApplicationInfo(obj: ApplicationInfo): JObject = {
("starttime" -> obj.startTime) ~
("id" -> obj.id) ~
("name" -> obj.desc.name) ~
@@ -51,7 +52,7 @@ private[deploy] object JsonProtocol {
("duration" -> obj.duration)
}
- def writeApplicationDescription(obj: ApplicationDescription) = {
+ def writeApplicationDescription(obj: ApplicationDescription): JObject = {
("name" -> obj.name) ~
("cores" -> obj.maxCores) ~
("memoryperslave" -> obj.memoryPerSlave) ~
@@ -59,14 +60,14 @@ private[deploy] object JsonProtocol {
("command" -> obj.command.toString)
}
- def writeExecutorRunner(obj: ExecutorRunner) = {
+ def writeExecutorRunner(obj: ExecutorRunner): JObject = {
("id" -> obj.execId) ~
("memory" -> obj.memory) ~
("appid" -> obj.appId) ~
("appdesc" -> writeApplicationDescription(obj.appDesc))
}
- def writeDriverInfo(obj: DriverInfo) = {
+ def writeDriverInfo(obj: DriverInfo): JObject = {
("id" -> obj.id) ~
("starttime" -> obj.startTime.toString) ~
("state" -> obj.state.toString) ~
@@ -74,7 +75,7 @@ private[deploy] object JsonProtocol {
("memory" -> obj.desc.mem)
}
- def writeMasterState(obj: MasterStateResponse) = {
+ def writeMasterState(obj: MasterStateResponse): JObject = {
("url" -> obj.uri) ~
("workers" -> obj.workers.toList.map(writeWorkerInfo)) ~
("cores" -> obj.workers.map(_.cores).sum) ~
@@ -87,7 +88,7 @@ private[deploy] object JsonProtocol {
("status" -> obj.status.toString)
}
- def writeWorkerState(obj: WorkerStateResponse) = {
+ def writeWorkerState(obj: WorkerStateResponse): JObject = {
("id" -> obj.workerId) ~
("masterurl" -> obj.masterUrl) ~
("masterwebuiurl" -> obj.masterWebUiUrl) ~
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index e0a32fb65cd51929e29b416d1da19516784d3b07..c2568eb4b60acbdfaa889000ca31c8868db5d1dd 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -193,7 +193,7 @@ class SparkHadoopUtil extends Logging {
* that file.
*/
def listLeafStatuses(fs: FileSystem, basePath: Path): Seq[FileStatus] = {
- def recurse(path: Path) = {
+ def recurse(path: Path): Array[FileStatus] = {
val (directories, leaves) = fs.listStatus(path).partition(_.isDir)
leaves ++ directories.flatMap(f => listLeafStatuses(fs, f.getPath))
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 4f506be63fe594a8b0b6029a69559897194df281..660307d19eab4de29983575aa379bd223fbe18e3 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -777,7 +777,7 @@ private[deploy] object SparkSubmitUtils {
}
/** A nice function to use in tests as well. Values are dummy strings. */
- def getModuleDescriptor = DefaultModuleDescriptor.newDefaultInstance(
+ def getModuleDescriptor: DefaultModuleDescriptor = DefaultModuleDescriptor.newDefaultInstance(
ModuleRevisionId.newInstance("org.apache.spark", "spark-submit-parent", "1.0"))
/**
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index 2250d5a28e4ef6f277c189847a36624c5a458567..6eb73c43470a5ef702ec7c6e504dcde4e7b2b590 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -252,7 +252,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
master.startsWith("spark://") && deployMode == "cluster"
}
- override def toString = {
+ override def toString: String = {
s"""Parsed arguments:
| master $master
| deployMode $deployMode
diff --git a/core/src/main/scala/org/apache/spark/deploy/client/AppClient.scala b/core/src/main/scala/org/apache/spark/deploy/client/AppClient.scala
index 2d24083a77b73920b56fefe9a9a48a67a3d1d5f4..3b729725257efe83f50ed9ffe2869eb3fc6e58c0 100644
--- a/core/src/main/scala/org/apache/spark/deploy/client/AppClient.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/client/AppClient.scala
@@ -116,7 +116,7 @@ private[spark] class AppClient(
masterAkkaUrls.map(AddressFromURIString(_).hostPort).contains(remoteUrl.hostPort)
}
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case RegisteredApplication(appId_, masterUrl) =>
appId = appId_
registered = true
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index db7c499661319c91e771193206caf1cdbd72c429..80c9c13ddec1e9aaa9870c1d7d6f0251b6d5919f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -93,7 +93,7 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis
*/
private def getRunner(operateFun: () => Unit): Runnable = {
new Runnable() {
- override def run() = Utils.tryOrExit {
+ override def run(): Unit = Utils.tryOrExit {
operateFun()
}
}
@@ -141,7 +141,7 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis
}
}
- override def getListing() = applications.values
+ override def getListing(): Iterable[FsApplicationHistoryInfo] = applications.values
override def getAppUI(appId: String): Option[SparkUI] = {
try {
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index af483d560b33e269eb2ef4fab10e84f390ecd133..72f6048239297a146024a415d898ee92c93f22b4 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -61,7 +61,7 @@ class HistoryServer(
private val appCache = CacheBuilder.newBuilder()
.maximumSize(retainedApplications)
.removalListener(new RemovalListener[String, SparkUI] {
- override def onRemoval(rm: RemovalNotification[String, SparkUI]) = {
+ override def onRemoval(rm: RemovalNotification[String, SparkUI]): Unit = {
detachSparkUI(rm.getValue())
}
})
@@ -149,14 +149,14 @@ class HistoryServer(
*
* @return List of all known applications.
*/
- def getApplicationList() = provider.getListing()
+ def getApplicationList(): Iterable[ApplicationHistoryInfo] = provider.getListing()
/**
* Returns the provider configuration to show in the listing page.
*
* @return A map with the provider's configuration.
*/
- def getProviderConfig() = provider.getConfig()
+ def getProviderConfig(): Map[String, String] = provider.getConfig()
}
@@ -195,9 +195,7 @@ object HistoryServer extends Logging {
server.bind()
Runtime.getRuntime().addShutdownHook(new Thread("HistoryServerStopper") {
- override def run() = {
- server.stop()
- }
+ override def run(): Unit = server.stop()
})
// Wait until the end of the world... or if the HistoryServer process is manually stopped
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
index d2d30bfd7fcba2d469e20ecae9ba4962e19377d1..32499b3a784a1ad37d3591c2e681465f1a6eb6ba 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
@@ -48,7 +48,7 @@ private[master] class FileSystemPersistenceEngine(
new File(dir + File.separator + name).delete()
}
- override def read[T: ClassTag](prefix: String) = {
+ override def read[T: ClassTag](prefix: String): Seq[T] = {
val files = new File(dir).listFiles().filter(_.getName.startsWith(prefix))
files.map(deserializeFromFile[T])
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 1b42121c8db05be3bb159acf4a4f5098e24f2f63..80506621f4d247982d09f326c53f82dd0b44a0da 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -204,7 +204,7 @@ private[master] class Master(
self ! RevokedLeadership
}
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case ElectedLeader => {
val (storedApps, storedDrivers, storedWorkers) = persistenceEngine.readPersistedData()
state = if (storedApps.isEmpty && storedDrivers.isEmpty && storedWorkers.isEmpty) {
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala b/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala
index 1583bf1f600320d218778940094602ed0bc4c7a5..351db8fab2041a584971162390f8e7aadfcdb6a1 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/RecoveryModeFactory.scala
@@ -51,20 +51,27 @@ abstract class StandaloneRecoveryModeFactory(conf: SparkConf, serializer: Serial
*/
private[master] class FileSystemRecoveryModeFactory(conf: SparkConf, serializer: Serialization)
extends StandaloneRecoveryModeFactory(conf, serializer) with Logging {
+
val RECOVERY_DIR = conf.get("spark.deploy.recoveryDirectory", "")
- def createPersistenceEngine() = {
+ def createPersistenceEngine(): PersistenceEngine = {
logInfo("Persisting recovery state to directory: " + RECOVERY_DIR)
new FileSystemPersistenceEngine(RECOVERY_DIR, serializer)
}
- def createLeaderElectionAgent(master: LeaderElectable) = new MonarchyLeaderAgent(master)
+ def createLeaderElectionAgent(master: LeaderElectable): LeaderElectionAgent = {
+ new MonarchyLeaderAgent(master)
+ }
}
private[master] class ZooKeeperRecoveryModeFactory(conf: SparkConf, serializer: Serialization)
extends StandaloneRecoveryModeFactory(conf, serializer) {
- def createPersistenceEngine() = new ZooKeeperPersistenceEngine(conf, serializer)
- def createLeaderElectionAgent(master: LeaderElectable) =
+ def createPersistenceEngine(): PersistenceEngine = {
+ new ZooKeeperPersistenceEngine(conf, serializer)
+ }
+
+ def createLeaderElectionAgent(master: LeaderElectable): LeaderElectionAgent = {
new ZooKeeperLeaderElectionAgent(master, conf)
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala b/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala
index e94aae93e4495af9ad390b007eae690be11372c0..9b3d48c6edc8483094485c32d1c000cb76d22a20 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala
@@ -104,7 +104,7 @@ private[spark] class WorkerInfo(
"http://" + this.publicAddress + ":" + this.webUiPort
}
- def setState(state: WorkerState.Value) = {
+ def setState(state: WorkerState.Value): Unit = {
this.state = state
}
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
index 1ac6677ad2b6d795c130bacfa38905d7f7f324d9..a285783f7200082cc417cf5c291c90987c873ef8 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
@@ -46,7 +46,7 @@ private[master] class ZooKeeperPersistenceEngine(conf: SparkConf, val serializat
zk.delete().forPath(WORKING_DIR + "/" + name)
}
- override def read[T: ClassTag](prefix: String) = {
+ override def read[T: ClassTag](prefix: String): Seq[T] = {
val file = zk.getChildren.forPath(WORKING_DIR).filter(_.startsWith(prefix))
file.map(deserializeFromFile[T]).flatten
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
index dee2e4a447c6e5d85d31545799cdd9f1e3daeffd..46509e39c0f2366e6b2ed02e93f3ea67c47da9d2 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
@@ -95,7 +95,7 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
// For now we only show driver information if the user has submitted drivers to the cluster.
// This is until we integrate the notion of drivers and applications in the UI.
- def hasDrivers = activeDrivers.length > 0 || completedDrivers.length > 0
+ def hasDrivers: Boolean = activeDrivers.length > 0 || completedDrivers.length > 0
val content =
<div class="row-fluid">
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
index 27a9eabb1ede7dda6a2dbd1ae9a3acf00932028a..e0948e16ef35421ba3803938d328475e5c0c3af3 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
@@ -56,8 +56,14 @@ private[deploy] class DriverRunner(
private var finalExitCode: Option[Int] = None
// Decoupled for testing
- def setClock(_clock: Clock) = clock = _clock
- def setSleeper(_sleeper: Sleeper) = sleeper = _sleeper
+ def setClock(_clock: Clock): Unit = {
+ clock = _clock
+ }
+
+ def setSleeper(_sleeper: Sleeper): Unit = {
+ sleeper = _sleeper
+ }
+
private var clock: Clock = new SystemClock()
private var sleeper = new Sleeper {
def sleep(seconds: Int): Unit = (0 until seconds).takeWhile(f => {Thread.sleep(1000); !killed})
@@ -155,7 +161,7 @@ private[deploy] class DriverRunner(
private def launchDriver(builder: ProcessBuilder, baseDir: File, supervise: Boolean) {
builder.directory(baseDir)
- def initialize(process: Process) = {
+ def initialize(process: Process): Unit = {
// Redirect stdout and stderr to files
val stdout = new File(baseDir, "stdout")
CommandUtils.redirectStream(process.getInputStream, stdout)
@@ -169,8 +175,8 @@ private[deploy] class DriverRunner(
runCommandWithRetry(ProcessBuilderLike(builder), initialize, supervise)
}
- def runCommandWithRetry(command: ProcessBuilderLike, initialize: Process => Unit,
- supervise: Boolean) {
+ def runCommandWithRetry(
+ command: ProcessBuilderLike, initialize: Process => Unit, supervise: Boolean): Unit = {
// Time to wait between submission retries.
var waitSeconds = 1
// A run of this many seconds resets the exponential back-off.
@@ -216,8 +222,8 @@ private[deploy] trait ProcessBuilderLike {
}
private[deploy] object ProcessBuilderLike {
- def apply(processBuilder: ProcessBuilder) = new ProcessBuilderLike {
- def start() = processBuilder.start()
- def command = processBuilder.command()
+ def apply(processBuilder: ProcessBuilder): ProcessBuilderLike = new ProcessBuilderLike {
+ override def start(): Process = processBuilder.start()
+ override def command: Seq[String] = processBuilder.command()
}
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
index c1b0a295f9f74cfe798b42529804bc8dd877d927..c4c24a7866aa3155d5f4e35547c3cbfb6467121d 100755
--- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
@@ -275,7 +275,7 @@ private[worker] class Worker(
}
}
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case RegisteredWorker(masterUrl, masterWebUiUrl) =>
logInfo("Successfully registered with master " + masterUrl)
registered = true
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala b/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
index 09d866fb0cd906cdf22be77113af97c262fe1a87..e0790274d7d3e414d74ca0d2242154022726da06 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/WorkerWatcher.scala
@@ -50,7 +50,7 @@ private[spark] class WorkerWatcher(workerUrl: String)
private def exitNonZero() = if (isTesting) isShutDown = true else System.exit(-1)
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case AssociatedEvent(localAddress, remoteAddress, inbound) if isWorker(remoteAddress) =>
logInfo(s"Successfully connected to $workerUrl")
diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
index dd19e4947db1ec1bacb5475db922175ad611430e..b5205d4e997ae9d6a50339bcdc83ca84479f1428 100644
--- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
+++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
@@ -62,7 +62,7 @@ private[spark] class CoarseGrainedExecutorBackend(
.map(e => (e._1.substring(prefix.length).toLowerCase, e._2))
}
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case RegisteredExecutor =>
logInfo("Successfully registered with driver")
val (hostname, _) = Utils.parseHostPort(hostPort)
diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorActor.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorActor.scala
index 41925f7e97e8404cc11b43c0f1642de389fb4c02..3e47d13f7545da9e96c629676fafb1f5aaad65bc 100644
--- a/core/src/main/scala/org/apache/spark/executor/ExecutorActor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ExecutorActor.scala
@@ -33,7 +33,7 @@ private[spark] case object TriggerThreadDump
private[spark]
class ExecutorActor(executorId: String) extends Actor with ActorLogReceive with Logging {
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case TriggerThreadDump =>
sender ! Utils.getThreadDump()
}
diff --git a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala
index 07b152651dedfe3363f8619fb2b4cad967ce2869..06152f16ae61843e98627a347e18195110663dd0 100644
--- a/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala
+++ b/core/src/main/scala/org/apache/spark/executor/TaskMetrics.scala
@@ -17,13 +17,10 @@
package org.apache.spark.executor
-import java.util.concurrent.atomic.AtomicLong
-
-import org.apache.spark.executor.DataReadMethod.DataReadMethod
-
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.annotation.DeveloperApi
+import org.apache.spark.executor.DataReadMethod.DataReadMethod
import org.apache.spark.storage.{BlockId, BlockStatus}
/**
@@ -44,14 +41,14 @@ class TaskMetrics extends Serializable {
* Host's name the task runs on
*/
private var _hostname: String = _
- def hostname = _hostname
+ def hostname: String = _hostname
private[spark] def setHostname(value: String) = _hostname = value
/**
* Time taken on the executor to deserialize this task
*/
private var _executorDeserializeTime: Long = _
- def executorDeserializeTime = _executorDeserializeTime
+ def executorDeserializeTime: Long = _executorDeserializeTime
private[spark] def setExecutorDeserializeTime(value: Long) = _executorDeserializeTime = value
@@ -59,14 +56,14 @@ class TaskMetrics extends Serializable {
* Time the executor spends actually running the task (including fetching shuffle data)
*/
private var _executorRunTime: Long = _
- def executorRunTime = _executorRunTime
+ def executorRunTime: Long = _executorRunTime
private[spark] def setExecutorRunTime(value: Long) = _executorRunTime = value
/**
* The number of bytes this task transmitted back to the driver as the TaskResult
*/
private var _resultSize: Long = _
- def resultSize = _resultSize
+ def resultSize: Long = _resultSize
private[spark] def setResultSize(value: Long) = _resultSize = value
@@ -74,31 +71,31 @@ class TaskMetrics extends Serializable {
* Amount of time the JVM spent in garbage collection while executing this task
*/
private var _jvmGCTime: Long = _
- def jvmGCTime = _jvmGCTime
+ def jvmGCTime: Long = _jvmGCTime
private[spark] def setJvmGCTime(value: Long) = _jvmGCTime = value
/**
* Amount of time spent serializing the task result
*/
private var _resultSerializationTime: Long = _
- def resultSerializationTime = _resultSerializationTime
+ def resultSerializationTime: Long = _resultSerializationTime
private[spark] def setResultSerializationTime(value: Long) = _resultSerializationTime = value
/**
* The number of in-memory bytes spilled by this task
*/
private var _memoryBytesSpilled: Long = _
- def memoryBytesSpilled = _memoryBytesSpilled
- private[spark] def incMemoryBytesSpilled(value: Long) = _memoryBytesSpilled += value
- private[spark] def decMemoryBytesSpilled(value: Long) = _memoryBytesSpilled -= value
+ def memoryBytesSpilled: Long = _memoryBytesSpilled
+ private[spark] def incMemoryBytesSpilled(value: Long): Unit = _memoryBytesSpilled += value
+ private[spark] def decMemoryBytesSpilled(value: Long): Unit = _memoryBytesSpilled -= value
/**
* The number of on-disk bytes spilled by this task
*/
private var _diskBytesSpilled: Long = _
- def diskBytesSpilled = _diskBytesSpilled
- def incDiskBytesSpilled(value: Long) = _diskBytesSpilled += value
- def decDiskBytesSpilled(value: Long) = _diskBytesSpilled -= value
+ def diskBytesSpilled: Long = _diskBytesSpilled
+ def incDiskBytesSpilled(value: Long): Unit = _diskBytesSpilled += value
+ def decDiskBytesSpilled(value: Long): Unit = _diskBytesSpilled -= value
/**
* If this task reads from a HadoopRDD or from persisted data, metrics on how much data was read
@@ -106,7 +103,7 @@ class TaskMetrics extends Serializable {
*/
private var _inputMetrics: Option[InputMetrics] = None
- def inputMetrics = _inputMetrics
+ def inputMetrics: Option[InputMetrics] = _inputMetrics
/**
* This should only be used when recreating TaskMetrics, not when updating input metrics in
@@ -128,7 +125,7 @@ class TaskMetrics extends Serializable {
*/
private var _shuffleReadMetrics: Option[ShuffleReadMetrics] = None
- def shuffleReadMetrics = _shuffleReadMetrics
+ def shuffleReadMetrics: Option[ShuffleReadMetrics] = _shuffleReadMetrics
/**
* This should only be used when recreating TaskMetrics, not when updating read metrics in
@@ -177,17 +174,18 @@ class TaskMetrics extends Serializable {
* Once https://issues.apache.org/jira/browse/SPARK-5225 is addressed,
* we can store all the different inputMetrics (one per readMethod).
*/
- private[spark] def getInputMetricsForReadMethod(
- readMethod: DataReadMethod): InputMetrics = synchronized {
- _inputMetrics match {
- case None =>
- val metrics = new InputMetrics(readMethod)
- _inputMetrics = Some(metrics)
- metrics
- case Some(metrics @ InputMetrics(method)) if method == readMethod =>
- metrics
- case Some(InputMetrics(method)) =>
- new InputMetrics(readMethod)
+ private[spark] def getInputMetricsForReadMethod(readMethod: DataReadMethod): InputMetrics = {
+ synchronized {
+ _inputMetrics match {
+ case None =>
+ val metrics = new InputMetrics(readMethod)
+ _inputMetrics = Some(metrics)
+ metrics
+ case Some(metrics @ InputMetrics(method)) if method == readMethod =>
+ metrics
+ case Some(InputMetrics(method)) =>
+ new InputMetrics(readMethod)
+ }
}
}
@@ -256,14 +254,14 @@ case class InputMetrics(readMethod: DataReadMethod.Value) {
*/
private var _bytesRead: Long = _
def bytesRead: Long = _bytesRead
- def incBytesRead(bytes: Long) = _bytesRead += bytes
+ def incBytesRead(bytes: Long): Unit = _bytesRead += bytes
/**
* Total records read.
*/
private var _recordsRead: Long = _
def recordsRead: Long = _recordsRead
- def incRecordsRead(records: Long) = _recordsRead += records
+ def incRecordsRead(records: Long): Unit = _recordsRead += records
/**
* Invoke the bytesReadCallback and mutate bytesRead.
@@ -293,15 +291,15 @@ case class OutputMetrics(writeMethod: DataWriteMethod.Value) {
* Total bytes written
*/
private var _bytesWritten: Long = _
- def bytesWritten = _bytesWritten
- private[spark] def setBytesWritten(value : Long) = _bytesWritten = value
+ def bytesWritten: Long = _bytesWritten
+ private[spark] def setBytesWritten(value : Long): Unit = _bytesWritten = value
/**
* Total records written
*/
private var _recordsWritten: Long = 0L
- def recordsWritten = _recordsWritten
- private[spark] def setRecordsWritten(value: Long) = _recordsWritten = value
+ def recordsWritten: Long = _recordsWritten
+ private[spark] def setRecordsWritten(value: Long): Unit = _recordsWritten = value
}
/**
@@ -314,7 +312,7 @@ class ShuffleReadMetrics extends Serializable {
* Number of remote blocks fetched in this shuffle by this task
*/
private var _remoteBlocksFetched: Int = _
- def remoteBlocksFetched = _remoteBlocksFetched
+ def remoteBlocksFetched: Int = _remoteBlocksFetched
private[spark] def incRemoteBlocksFetched(value: Int) = _remoteBlocksFetched += value
private[spark] def decRemoteBlocksFetched(value: Int) = _remoteBlocksFetched -= value
@@ -322,7 +320,7 @@ class ShuffleReadMetrics extends Serializable {
* Number of local blocks fetched in this shuffle by this task
*/
private var _localBlocksFetched: Int = _
- def localBlocksFetched = _localBlocksFetched
+ def localBlocksFetched: Int = _localBlocksFetched
private[spark] def incLocalBlocksFetched(value: Int) = _localBlocksFetched += value
private[spark] def decLocalBlocksFetched(value: Int) = _localBlocksFetched -= value
@@ -332,7 +330,7 @@ class ShuffleReadMetrics extends Serializable {
* still not finished processing block A, it is not considered to be blocking on block B.
*/
private var _fetchWaitTime: Long = _
- def fetchWaitTime = _fetchWaitTime
+ def fetchWaitTime: Long = _fetchWaitTime
private[spark] def incFetchWaitTime(value: Long) = _fetchWaitTime += value
private[spark] def decFetchWaitTime(value: Long) = _fetchWaitTime -= value
@@ -340,7 +338,7 @@ class ShuffleReadMetrics extends Serializable {
* Total number of remote bytes read from the shuffle by this task
*/
private var _remoteBytesRead: Long = _
- def remoteBytesRead = _remoteBytesRead
+ def remoteBytesRead: Long = _remoteBytesRead
private[spark] def incRemoteBytesRead(value: Long) = _remoteBytesRead += value
private[spark] def decRemoteBytesRead(value: Long) = _remoteBytesRead -= value
@@ -348,24 +346,24 @@ class ShuffleReadMetrics extends Serializable {
* Shuffle data that was read from the local disk (as opposed to from a remote executor).
*/
private var _localBytesRead: Long = _
- def localBytesRead = _localBytesRead
+ def localBytesRead: Long = _localBytesRead
private[spark] def incLocalBytesRead(value: Long) = _localBytesRead += value
/**
* Total bytes fetched in the shuffle by this task (both remote and local).
*/
- def totalBytesRead = _remoteBytesRead + _localBytesRead
+ def totalBytesRead: Long = _remoteBytesRead + _localBytesRead
/**
* Number of blocks fetched in this shuffle by this task (remote or local)
*/
- def totalBlocksFetched = _remoteBlocksFetched + _localBlocksFetched
+ def totalBlocksFetched: Int = _remoteBlocksFetched + _localBlocksFetched
/**
* Total number of records read from the shuffle by this task
*/
private var _recordsRead: Long = _
- def recordsRead = _recordsRead
+ def recordsRead: Long = _recordsRead
private[spark] def incRecordsRead(value: Long) = _recordsRead += value
private[spark] def decRecordsRead(value: Long) = _recordsRead -= value
}
@@ -380,7 +378,7 @@ class ShuffleWriteMetrics extends Serializable {
* Number of bytes written for the shuffle by this task
*/
@volatile private var _shuffleBytesWritten: Long = _
- def shuffleBytesWritten = _shuffleBytesWritten
+ def shuffleBytesWritten: Long = _shuffleBytesWritten
private[spark] def incShuffleBytesWritten(value: Long) = _shuffleBytesWritten += value
private[spark] def decShuffleBytesWritten(value: Long) = _shuffleBytesWritten -= value
@@ -388,7 +386,7 @@ class ShuffleWriteMetrics extends Serializable {
* Time the task spent blocking on writes to disk or buffer cache, in nanoseconds
*/
@volatile private var _shuffleWriteTime: Long = _
- def shuffleWriteTime= _shuffleWriteTime
+ def shuffleWriteTime: Long = _shuffleWriteTime
private[spark] def incShuffleWriteTime(value: Long) = _shuffleWriteTime += value
private[spark] def decShuffleWriteTime(value: Long) = _shuffleWriteTime -= value
@@ -396,7 +394,7 @@ class ShuffleWriteMetrics extends Serializable {
* Total number of records written to the shuffle by this task
*/
@volatile private var _shuffleRecordsWritten: Long = _
- def shuffleRecordsWritten = _shuffleRecordsWritten
+ def shuffleRecordsWritten: Long = _shuffleRecordsWritten
private[spark] def incShuffleRecordsWritten(value: Long) = _shuffleRecordsWritten += value
private[spark] def decShuffleRecordsWritten(value: Long) = _shuffleRecordsWritten -= value
private[spark] def setShuffleRecordsWritten(value: Long) = _shuffleRecordsWritten = value
diff --git a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
index 593a62b3e3b3288c0587e32b66f9e36da0f8cae6..6cda7772f77bce1b2e865866a719e2575c74a3c0 100644
--- a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
+++ b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
@@ -73,16 +73,16 @@ private[spark] abstract class StreamBasedRecordReader[T](
private var key = ""
private var value: T = null.asInstanceOf[T]
- override def initialize(split: InputSplit, context: TaskAttemptContext) = {}
- override def close() = {}
+ override def initialize(split: InputSplit, context: TaskAttemptContext): Unit = {}
+ override def close(): Unit = {}
- override def getProgress = if (processed) 1.0f else 0.0f
+ override def getProgress: Float = if (processed) 1.0f else 0.0f
- override def getCurrentKey = key
+ override def getCurrentKey: String = key
- override def getCurrentValue = value
+ override def getCurrentValue: T = value
- override def nextKeyValue = {
+ override def nextKeyValue: Boolean = {
if (!processed) {
val fileIn = new PortableDataStream(split, context, index)
value = parseStream(fileIn)
@@ -119,7 +119,8 @@ private[spark] class StreamRecordReader(
* The format for the PortableDataStream files
*/
private[spark] class StreamInputFormat extends StreamFileInputFormat[PortableDataStream] {
- override def createRecordReader(split: InputSplit, taContext: TaskAttemptContext) = {
+ override def createRecordReader(split: InputSplit, taContext: TaskAttemptContext)
+ : CombineFileRecordReader[String, PortableDataStream] = {
new CombineFileRecordReader[String, PortableDataStream](
split.asInstanceOf[CombineFileSplit], taContext, classOf[StreamRecordReader])
}
@@ -204,7 +205,7 @@ class PortableDataStream(
/**
* Close the file (if it is currently open)
*/
- def close() = {
+ def close(): Unit = {
if (isOpen) {
try {
fileIn.close()
diff --git a/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala b/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala
index 21b782edd2a9e0ef4524097d2f90131883232f6a..87c2aa481095d3a1c462517211455d8bd6bd4790 100644
--- a/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala
+++ b/core/src/main/scala/org/apache/spark/mapred/SparkHadoopMapRedUtil.scala
@@ -52,7 +52,7 @@ trait SparkHadoopMapRedUtil {
jobId: Int,
isMap: Boolean,
taskId: Int,
- attemptId: Int) = {
+ attemptId: Int): TaskAttemptID = {
new TaskAttemptID(jtIdentifier, jobId, isMap, taskId, attemptId)
}
diff --git a/core/src/main/scala/org/apache/spark/mapreduce/SparkHadoopMapReduceUtil.scala b/core/src/main/scala/org/apache/spark/mapreduce/SparkHadoopMapReduceUtil.scala
index 3340673f91156ecd86bc1baca009df7e8acf5ddc..cfd20392d12f16a2fd3f3070ae91a35437845d31 100644
--- a/core/src/main/scala/org/apache/spark/mapreduce/SparkHadoopMapReduceUtil.scala
+++ b/core/src/main/scala/org/apache/spark/mapreduce/SparkHadoopMapReduceUtil.scala
@@ -45,7 +45,7 @@ trait SparkHadoopMapReduceUtil {
jobId: Int,
isMap: Boolean,
taskId: Int,
- attemptId: Int) = {
+ attemptId: Int): TaskAttemptID = {
val klass = Class.forName("org.apache.hadoop.mapreduce.TaskAttemptID")
try {
// First, attempt to use the old-style constructor that takes a boolean isMap
diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
index 345db36630fd557c5c5368e824752e62c0b46282..9150ad35712a1897a21b741e15c9c55cba34ce7a 100644
--- a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
@@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit
import scala.collection.mutable
import com.codahale.metrics.{Metric, MetricFilter, MetricRegistry}
+import org.eclipse.jetty.servlet.ServletContextHandler
import org.apache.spark.{Logging, SecurityManager, SparkConf}
import org.apache.spark.metrics.sink.{MetricsServlet, Sink}
@@ -84,7 +85,7 @@ private[spark] class MetricsSystem private (
/**
* Get any UI handlers used by this metrics system; can only be called after start().
*/
- def getServletHandlers = {
+ def getServletHandlers: Array[ServletContextHandler] = {
require(running, "Can only call getServletHandlers on a running MetricsSystem")
metricsServlet.map(_.getHandlers).getOrElse(Array())
}
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala b/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala
index 2f65bc8b466091bfacd29f949a56ce47bab0b3d1..0c2e212a330746c274115a567a21233691ba75fa 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala
@@ -30,8 +30,12 @@ import org.eclipse.jetty.servlet.ServletContextHandler
import org.apache.spark.SecurityManager
import org.apache.spark.ui.JettyUtils._
-private[spark] class MetricsServlet(val property: Properties, val registry: MetricRegistry,
- securityMgr: SecurityManager) extends Sink {
+private[spark] class MetricsServlet(
+ val property: Properties,
+ val registry: MetricRegistry,
+ securityMgr: SecurityManager)
+ extends Sink {
+
val SERVLET_KEY_PATH = "path"
val SERVLET_KEY_SAMPLE = "sample"
@@ -45,10 +49,12 @@ private[spark] class MetricsServlet(val property: Properties, val registry: Metr
val mapper = new ObjectMapper().registerModule(
new MetricsModule(TimeUnit.SECONDS, TimeUnit.MILLISECONDS, servletShowSample))
- def getHandlers = Array[ServletContextHandler](
- createServletHandler(servletPath,
- new ServletParams(request => getMetricsSnapshot(request), "text/json"), securityMgr)
- )
+ def getHandlers: Array[ServletContextHandler] = {
+ Array[ServletContextHandler](
+ createServletHandler(servletPath,
+ new ServletParams(request => getMetricsSnapshot(request), "text/json"), securityMgr)
+ )
+ }
def getMetricsSnapshot(request: HttpServletRequest): String = {
mapper.writeValueAsString(registry)
diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/Sink.scala b/core/src/main/scala/org/apache/spark/metrics/sink/Sink.scala
index 0d83d8c425ca465d76e16d595a51d2e8672f529e..9fad4e7deacb6930b2a38e0c26b829abc0233a8c 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/Sink.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/Sink.scala
@@ -18,7 +18,7 @@
package org.apache.spark.metrics.sink
private[spark] trait Sink {
- def start: Unit
- def stop: Unit
+ def start(): Unit
+ def stop(): Unit
def report(): Unit
}
diff --git a/core/src/main/scala/org/apache/spark/network/nio/BlockMessageArray.scala b/core/src/main/scala/org/apache/spark/network/nio/BlockMessageArray.scala
index a1a2c00ed154238afb8cc3bb10277e721b135909..1ba25aa74aa02dcbe3cf2caa7a60876bfe534c72 100644
--- a/core/src/main/scala/org/apache/spark/network/nio/BlockMessageArray.scala
+++ b/core/src/main/scala/org/apache/spark/network/nio/BlockMessageArray.scala
@@ -32,11 +32,11 @@ class BlockMessageArray(var blockMessages: Seq[BlockMessage])
def this() = this(null.asInstanceOf[Seq[BlockMessage]])
- def apply(i: Int) = blockMessages(i)
+ def apply(i: Int): BlockMessage = blockMessages(i)
- def iterator = blockMessages.iterator
+ def iterator: Iterator[BlockMessage] = blockMessages.iterator
- def length = blockMessages.length
+ def length: Int = blockMessages.length
def set(bufferMessage: BufferMessage) {
val startTime = System.currentTimeMillis
diff --git a/core/src/main/scala/org/apache/spark/network/nio/BufferMessage.scala b/core/src/main/scala/org/apache/spark/network/nio/BufferMessage.scala
index 3b245c5c7a4f300e623d391b11926e1a4ac0e6bd..9a9e22b0c23662f5c67ac3fb1402d0848e0fbfeb 100644
--- a/core/src/main/scala/org/apache/spark/network/nio/BufferMessage.scala
+++ b/core/src/main/scala/org/apache/spark/network/nio/BufferMessage.scala
@@ -31,9 +31,9 @@ class BufferMessage(id_ : Int, val buffers: ArrayBuffer[ByteBuffer], var ackId:
val initialSize = currentSize()
var gotChunkForSendingOnce = false
- def size = initialSize
+ def size: Int = initialSize
- def currentSize() = {
+ def currentSize(): Int = {
if (buffers == null || buffers.isEmpty) {
0
} else {
@@ -100,11 +100,11 @@ class BufferMessage(id_ : Int, val buffers: ArrayBuffer[ByteBuffer], var ackId:
buffers.foreach(_.flip)
}
- def hasAckId() = (ackId != 0)
+ def hasAckId(): Boolean = ackId != 0
- def isCompletelyReceived() = !buffers(0).hasRemaining
+ def isCompletelyReceived: Boolean = !buffers(0).hasRemaining
- override def toString = {
+ override def toString: String = {
if (hasAckId) {
"BufferAckMessage(aid = " + ackId + ", id = " + id + ", size = " + size + ")"
} else {
diff --git a/core/src/main/scala/org/apache/spark/network/nio/Connection.scala b/core/src/main/scala/org/apache/spark/network/nio/Connection.scala
index c2d9578be7ebbc1ce116b9be75edcf54c5e5d3fb..04eb2bf9ba4ab26eb35c66a97e1b77b079b8bdfc 100644
--- a/core/src/main/scala/org/apache/spark/network/nio/Connection.scala
+++ b/core/src/main/scala/org/apache/spark/network/nio/Connection.scala
@@ -101,9 +101,11 @@ abstract class Connection(val channel: SocketChannel, val selector: Selector,
socketRemoteConnectionManagerId
}
- def key() = channel.keyFor(selector)
+ def key(): SelectionKey = channel.keyFor(selector)
- def getRemoteAddress() = channel.socket.getRemoteSocketAddress().asInstanceOf[InetSocketAddress]
+ def getRemoteAddress(): InetSocketAddress = {
+ channel.socket.getRemoteSocketAddress().asInstanceOf[InetSocketAddress]
+ }
// Returns whether we have to register for further reads or not.
def read(): Boolean = {
@@ -280,7 +282,7 @@ class SendingConnection(val address: InetSocketAddress, selector_ : Selector,
/* channel.socket.setSendBufferSize(256 * 1024) */
- override def getRemoteAddress() = address
+ override def getRemoteAddress(): InetSocketAddress = address
val DEFAULT_INTEREST = SelectionKey.OP_READ
diff --git a/core/src/main/scala/org/apache/spark/network/nio/ConnectionId.scala b/core/src/main/scala/org/apache/spark/network/nio/ConnectionId.scala
index 764dc5e5503ed318ca61300c8767e8625434c3c6..b3b281ff465f1cd1136983fc1799afb675652050 100644
--- a/core/src/main/scala/org/apache/spark/network/nio/ConnectionId.scala
+++ b/core/src/main/scala/org/apache/spark/network/nio/ConnectionId.scala
@@ -18,7 +18,9 @@
package org.apache.spark.network.nio
private[nio] case class ConnectionId(connectionManagerId: ConnectionManagerId, uniqId: Int) {
- override def toString = connectionManagerId.host + "_" + connectionManagerId.port + "_" + uniqId
+ override def toString: String = {
+ connectionManagerId.host + "_" + connectionManagerId.port + "_" + uniqId
+ }
}
private[nio] object ConnectionId {
diff --git a/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala b/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala
index ee22c6656e69e2bfa80d93d8dee11706598dec76..741fe3e1ea750497b3738065f7f5ad8adcd673e9 100644
--- a/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala
+++ b/core/src/main/scala/org/apache/spark/network/nio/ConnectionManager.scala
@@ -188,7 +188,7 @@ private[nio] class ConnectionManager(
private val readRunnableStarted: HashSet[SelectionKey] = new HashSet[SelectionKey]()
private val selectorThread = new Thread("connection-manager-thread") {
- override def run() = ConnectionManager.this.run()
+ override def run(): Unit = ConnectionManager.this.run()
}
selectorThread.setDaemon(true)
// start this thread last, since it invokes run(), which accesses members above
diff --git a/core/src/main/scala/org/apache/spark/network/nio/ConnectionManagerId.scala b/core/src/main/scala/org/apache/spark/network/nio/ConnectionManagerId.scala
index cbb37ec5ced1f74ad491b657bf7377f4a56dd25e..1cd13d887c6f6d6e228fb96725f5b8e6fdfa304a 100644
--- a/core/src/main/scala/org/apache/spark/network/nio/ConnectionManagerId.scala
+++ b/core/src/main/scala/org/apache/spark/network/nio/ConnectionManagerId.scala
@@ -26,7 +26,7 @@ private[nio] case class ConnectionManagerId(host: String, port: Int) {
Utils.checkHost(host)
assert (port > 0)
- def toSocketAddress() = new InetSocketAddress(host, port)
+ def toSocketAddress(): InetSocketAddress = new InetSocketAddress(host, port)
}
diff --git a/core/src/main/scala/org/apache/spark/network/nio/Message.scala b/core/src/main/scala/org/apache/spark/network/nio/Message.scala
index fb4a979b824c3bb239c401d8aae60deab2673900..85d2fe2bf9c20a4357fb8eebb76730175ef1a2d5 100644
--- a/core/src/main/scala/org/apache/spark/network/nio/Message.scala
+++ b/core/src/main/scala/org/apache/spark/network/nio/Message.scala
@@ -42,7 +42,9 @@ private[nio] abstract class Message(val typ: Long, val id: Int) {
def timeTaken(): String = (finishTime - startTime).toString + " ms"
- override def toString = this.getClass.getSimpleName + "(id = " + id + ", size = " + size + ")"
+ override def toString: String = {
+ this.getClass.getSimpleName + "(id = " + id + ", size = " + size + ")"
+ }
}
@@ -51,7 +53,7 @@ private[nio] object Message {
var lastId = 1
- def getNewId() = synchronized {
+ def getNewId(): Int = synchronized {
lastId += 1
if (lastId == 0) {
lastId += 1
diff --git a/core/src/main/scala/org/apache/spark/network/nio/MessageChunk.scala b/core/src/main/scala/org/apache/spark/network/nio/MessageChunk.scala
index 278c5ac356ef2148d16501a49d31ae2c8d35b3e0..a4568e849fa13c76e8b03d62915cf5b0fd7ad9c3 100644
--- a/core/src/main/scala/org/apache/spark/network/nio/MessageChunk.scala
+++ b/core/src/main/scala/org/apache/spark/network/nio/MessageChunk.scala
@@ -24,9 +24,9 @@ import scala.collection.mutable.ArrayBuffer
private[nio]
class MessageChunk(val header: MessageChunkHeader, val buffer: ByteBuffer) {
- val size = if (buffer == null) 0 else buffer.remaining
+ val size: Int = if (buffer == null) 0 else buffer.remaining
- lazy val buffers = {
+ lazy val buffers: ArrayBuffer[ByteBuffer] = {
val ab = new ArrayBuffer[ByteBuffer]()
ab += header.buffer
if (buffer != null) {
@@ -35,7 +35,7 @@ class MessageChunk(val header: MessageChunkHeader, val buffer: ByteBuffer) {
ab
}
- override def toString = {
+ override def toString: String = {
"" + this.getClass.getSimpleName + " (id = " + header.id + ", size = " + size + ")"
}
}
diff --git a/core/src/main/scala/org/apache/spark/network/nio/MessageChunkHeader.scala b/core/src/main/scala/org/apache/spark/network/nio/MessageChunkHeader.scala
index 6e20f291c5cecd6188d6d809757a5c04d5375dfe..7b3da4bb9d5ee42a62b2fc36650e8823e035294d 100644
--- a/core/src/main/scala/org/apache/spark/network/nio/MessageChunkHeader.scala
+++ b/core/src/main/scala/org/apache/spark/network/nio/MessageChunkHeader.scala
@@ -50,8 +50,10 @@ private[nio] class MessageChunkHeader(
flip.asInstanceOf[ByteBuffer]
}
- override def toString = "" + this.getClass.getSimpleName + ":" + id + " of type " + typ +
+ override def toString: String = {
+ "" + this.getClass.getSimpleName + ":" + id + " of type " + typ +
" and sizes " + totalSize + " / " + chunkSize + " bytes, securityNeg: " + securityNeg
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/partial/PartialResult.scala b/core/src/main/scala/org/apache/spark/partial/PartialResult.scala
index cadd0c7ed19baae9bf49ba3969eabd703490888d..53c4b32c95ab318f83e41ed621f56d76dd1d614e 100644
--- a/core/src/main/scala/org/apache/spark/partial/PartialResult.scala
+++ b/core/src/main/scala/org/apache/spark/partial/PartialResult.scala
@@ -99,7 +99,7 @@ class PartialResult[R](initialVal: R, isFinal: Boolean) {
case None => "(partial: " + initialValue + ")"
}
}
- def getFinalValueInternal() = PartialResult.this.getFinalValueInternal().map(f)
+ def getFinalValueInternal(): Option[T] = PartialResult.this.getFinalValueInternal().map(f)
}
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/CartesianRDD.scala b/core/src/main/scala/org/apache/spark/rdd/CartesianRDD.scala
index 1cbd684224b7c98cbbd4e2b398dba660607c1d4d..9059eb13bb5d8fdada6b4b09460d4020bee638e9 100644
--- a/core/src/main/scala/org/apache/spark/rdd/CartesianRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/CartesianRDD.scala
@@ -70,7 +70,7 @@ class CartesianRDD[T: ClassTag, U: ClassTag](
(rdd1.preferredLocations(currSplit.s1) ++ rdd2.preferredLocations(currSplit.s2)).distinct
}
- override def compute(split: Partition, context: TaskContext) = {
+ override def compute(split: Partition, context: TaskContext): Iterator[(T, U)] = {
val currSplit = split.asInstanceOf[CartesianPartition]
for (x <- rdd1.iterator(currSplit.s1, context);
y <- rdd2.iterator(currSplit.s2, context)) yield (x, y)
diff --git a/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala
index b073eba8a1574f96b567ec506b24c7f33fa7e9b6..5117ccfabfcc24590312b6dc813695112ed5e507 100644
--- a/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/CoalescedRDD.scala
@@ -186,7 +186,7 @@ private class PartitionCoalescer(maxPartitions: Int, prev: RDD[_], balanceSlack:
override val isEmpty = !it.hasNext
// initializes/resets to start iterating from the beginning
- def resetIterator() = {
+ def resetIterator(): Iterator[(String, Partition)] = {
val iterators = (0 to 2).map( x =>
prev.partitions.iterator.flatMap(p => {
if (currPrefLocs(p).size > x) Some((currPrefLocs(p)(x), p)) else None
@@ -196,10 +196,10 @@ private class PartitionCoalescer(maxPartitions: Int, prev: RDD[_], balanceSlack:
}
// hasNext() is false iff there are no preferredLocations for any of the partitions of the RDD
- def hasNext(): Boolean = { !isEmpty }
+ override def hasNext: Boolean = { !isEmpty }
// return the next preferredLocation of some partition of the RDD
- def next(): (String, Partition) = {
+ override def next(): (String, Partition) = {
if (it.hasNext) {
it.next()
} else {
@@ -237,7 +237,7 @@ private class PartitionCoalescer(maxPartitions: Int, prev: RDD[_], balanceSlack:
val rotIt = new LocationIterator(prev)
// deal with empty case, just create targetLen partition groups with no preferred location
- if (!rotIt.hasNext()) {
+ if (!rotIt.hasNext) {
(1 to targetLen).foreach(x => groupArr += PartitionGroup())
return
}
@@ -343,7 +343,7 @@ private class PartitionCoalescer(maxPartitions: Int, prev: RDD[_], balanceSlack:
private case class PartitionGroup(prefLoc: Option[String] = None) {
var arr = mutable.ArrayBuffer[Partition]()
- def size = arr.size
+ def size: Int = arr.size
}
private object PartitionGroup {
diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
index 486e86ce1bb1934676e08ca00208ddacd53a2ef3..f77abac42b623d2d44ce3a8d27db48f88bbd3e3d 100644
--- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
@@ -215,8 +215,7 @@ class HadoopRDD[K, V](
logInfo("Input split: " + split.inputSplit)
val jobConf = getJobConf()
- val inputMetrics = context.taskMetrics
- .getInputMetricsForReadMethod(DataReadMethod.Hadoop)
+ val inputMetrics = context.taskMetrics.getInputMetricsForReadMethod(DataReadMethod.Hadoop)
// Find a function that will return the FileSystem bytes read by this thread. Do this before
// creating RecordReader, because RecordReader's constructor might read some bytes
@@ -240,7 +239,7 @@ class HadoopRDD[K, V](
val key: K = reader.createKey()
val value: V = reader.createValue()
- override def getNext() = {
+ override def getNext(): (K, V) = {
try {
finished = !reader.next(key, value)
} catch {
@@ -337,11 +336,11 @@ private[spark] object HadoopRDD extends Logging {
* The three methods below are helpers for accessing the local map, a property of the SparkEnv of
* the local process.
*/
- def getCachedMetadata(key: String) = SparkEnv.get.hadoopJobMetadata.get(key)
+ def getCachedMetadata(key: String): Any = SparkEnv.get.hadoopJobMetadata.get(key)
- def containsCachedMetadata(key: String) = SparkEnv.get.hadoopJobMetadata.containsKey(key)
+ def containsCachedMetadata(key: String): Boolean = SparkEnv.get.hadoopJobMetadata.containsKey(key)
- def putCachedMetadata(key: String, value: Any) =
+ private def putCachedMetadata(key: String, value: Any): Unit =
SparkEnv.get.hadoopJobMetadata.put(key, value)
/** Add Hadoop configuration specific to a single partition and attempt. */
@@ -371,7 +370,7 @@ private[spark] object HadoopRDD extends Logging {
override def getPartitions: Array[Partition] = firstParent[T].partitions
- override def compute(split: Partition, context: TaskContext) = {
+ override def compute(split: Partition, context: TaskContext): Iterator[U] = {
val partition = split.asInstanceOf[HadoopPartition]
val inputSplit = partition.inputSplit.value
f(inputSplit, firstParent[T].iterator(split, context))
diff --git a/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala b/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
index e2267861e79df01a89b204df884417b2c58b9622..0c28f045e46e9720c655bb223118ec8794b1733b 100644
--- a/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/JdbcRDD.scala
@@ -17,7 +17,7 @@
package org.apache.spark.rdd
-import java.sql.{Connection, ResultSet}
+import java.sql.{PreparedStatement, Connection, ResultSet}
import scala.reflect.ClassTag
@@ -28,8 +28,9 @@ import org.apache.spark.util.NextIterator
import org.apache.spark.{Logging, Partition, SparkContext, TaskContext}
private[spark] class JdbcPartition(idx: Int, val lower: Long, val upper: Long) extends Partition {
- override def index = idx
+ override def index: Int = idx
}
+
// TODO: Expose a jdbcRDD function in SparkContext and mark this as semi-private
/**
* An RDD that executes an SQL query on a JDBC connection and reads results.
@@ -70,7 +71,8 @@ class JdbcRDD[T: ClassTag](
}).toArray
}
- override def compute(thePart: Partition, context: TaskContext) = new NextIterator[T] {
+ override def compute(thePart: Partition, context: TaskContext): Iterator[T] = new NextIterator[T]
+ {
context.addTaskCompletionListener{ context => closeIfNeeded() }
val part = thePart.asInstanceOf[JdbcPartition]
val conn = getConnection()
@@ -88,7 +90,7 @@ class JdbcRDD[T: ClassTag](
stmt.setLong(2, part.upper)
val rs = stmt.executeQuery()
- override def getNext: T = {
+ override def getNext(): T = {
if (rs.next()) {
mapRow(rs)
} else {
diff --git a/core/src/main/scala/org/apache/spark/rdd/MapPartitionsRDD.scala b/core/src/main/scala/org/apache/spark/rdd/MapPartitionsRDD.scala
index 4883fb828814c34a50de4fb8807164078be59f51..a838aac6e8d1a17008571ebb1d89039e0c5e61b6 100644
--- a/core/src/main/scala/org/apache/spark/rdd/MapPartitionsRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/MapPartitionsRDD.scala
@@ -31,6 +31,6 @@ private[spark] class MapPartitionsRDD[U: ClassTag, T: ClassTag](
override def getPartitions: Array[Partition] = firstParent[T].partitions
- override def compute(split: Partition, context: TaskContext) =
+ override def compute(split: Partition, context: TaskContext): Iterator[U] =
f(context, split.index, firstParent[T].iterator(split, context))
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
index 7fb94840df99c2ffd149efde4e937be7d4cf0101..2ab967f4bb313e7607866fdc021115f2a6bce9ef 100644
--- a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
@@ -238,7 +238,7 @@ private[spark] object NewHadoopRDD {
override def getPartitions: Array[Partition] = firstParent[T].partitions
- override def compute(split: Partition, context: TaskContext) = {
+ override def compute(split: Partition, context: TaskContext): Iterator[U] = {
val partition = split.asInstanceOf[NewHadoopPartition]
val inputSplit = partition.serializableHadoopSplit.value
f(inputSplit, firstParent[T].iterator(split, context))
diff --git a/core/src/main/scala/org/apache/spark/rdd/ParallelCollectionRDD.scala b/core/src/main/scala/org/apache/spark/rdd/ParallelCollectionRDD.scala
index f12d0cffaba341234b83202faa72b2864192e69a..e2394e28f8d264039b71a71d5cd5fa7d1d735a84 100644
--- a/core/src/main/scala/org/apache/spark/rdd/ParallelCollectionRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/ParallelCollectionRDD.scala
@@ -98,7 +98,7 @@ private[spark] class ParallelCollectionRDD[T: ClassTag](
slices.indices.map(i => new ParallelCollectionPartition(id, i, slices(i))).toArray
}
- override def compute(s: Partition, context: TaskContext) = {
+ override def compute(s: Partition, context: TaskContext): Iterator[T] = {
new InterruptibleIterator(context, s.asInstanceOf[ParallelCollectionPartition[T]].iterator)
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/PartitionPruningRDD.scala b/core/src/main/scala/org/apache/spark/rdd/PartitionPruningRDD.scala
index f781a8d776f2ac9900f47843e1e42e7261bdb124..a00f4c1cdff913c5c8668d07fc6c7ea208d637d7 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PartitionPruningRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PartitionPruningRDD.scala
@@ -40,7 +40,7 @@ private[spark] class PruneDependency[T](rdd: RDD[T], @transient partitionFilterF
.filter(s => partitionFilterFunc(s.index)).zipWithIndex
.map { case(split, idx) => new PartitionPruningRDDPartition(idx, split) : Partition }
- override def getParents(partitionId: Int) = {
+ override def getParents(partitionId: Int): List[Int] = {
List(partitions(partitionId).asInstanceOf[PartitionPruningRDDPartition].parentSplit.index)
}
}
@@ -59,8 +59,10 @@ class PartitionPruningRDD[T: ClassTag](
@transient partitionFilterFunc: Int => Boolean)
extends RDD[T](prev.context, List(new PruneDependency(prev, partitionFilterFunc))) {
- override def compute(split: Partition, context: TaskContext) = firstParent[T].iterator(
- split.asInstanceOf[PartitionPruningRDDPartition].parentSplit, context)
+ override def compute(split: Partition, context: TaskContext): Iterator[T] = {
+ firstParent[T].iterator(
+ split.asInstanceOf[PartitionPruningRDDPartition].parentSplit, context)
+ }
override protected def getPartitions: Array[Partition] =
getDependencies.head.asInstanceOf[PruneDependency[T]].partitions
@@ -74,7 +76,7 @@ object PartitionPruningRDD {
* Create a PartitionPruningRDD. This function can be used to create the PartitionPruningRDD
* when its type T is not known at compile time.
*/
- def create[T](rdd: RDD[T], partitionFilterFunc: Int => Boolean) = {
+ def create[T](rdd: RDD[T], partitionFilterFunc: Int => Boolean): PartitionPruningRDD[T] = {
new PartitionPruningRDD[T](rdd, partitionFilterFunc)(rdd.elementClassTag)
}
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
index ed79032893d33768b867aa3957ed3f69fd894336..dc60d489276240b20a840f6de650b5a793d9f933 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
@@ -149,10 +149,10 @@ private[spark] class PipedRDD[T: ClassTag](
}.start()
// Return an iterator that read lines from the process's stdout
- val lines = Source.fromInputStream(proc.getInputStream).getLines
+ val lines = Source.fromInputStream(proc.getInputStream).getLines()
new Iterator[String] {
- def next() = lines.next()
- def hasNext = {
+ def next(): String = lines.next()
+ def hasNext: Boolean = {
if (lines.hasNext) {
true
} else {
@@ -162,7 +162,7 @@ private[spark] class PipedRDD[T: ClassTag](
}
// cleanup task working directory if used
- if (workInTaskDirectory == true) {
+ if (workInTaskDirectory) {
scala.util.control.Exception.ignoring(classOf[IOException]) {
Utils.deleteRecursively(new File(taskDirectory))
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index a4c74ed03e330a7d1675c63ddb4c05adfd01ae30..ddbfd5624e741ae218bdc6c2e97924f9a84deb69 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -186,7 +186,7 @@ abstract class RDD[T: ClassTag](
}
/** Get the RDD's current storage level, or StorageLevel.NONE if none is set. */
- def getStorageLevel = storageLevel
+ def getStorageLevel: StorageLevel = storageLevel
// Our dependencies and partitions will be gotten by calling subclass's methods below, and will
// be overwritten when we're checkpointed
@@ -746,13 +746,13 @@ abstract class RDD[T: ClassTag](
def zip[U: ClassTag](other: RDD[U]): RDD[(T, U)] = {
zipPartitions(other, preservesPartitioning = false) { (thisIter, otherIter) =>
new Iterator[(T, U)] {
- def hasNext = (thisIter.hasNext, otherIter.hasNext) match {
+ def hasNext: Boolean = (thisIter.hasNext, otherIter.hasNext) match {
case (true, true) => true
case (false, false) => false
case _ => throw new SparkException("Can only zip RDDs with " +
"same number of elements in each partition")
}
- def next = (thisIter.next, otherIter.next)
+ def next(): (T, U) = (thisIter.next(), otherIter.next())
}
}
}
@@ -868,8 +868,8 @@ abstract class RDD[T: ClassTag](
// Our partitioner knows how to handle T (which, since we have a partitioner, is
// really (K, V)) so make a new Partitioner that will de-tuple our fake tuples
val p2 = new Partitioner() {
- override def numPartitions = p.numPartitions
- override def getPartition(k: Any) = p.getPartition(k.asInstanceOf[(Any, _)]._1)
+ override def numPartitions: Int = p.numPartitions
+ override def getPartition(k: Any): Int = p.getPartition(k.asInstanceOf[(Any, _)]._1)
}
// Unfortunately, since we're making a new p2, we'll get ShuffleDependencies
// anyway, and when calling .keys, will not have a partitioner set, even though
@@ -1394,7 +1394,7 @@ abstract class RDD[T: ClassTag](
}
/** The [[org.apache.spark.SparkContext]] that this RDD was created on. */
- def context = sc
+ def context: SparkContext = sc
/**
* Private API for changing an RDD's ClassTag.
diff --git a/core/src/main/scala/org/apache/spark/rdd/ShuffledRDD.scala b/core/src/main/scala/org/apache/spark/rdd/ShuffledRDD.scala
index d9fe6847254faf91ba76c653fdba796b18063d40..2dc47f95937cb26459ec97fc7bdbdfe67c353a38 100644
--- a/core/src/main/scala/org/apache/spark/rdd/ShuffledRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/ShuffledRDD.scala
@@ -17,14 +17,12 @@
package org.apache.spark.rdd
-import scala.reflect.ClassTag
-
import org.apache.spark._
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.serializer.Serializer
private[spark] class ShuffledRDDPartition(val idx: Int) extends Partition {
- override val index = idx
+ override val index: Int = idx
override def hashCode(): Int = idx
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
index ed24ea22a661c037d9b5d23cb25c7febb2240ca0..c27f435eb9c5aef8eb30cd3a9872cbd0ca6050ec 100644
--- a/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
@@ -105,7 +105,7 @@ private[spark] class SubtractedRDD[K: ClassTag, V: ClassTag, W: ClassTag](
seq
}
}
- def integrate(dep: CoGroupSplitDep, op: Product2[K, V] => Unit) = dep match {
+ def integrate(dep: CoGroupSplitDep, op: Product2[K, V] => Unit): Unit = dep match {
case NarrowCoGroupSplitDep(rdd, _, itsSplit) =>
rdd.iterator(itsSplit, context).asInstanceOf[Iterator[Product2[K, V]]].foreach(op)
diff --git a/core/src/main/scala/org/apache/spark/rdd/UnionRDD.scala b/core/src/main/scala/org/apache/spark/rdd/UnionRDD.scala
index aece683ff31996964a416b8c752a431f2fba8120..4239e7e22af89a97bd602db2e31f846c002b046a 100644
--- a/core/src/main/scala/org/apache/spark/rdd/UnionRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/UnionRDD.scala
@@ -44,7 +44,7 @@ private[spark] class UnionPartition[T: ClassTag](
var parentPartition: Partition = rdd.partitions(parentRddPartitionIndex)
- def preferredLocations() = rdd.preferredLocations(parentPartition)
+ def preferredLocations(): Seq[String] = rdd.preferredLocations(parentPartition)
override val index: Int = idx
diff --git a/core/src/main/scala/org/apache/spark/rdd/ZippedPartitionsRDD.scala b/core/src/main/scala/org/apache/spark/rdd/ZippedPartitionsRDD.scala
index 95b2dd954e9f4b88ce52487477b76b1c409653fc..d0be304762e1fc10056eb2cf5fcaae8111f8f5e2 100644
--- a/core/src/main/scala/org/apache/spark/rdd/ZippedPartitionsRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/ZippedPartitionsRDD.scala
@@ -32,7 +32,7 @@ private[spark] class ZippedPartitionsPartition(
override val index: Int = idx
var partitionValues = rdds.map(rdd => rdd.partitions(idx))
- def partitions = partitionValues
+ def partitions: Seq[Partition] = partitionValues
@throws(classOf[IOException])
private def writeObject(oos: ObjectOutputStream): Unit = Utils.tryOrIOException {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/AccumulableInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/AccumulableInfo.scala
index fa83372bb4d118ee04e6004748965fcd01fcfe89..e0edd7d4ae9685f7641b25d57912a0a2e49f52ce 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/AccumulableInfo.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/AccumulableInfo.scala
@@ -39,8 +39,11 @@ class AccumulableInfo (
}
object AccumulableInfo {
- def apply(id: Long, name: String, update: Option[String], value: String) =
+ def apply(id: Long, name: String, update: Option[String], value: String): AccumulableInfo = {
new AccumulableInfo(id, name, update, value)
+ }
- def apply(id: Long, name: String, value: String) = new AccumulableInfo(id, name, None, value)
+ def apply(id: Long, name: String, value: String): AccumulableInfo = {
+ new AccumulableInfo(id, name, None, value)
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
index 8feac6cb6b7a1adaa5bbff379179597b3fee2f0e..b405bd3338e7c505555df4d2548d1dc3e27c12d5 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
@@ -946,7 +946,7 @@ class DAGScheduler(
val stage = stageIdToStage(task.stageId)
- def markStageAsFinished(stage: Stage, errorMessage: Option[String] = None) = {
+ def markStageAsFinished(stage: Stage, errorMessage: Option[String] = None): Unit = {
val serviceTime = stage.latestInfo.submissionTime match {
case Some(t) => "%.03f".format((clock.getTimeMillis() - t) / 1000.0)
case _ => "Unknown"
diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index 34fa6d27c3a45a0b2a5010a743f341d1d5718889..c0d889360ae99cc22505be2016d7f6b0da8afd89 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -149,47 +149,60 @@ private[spark] class EventLoggingListener(
}
// Events that do not trigger a flush
- override def onStageSubmitted(event: SparkListenerStageSubmitted) =
- logEvent(event)
- override def onTaskStart(event: SparkListenerTaskStart) =
- logEvent(event)
- override def onTaskGettingResult(event: SparkListenerTaskGettingResult) =
- logEvent(event)
- override def onTaskEnd(event: SparkListenerTaskEnd) =
- logEvent(event)
- override def onEnvironmentUpdate(event: SparkListenerEnvironmentUpdate) =
- logEvent(event)
+ override def onStageSubmitted(event: SparkListenerStageSubmitted): Unit = logEvent(event)
+
+ override def onTaskStart(event: SparkListenerTaskStart): Unit = logEvent(event)
+
+ override def onTaskGettingResult(event: SparkListenerTaskGettingResult): Unit = logEvent(event)
+
+ override def onTaskEnd(event: SparkListenerTaskEnd): Unit = logEvent(event)
+
+ override def onEnvironmentUpdate(event: SparkListenerEnvironmentUpdate): Unit = logEvent(event)
// Events that trigger a flush
- override def onStageCompleted(event: SparkListenerStageCompleted) =
- logEvent(event, flushLogger = true)
- override def onJobStart(event: SparkListenerJobStart) =
- logEvent(event, flushLogger = true)
- override def onJobEnd(event: SparkListenerJobEnd) =
+ override def onStageCompleted(event: SparkListenerStageCompleted): Unit = {
logEvent(event, flushLogger = true)
- override def onBlockManagerAdded(event: SparkListenerBlockManagerAdded) =
+ }
+
+ override def onJobStart(event: SparkListenerJobStart): Unit = logEvent(event, flushLogger = true)
+
+ override def onJobEnd(event: SparkListenerJobEnd): Unit = logEvent(event, flushLogger = true)
+
+ override def onBlockManagerAdded(event: SparkListenerBlockManagerAdded): Unit = {
logEvent(event, flushLogger = true)
- override def onBlockManagerRemoved(event: SparkListenerBlockManagerRemoved) =
+ }
+
+ override def onBlockManagerRemoved(event: SparkListenerBlockManagerRemoved): Unit = {
logEvent(event, flushLogger = true)
- override def onUnpersistRDD(event: SparkListenerUnpersistRDD) =
+ }
+
+ override def onUnpersistRDD(event: SparkListenerUnpersistRDD): Unit = {
logEvent(event, flushLogger = true)
- override def onApplicationStart(event: SparkListenerApplicationStart) =
+ }
+
+ override def onApplicationStart(event: SparkListenerApplicationStart): Unit = {
logEvent(event, flushLogger = true)
- override def onApplicationEnd(event: SparkListenerApplicationEnd) =
+ }
+
+ override def onApplicationEnd(event: SparkListenerApplicationEnd): Unit = {
logEvent(event, flushLogger = true)
- override def onExecutorAdded(event: SparkListenerExecutorAdded) =
+ }
+ override def onExecutorAdded(event: SparkListenerExecutorAdded): Unit = {
logEvent(event, flushLogger = true)
- override def onExecutorRemoved(event: SparkListenerExecutorRemoved) =
+ }
+
+ override def onExecutorRemoved(event: SparkListenerExecutorRemoved): Unit = {
logEvent(event, flushLogger = true)
+ }
// No-op because logging every update would be overkill
- override def onExecutorMetricsUpdate(event: SparkListenerExecutorMetricsUpdate) { }
+ override def onExecutorMetricsUpdate(event: SparkListenerExecutorMetricsUpdate): Unit = { }
/**
* Stop logging events. The event log file will be renamed so that it loses the
* ".inprogress" suffix.
*/
- def stop() = {
+ def stop(): Unit = {
writer.foreach(_.close())
val target = new Path(logPath)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala b/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala
index 8aa528ac573d0f9ddc45ba809410762cf737e619..e55b76c36cc5f229f666172023ea23771172d84d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala
@@ -57,7 +57,7 @@ class JobLogger(val user: String, val logDirName: String) extends SparkListener
private val stageIdToJobId = new HashMap[Int, Int]
private val jobIdToStageIds = new HashMap[Int, Seq[Int]]
private val dateFormat = new ThreadLocal[SimpleDateFormat]() {
- override def initialValue() = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss")
+ override def initialValue(): SimpleDateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss")
}
createLogDir()
diff --git a/core/src/main/scala/org/apache/spark/scheduler/JobWaiter.scala b/core/src/main/scala/org/apache/spark/scheduler/JobWaiter.scala
index 29879b374b8014fffc167bc9831690a247f59ce9..382b09422a4a0a19efe71a58281c1e0c076661da 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/JobWaiter.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/JobWaiter.scala
@@ -34,7 +34,7 @@ private[spark] class JobWaiter[T](
@volatile
private var _jobFinished = totalTasks == 0
- def jobFinished = _jobFinished
+ def jobFinished: Boolean = _jobFinished
// If the job is finished, this will be its result. In the case of 0 task jobs (e.g. zero
// partition RDDs), we set the jobResult directly to JobSucceeded.
diff --git a/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala b/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
index 759df023a6dcf7f00b746ca0dbd43b6ad582b9a2..a3caa9f000c8946dfd3e62a09ea7b9bfc76ba8c1 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
@@ -160,7 +160,7 @@ private[spark] object OutputCommitCoordinator {
class OutputCommitCoordinatorActor(outputCommitCoordinator: OutputCommitCoordinator)
extends Actor with ActorLogReceive with Logging {
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case AskPermissionToCommitOutput(stage, partition, taskAttempt) =>
sender ! outputCommitCoordinator.handleAskPermissionToCommit(stage, partition, taskAttempt)
case StopCoordinator =>
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
index 4a9ff918afe25e7dcda2618d98a0cd3a50b42da5..e074ce6ebff0bbd697b71a6ec257ab476b65d804 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
@@ -64,5 +64,5 @@ private[spark] class ResultTask[T, U](
// This is only callable on the driver side.
override def preferredLocations: Seq[TaskLocation] = preferredLocs
- override def toString = "ResultTask(" + stageId + ", " + partitionId + ")"
+ override def toString: String = "ResultTask(" + stageId + ", " + partitionId + ")"
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
index 79709089c0da4ff3c702615627708858ee92a0e5..fd0d484b45460e1bb20e958dcc5b7120f41cac82 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
@@ -47,7 +47,7 @@ private[spark] class ShuffleMapTask(
/** A constructor used only in test suites. This does not require passing in an RDD. */
def this(partitionId: Int) {
- this(0, null, new Partition { override def index = 0 }, null)
+ this(0, null, new Partition { override def index: Int = 0 }, null)
}
@transient private val preferredLocs: Seq[TaskLocation] = {
@@ -83,5 +83,5 @@ private[spark] class ShuffleMapTask(
override def preferredLocations: Seq[TaskLocation] = preferredLocs
- override def toString = "ShuffleMapTask(%d, %d)".format(stageId, partitionId)
+ override def toString: String = "ShuffleMapTask(%d, %d)".format(stageId, partitionId)
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
index 52720d48ca67fd72307cab22c6e0b522844d627e..b711ff209af947dac1665ad47de36793a21cd9bc 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
@@ -300,7 +300,7 @@ private[spark] object StatsReportListener extends Logging {
}
def showDistribution(heading: String, dOpt: Option[Distribution], format:String) {
- def f(d: Double) = format.format(d)
+ def f(d: Double): String = format.format(d)
showDistribution(heading, dOpt, f _)
}
@@ -346,7 +346,7 @@ private[spark] object StatsReportListener extends Logging {
/**
* Reformat a time interval in milliseconds to a prettier format for output
*/
- def millisToString(ms: Long) = {
+ def millisToString(ms: Long): String = {
val (size, units) =
if (ms > hours) {
(ms.toDouble / hours, "hours")
diff --git a/core/src/main/scala/org/apache/spark/scheduler/Stage.scala b/core/src/main/scala/org/apache/spark/scheduler/Stage.scala
index cc13f57a49b8963fb368277d2fa19c6e2c2def93..4cbc6e84a6bdd26f6e0923fec6cbb9a63d654d6e 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/Stage.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/Stage.scala
@@ -133,7 +133,7 @@ private[spark] class Stage(
def attemptId: Int = nextAttemptId
- override def toString = "Stage " + id
+ override def toString: String = "Stage " + id
override def hashCode(): Int = id
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskLocation.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskLocation.scala
index 10c685f29d3acbfe57187ebbc655837135537b4b..da07ce2c6ea4904cd6087327293eb58c9637a2ed 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskLocation.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskLocation.scala
@@ -29,23 +29,22 @@ private[spark] sealed trait TaskLocation {
/**
* A location that includes both a host and an executor id on that host.
*/
-private [spark] case class ExecutorCacheTaskLocation(override val host: String,
- val executorId: String) extends TaskLocation {
-}
+private [spark]
+case class ExecutorCacheTaskLocation(override val host: String, executorId: String)
+ extends TaskLocation
/**
* A location on a host.
*/
private [spark] case class HostTaskLocation(override val host: String) extends TaskLocation {
- override def toString = host
+ override def toString: String = host
}
/**
* A location on a host that is cached by HDFS.
*/
-private [spark] case class HDFSCacheTaskLocation(override val host: String)
- extends TaskLocation {
- override def toString = TaskLocation.inMemoryLocationTag + host
+private [spark] case class HDFSCacheTaskLocation(override val host: String) extends TaskLocation {
+ override def toString: String = TaskLocation.inMemoryLocationTag + host
}
private[spark] object TaskLocation {
@@ -54,14 +53,16 @@ private[spark] object TaskLocation {
// confusion. See RFC 952 and RFC 1123 for information about the format of hostnames.
val inMemoryLocationTag = "hdfs_cache_"
- def apply(host: String, executorId: String) = new ExecutorCacheTaskLocation(host, executorId)
+ def apply(host: String, executorId: String): TaskLocation = {
+ new ExecutorCacheTaskLocation(host, executorId)
+ }
/**
* Create a TaskLocation from a string returned by getPreferredLocations.
* These strings have the form [hostname] or hdfs_cache_[hostname], depending on whether the
* location is cached.
*/
- def apply(str: String) = {
+ def apply(str: String): TaskLocation = {
val hstr = str.stripPrefix(inMemoryLocationTag)
if (hstr.equals(str)) {
new HostTaskLocation(str)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index f33fd4450b2a691d9887edeada8f2b54634ab4bb..076b36e86c0ce30923482bf8478aac629990888a 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -373,17 +373,17 @@ private[spark] class TaskSchedulerImpl(
}
def handleSuccessfulTask(
- taskSetManager: TaskSetManager,
- tid: Long,
- taskResult: DirectTaskResult[_]) = synchronized {
+ taskSetManager: TaskSetManager,
+ tid: Long,
+ taskResult: DirectTaskResult[_]): Unit = synchronized {
taskSetManager.handleSuccessfulTask(tid, taskResult)
}
def handleFailedTask(
- taskSetManager: TaskSetManager,
- tid: Long,
- taskState: TaskState,
- reason: TaskEndReason) = synchronized {
+ taskSetManager: TaskSetManager,
+ tid: Long,
+ taskState: TaskState,
+ reason: TaskEndReason): Unit = synchronized {
taskSetManager.handleFailedTask(tid, taskState, reason)
if (!taskSetManager.isZombie && taskState != TaskState.KILLED) {
// Need to revive offers again now that the task set manager state has been updated to
@@ -423,7 +423,7 @@ private[spark] class TaskSchedulerImpl(
starvationTimer.cancel()
}
- override def defaultParallelism() = backend.defaultParallelism()
+ override def defaultParallelism(): Int = backend.defaultParallelism()
// Check for speculatable tasks in all our active jobs.
def checkSpeculatableTasks() {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index 529237f0d35dc20baefda2e93a1907b774504a66..d509881c74feffd4701f4d1106c62f5054d695e5 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -20,6 +20,7 @@ package org.apache.spark.scheduler
import java.io.NotSerializableException
import java.nio.ByteBuffer
import java.util.Arrays
+import java.util.concurrent.ConcurrentLinkedQueue
import scala.collection.mutable.ArrayBuffer
import scala.collection.mutable.HashMap
@@ -29,6 +30,7 @@ import scala.util.control.NonFatal
import org.apache.spark._
import org.apache.spark.executor.TaskMetrics
+import org.apache.spark.scheduler.SchedulingMode._
import org.apache.spark.TaskState.TaskState
import org.apache.spark.util.{Clock, SystemClock, Utils}
@@ -97,7 +99,8 @@ private[spark] class TaskSetManager(
var calculatedTasks = 0
val runningTasksSet = new HashSet[Long]
- override def runningTasks = runningTasksSet.size
+
+ override def runningTasks: Int = runningTasksSet.size
// True once no more tasks should be launched for this task set manager. TaskSetManagers enter
// the zombie state once at least one attempt of each task has completed successfully, or if the
@@ -168,9 +171,9 @@ private[spark] class TaskSetManager(
var currentLocalityIndex = 0 // Index of our current locality level in validLocalityLevels
var lastLaunchTime = clock.getTimeMillis() // Time we last launched a task at this level
- override def schedulableQueue = null
+ override def schedulableQueue: ConcurrentLinkedQueue[Schedulable] = null
- override def schedulingMode = SchedulingMode.NONE
+ override def schedulingMode: SchedulingMode = SchedulingMode.NONE
var emittedTaskSizeWarning = false
@@ -585,7 +588,7 @@ private[spark] class TaskSetManager(
/**
* Marks the task as getting result and notifies the DAG Scheduler
*/
- def handleTaskGettingResult(tid: Long) = {
+ def handleTaskGettingResult(tid: Long): Unit = {
val info = taskInfos(tid)
info.markGettingResult()
sched.dagScheduler.taskGettingResult(info)
@@ -612,7 +615,7 @@ private[spark] class TaskSetManager(
/**
* Marks the task as successful and notifies the DAGScheduler that a task has ended.
*/
- def handleSuccessfulTask(tid: Long, result: DirectTaskResult[_]) = {
+ def handleSuccessfulTask(tid: Long, result: DirectTaskResult[_]): Unit = {
val info = taskInfos(tid)
val index = info.index
info.markSuccessful()
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index 87ebf31139ce9fc2776636d936c6ab74b107d472..5d258d9da4d1a28c46c486625eb8c94ec0157810 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -85,7 +85,7 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val actorSyste
context.system.scheduler.schedule(0.millis, reviveInterval.millis, self, ReviveOffers)
}
- def receiveWithLogging = {
+ def receiveWithLogging: PartialFunction[Any, Unit] = {
case RegisterExecutor(executorId, hostPort, cores, logUrls) =>
Utils.checkHostPort(hostPort, "Host port expected " + hostPort)
if (executorDataMap.contains(executorId)) {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala
index f14aaeea0a25cc7d34b721495fd921a96ca19426..5a38ad9f2b12c8ab7a9dd0de0f029f965de3525d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackend.scala
@@ -109,7 +109,7 @@ private[spark] abstract class YarnSchedulerBackend(
context.system.eventStream.subscribe(self, classOf[RemotingLifecycleEvent])
}
- override def receive = {
+ override def receive: PartialFunction[Any, Unit] = {
case RegisterClusterManager =>
logInfo(s"ApplicationMaster registered as $sender")
amActor = Some(sender)
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MemoryUtils.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MemoryUtils.scala
index aa3ec0f8cfb9c0424b333d9c701d217f00e30130..8df4f3b554c411d65825b7aefd393469a5d0cf6d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MemoryUtils.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MemoryUtils.scala
@@ -24,7 +24,7 @@ private[spark] object MemoryUtils {
val OVERHEAD_FRACTION = 0.10
val OVERHEAD_MINIMUM = 384
- def calculateTotalMemory(sc: SparkContext) = {
+ def calculateTotalMemory(sc: SparkContext): Int = {
sc.conf.getInt("spark.mesos.executor.memoryOverhead",
math.max(OVERHEAD_FRACTION * sc.executorMemory, OVERHEAD_MINIMUM).toInt) + sc.executorMemory
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
index 06bb52752214156bba0ecc530dd136dd2581c5f7..b3814368392272282a094ed4895107e5962cdc79 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
@@ -387,7 +387,7 @@ private[spark] class MesosSchedulerBackend(
}
// TODO: query Mesos for number of cores
- override def defaultParallelism() = sc.conf.getInt("spark.default.parallelism", 8)
+ override def defaultParallelism(): Int = sc.conf.getInt("spark.default.parallelism", 8)
override def applicationId(): String =
Option(appId).getOrElse {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/local/LocalBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/local/LocalBackend.scala
index d95426d918e193ac56ed985dce8941b9e9af157c..eb3f999b5b37527d94ecc6bffb25025d19f400a2 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/local/LocalBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/local/LocalBackend.scala
@@ -59,7 +59,7 @@ private[spark] class LocalActor(
private val executor = new Executor(
localExecutorId, localExecutorHostname, SparkEnv.get, isLocal = true)
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case ReviveOffers =>
reviveOffers()
@@ -117,7 +117,7 @@ private[spark] class LocalBackend(scheduler: TaskSchedulerImpl, val totalCores:
localActor ! ReviveOffers
}
- override def defaultParallelism() =
+ override def defaultParallelism(): Int =
scheduler.conf.getInt("spark.default.parallelism", totalCores)
override def killTask(taskId: Long, executorId: String, interruptThread: Boolean) {
diff --git a/core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala
index 1baa0e009f3ae231f4591c4e8e7b6b26e1235c38..dfbde7c8a1b0d99cb356e56c64183655004ae660 100644
--- a/core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala
@@ -59,9 +59,10 @@ private[spark] class JavaSerializationStream(
}
private[spark] class JavaDeserializationStream(in: InputStream, loader: ClassLoader)
-extends DeserializationStream {
+ extends DeserializationStream {
+
private val objIn = new ObjectInputStream(in) {
- override def resolveClass(desc: ObjectStreamClass) =
+ override def resolveClass(desc: ObjectStreamClass): Class[_] =
Class.forName(desc.getName, false, loader)
}
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index dc7aa99738c17cb04467717dfac90d4209c86f25..f83bcaa5cc09e15d60aa88a5cae0919922a79ca3 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -60,7 +60,7 @@ class KryoSerializer(conf: SparkConf)
.split(',')
.filter(!_.isEmpty)
- def newKryoOutput() = new KryoOutput(bufferSize, math.max(bufferSize, maxBufferSize))
+ def newKryoOutput(): KryoOutput = new KryoOutput(bufferSize, math.max(bufferSize, maxBufferSize))
def newKryo(): Kryo = {
val instantiator = new EmptyScalaKryoInstantiator
diff --git a/core/src/main/scala/org/apache/spark/shuffle/FileShuffleBlockManager.scala b/core/src/main/scala/org/apache/spark/shuffle/FileShuffleBlockManager.scala
index 7de2f9cbb2866acf954f070a50de1e09684a53d8..660df00bc32f53ee18b09bc02a164799eab00859 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/FileShuffleBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/FileShuffleBlockManager.scala
@@ -106,7 +106,7 @@ class FileShuffleBlockManager(conf: SparkConf)
* when the writers are closed successfully
*/
def forMapTask(shuffleId: Int, mapId: Int, numBuckets: Int, serializer: Serializer,
- writeMetrics: ShuffleWriteMetrics) = {
+ writeMetrics: ShuffleWriteMetrics): ShuffleWriterGroup = {
new ShuffleWriterGroup {
shuffleStates.putIfAbsent(shuffleId, new ShuffleState(numBuckets))
private val shuffleState = shuffleStates(shuffleId)
@@ -268,7 +268,7 @@ object FileShuffleBlockManager {
new PrimitiveVector[Long]()
}
- def apply(bucketId: Int) = files(bucketId)
+ def apply(bucketId: Int): File = files(bucketId)
def recordMapOutput(mapId: Int, offsets: Array[Long], lengths: Array[Long]) {
assert(offsets.length == lengths.length)
diff --git a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockManager.scala b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockManager.scala
index b292587d37028eb7b7dacabf3becbbb81dd9f054..87fd161e06c8567498956e6670853239c04214c9 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockManager.scala
@@ -80,7 +80,7 @@ class IndexShuffleBlockManager(conf: SparkConf) extends ShuffleBlockManager {
* end of the output file. This will be used by getBlockLocation to figure out where each block
* begins and ends.
* */
- def writeIndexFile(shuffleId: Int, mapId: Int, lengths: Array[Long]) = {
+ def writeIndexFile(shuffleId: Int, mapId: Int, lengths: Array[Long]): Unit = {
val indexFile = getIndexFile(shuffleId, mapId)
val out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(indexFile)))
try {
@@ -121,5 +121,5 @@ class IndexShuffleBlockManager(conf: SparkConf) extends ShuffleBlockManager {
}
}
- override def stop() = {}
+ override def stop(): Unit = {}
}
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockId.scala b/core/src/main/scala/org/apache/spark/storage/BlockId.scala
index 1f012941c85ab195b225e7c29467c0242a4c54e0..c186fd360fef69c272e1542daf364ecc53e90bb1 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockId.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockId.scala
@@ -35,13 +35,13 @@ sealed abstract class BlockId {
def name: String
// convenience methods
- def asRDDId = if (isRDD) Some(asInstanceOf[RDDBlockId]) else None
- def isRDD = isInstanceOf[RDDBlockId]
- def isShuffle = isInstanceOf[ShuffleBlockId]
- def isBroadcast = isInstanceOf[BroadcastBlockId]
+ def asRDDId: Option[RDDBlockId] = if (isRDD) Some(asInstanceOf[RDDBlockId]) else None
+ def isRDD: Boolean = isInstanceOf[RDDBlockId]
+ def isShuffle: Boolean = isInstanceOf[ShuffleBlockId]
+ def isBroadcast: Boolean = isInstanceOf[BroadcastBlockId]
- override def toString = name
- override def hashCode = name.hashCode
+ override def toString: String = name
+ override def hashCode: Int = name.hashCode
override def equals(other: Any): Boolean = other match {
case o: BlockId => getClass == o.getClass && name.equals(o.name)
case _ => false
@@ -50,54 +50,54 @@ sealed abstract class BlockId {
@DeveloperApi
case class RDDBlockId(rddId: Int, splitIndex: Int) extends BlockId {
- def name = "rdd_" + rddId + "_" + splitIndex
+ override def name: String = "rdd_" + rddId + "_" + splitIndex
}
// Format of the shuffle block ids (including data and index) should be kept in sync with
// org.apache.spark.network.shuffle.StandaloneShuffleBlockManager#getBlockData().
@DeveloperApi
case class ShuffleBlockId(shuffleId: Int, mapId: Int, reduceId: Int) extends BlockId {
- def name = "shuffle_" + shuffleId + "_" + mapId + "_" + reduceId
+ override def name: String = "shuffle_" + shuffleId + "_" + mapId + "_" + reduceId
}
@DeveloperApi
case class ShuffleDataBlockId(shuffleId: Int, mapId: Int, reduceId: Int) extends BlockId {
- def name = "shuffle_" + shuffleId + "_" + mapId + "_" + reduceId + ".data"
+ override def name: String = "shuffle_" + shuffleId + "_" + mapId + "_" + reduceId + ".data"
}
@DeveloperApi
case class ShuffleIndexBlockId(shuffleId: Int, mapId: Int, reduceId: Int) extends BlockId {
- def name = "shuffle_" + shuffleId + "_" + mapId + "_" + reduceId + ".index"
+ override def name: String = "shuffle_" + shuffleId + "_" + mapId + "_" + reduceId + ".index"
}
@DeveloperApi
case class BroadcastBlockId(broadcastId: Long, field: String = "") extends BlockId {
- def name = "broadcast_" + broadcastId + (if (field == "") "" else "_" + field)
+ override def name: String = "broadcast_" + broadcastId + (if (field == "") "" else "_" + field)
}
@DeveloperApi
case class TaskResultBlockId(taskId: Long) extends BlockId {
- def name = "taskresult_" + taskId
+ override def name: String = "taskresult_" + taskId
}
@DeveloperApi
case class StreamBlockId(streamId: Int, uniqueId: Long) extends BlockId {
- def name = "input-" + streamId + "-" + uniqueId
+ override def name: String = "input-" + streamId + "-" + uniqueId
}
/** Id associated with temporary local data managed as blocks. Not serializable. */
private[spark] case class TempLocalBlockId(id: UUID) extends BlockId {
- def name = "temp_local_" + id
+ override def name: String = "temp_local_" + id
}
/** Id associated with temporary shuffle data managed as blocks. Not serializable. */
private[spark] case class TempShuffleBlockId(id: UUID) extends BlockId {
- def name = "temp_shuffle_" + id
+ override def name: String = "temp_shuffle_" + id
}
// Intended only for testing purposes
private[spark] case class TestBlockId(id: String) extends BlockId {
- def name = "test_" + id
+ override def name: String = "test_" + id
}
@DeveloperApi
@@ -112,7 +112,7 @@ object BlockId {
val TEST = "test_(.*)".r
/** Converts a BlockId "name" String back into a BlockId. */
- def apply(id: String) = id match {
+ def apply(id: String): BlockId = id match {
case RDD(rddId, splitIndex) =>
RDDBlockId(rddId.toInt, splitIndex.toInt)
case SHUFFLE(shuffleId, mapId, reduceId) =>
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala
index b177a59c721df1797bc05809f5f5969f31ee323f..a6f1ebf325a7cb09fb4d6658fa72ec273169bda1 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerId.scala
@@ -77,11 +77,11 @@ class BlockManagerId private (
@throws(classOf[IOException])
private def readResolve(): Object = BlockManagerId.getCachedBlockManagerId(this)
- override def toString = s"BlockManagerId($executorId, $host, $port)"
+ override def toString: String = s"BlockManagerId($executorId, $host, $port)"
override def hashCode: Int = (executorId.hashCode * 41 + host.hashCode) * 41 + port
- override def equals(that: Any) = that match {
+ override def equals(that: Any): Boolean = that match {
case id: BlockManagerId =>
executorId == id.executorId && port == id.port && host == id.host
case _ =>
@@ -100,10 +100,10 @@ private[spark] object BlockManagerId {
* @param port Port of the block manager.
* @return A new [[org.apache.spark.storage.BlockManagerId]].
*/
- def apply(execId: String, host: String, port: Int) =
+ def apply(execId: String, host: String, port: Int): BlockManagerId =
getCachedBlockManagerId(new BlockManagerId(execId, host, port))
- def apply(in: ObjectInput) = {
+ def apply(in: ObjectInput): BlockManagerId = {
val obj = new BlockManagerId()
obj.readExternal(in)
getCachedBlockManagerId(obj)
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
index 654796f23c96e7c4c425208eefc734581739e21b..061964826f08beee41132ee11dd7cea19c4618fa 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
@@ -79,7 +79,7 @@ class BlockManagerMaster(
* Check if block manager master has a block. Note that this can be used to check for only
* those blocks that are reported to block manager master.
*/
- def contains(blockId: BlockId) = {
+ def contains(blockId: BlockId): Boolean = {
!getLocations(blockId).isEmpty
}
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterActor.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterActor.scala
index 787b0f96bec32954db305676f1b83b08e620b79e..5b5328016124eed8f42a17624edd8bde02ed74a6 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterActor.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterActor.scala
@@ -52,7 +52,7 @@ class BlockManagerMasterActor(val isLocal: Boolean, conf: SparkConf, listenerBus
private val akkaTimeout = AkkaUtils.askTimeout(conf)
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case RegisterBlockManager(blockManagerId, maxMemSize, slaveActor) =>
register(blockManagerId, maxMemSize, slaveActor)
sender ! true
@@ -421,7 +421,7 @@ private[spark] class BlockManagerInfo(
// Mapping from block id to its status.
private val _blocks = new JHashMap[BlockId, BlockStatus]
- def getStatus(blockId: BlockId) = Option(_blocks.get(blockId))
+ def getStatus(blockId: BlockId): Option[BlockStatus] = Option(_blocks.get(blockId))
def updateLastSeenMs() {
_lastSeenMs = System.currentTimeMillis()
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerSlaveActor.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerSlaveActor.scala
index 8462871e798a5f95ca966a324bc18d5b68d4c29f..52fb896c4e21f654c2a5f70f643b935a69559a77 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerSlaveActor.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerSlaveActor.scala
@@ -38,7 +38,7 @@ class BlockManagerSlaveActor(
import context.dispatcher
// Operations that involve removing blocks may be slow and should be done asynchronously
- override def receiveWithLogging = {
+ override def receiveWithLogging: PartialFunction[Any, Unit] = {
case RemoveBlock(blockId) =>
doAsync[Boolean]("removing block " + blockId, sender) {
blockManager.removeBlock(blockId)
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala b/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala
index 81164178b9e8ee235e291bf3a18a252a27dfbdc4..f703e50b6b0ace95cc1a914e62f783f549aa1962 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala
@@ -82,11 +82,13 @@ private[spark] class DiskBlockObjectWriter(
{
/** Intercepts write calls and tracks total time spent writing. Not thread safe. */
private class TimeTrackingOutputStream(out: OutputStream) extends OutputStream {
- def write(i: Int): Unit = callWithTiming(out.write(i))
- override def write(b: Array[Byte]) = callWithTiming(out.write(b))
- override def write(b: Array[Byte], off: Int, len: Int) = callWithTiming(out.write(b, off, len))
- override def close() = out.close()
- override def flush() = out.flush()
+ override def write(i: Int): Unit = callWithTiming(out.write(i))
+ override def write(b: Array[Byte]): Unit = callWithTiming(out.write(b))
+ override def write(b: Array[Byte], off: Int, len: Int): Unit = {
+ callWithTiming(out.write(b, off, len))
+ }
+ override def close(): Unit = out.close()
+ override def flush(): Unit = out.flush()
}
/** The file channel, used for repositioning / truncating the file. */
@@ -141,8 +143,9 @@ private[spark] class DiskBlockObjectWriter(
if (syncWrites) {
// Force outstanding writes to disk and track how long it takes
objOut.flush()
- def sync = fos.getFD.sync()
- callWithTiming(sync)
+ callWithTiming {
+ fos.getFD.sync()
+ }
}
objOut.close()
diff --git a/core/src/main/scala/org/apache/spark/storage/FileSegment.scala b/core/src/main/scala/org/apache/spark/storage/FileSegment.scala
index 132502b75f8cd060da712348cd719912756f3e8c..95e2d688d9b178a49a2ebfc7e11e227d1ba98b1c 100644
--- a/core/src/main/scala/org/apache/spark/storage/FileSegment.scala
+++ b/core/src/main/scala/org/apache/spark/storage/FileSegment.scala
@@ -24,5 +24,7 @@ import java.io.File
* based off an offset and a length.
*/
private[spark] class FileSegment(val file: File, val offset: Long, val length: Long) {
- override def toString = "(name=%s, offset=%d, length=%d)".format(file.getName, offset, length)
+ override def toString: String = {
+ "(name=%s, offset=%d, length=%d)".format(file.getName, offset, length)
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
index 120c327a7e580cc81fb3ba8315783200d02e94f9..0186eb30a19056fcd6d729edf961ed37dd7a2b4a 100644
--- a/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
+++ b/core/src/main/scala/org/apache/spark/storage/RDDInfo.scala
@@ -36,7 +36,7 @@ class RDDInfo(
def isCached: Boolean = (memSize + diskSize + tachyonSize > 0) && numCachedPartitions > 0
- override def toString = {
+ override def toString: String = {
import Utils.bytesToString
("RDD \"%s\" (%d) StorageLevel: %s; CachedPartitions: %d; TotalPartitions: %d; " +
"MemorySize: %s; TachyonSize: %s; DiskSize: %s").format(
@@ -44,7 +44,7 @@ class RDDInfo(
bytesToString(memSize), bytesToString(tachyonSize), bytesToString(diskSize))
}
- override def compare(that: RDDInfo) = {
+ override def compare(that: RDDInfo): Int = {
this.id - that.id
}
}
diff --git a/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala b/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala
index e5e1cf5a69a1912ff59aef47a6bbafa9d05c3a44..134abea86621871f0a159a6691d5c87516881c72 100644
--- a/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala
+++ b/core/src/main/scala/org/apache/spark/storage/StorageLevel.scala
@@ -50,11 +50,11 @@ class StorageLevel private(
def this() = this(false, true, false, false) // For deserialization
- def useDisk = _useDisk
- def useMemory = _useMemory
- def useOffHeap = _useOffHeap
- def deserialized = _deserialized
- def replication = _replication
+ def useDisk: Boolean = _useDisk
+ def useMemory: Boolean = _useMemory
+ def useOffHeap: Boolean = _useOffHeap
+ def deserialized: Boolean = _deserialized
+ def replication: Int = _replication
assert(replication < 40, "Replication restricted to be less than 40 for calculating hash codes")
@@ -80,7 +80,7 @@ class StorageLevel private(
false
}
- def isValid = (useMemory || useDisk || useOffHeap) && (replication > 0)
+ def isValid: Boolean = (useMemory || useDisk || useOffHeap) && (replication > 0)
def toInt: Int = {
var ret = 0
@@ -183,7 +183,7 @@ object StorageLevel {
useMemory: Boolean,
useOffHeap: Boolean,
deserialized: Boolean,
- replication: Int) = {
+ replication: Int): StorageLevel = {
getCachedStorageLevel(
new StorageLevel(useDisk, useMemory, useOffHeap, deserialized, replication))
}
@@ -197,7 +197,7 @@ object StorageLevel {
useDisk: Boolean,
useMemory: Boolean,
deserialized: Boolean,
- replication: Int = 1) = {
+ replication: Int = 1): StorageLevel = {
getCachedStorageLevel(new StorageLevel(useDisk, useMemory, false, deserialized, replication))
}
diff --git a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala b/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala
index def49e80a3605003b356906c2339483019e06fa2..7d75929b96f75d0cceef805a5c8ae633dd1adf46 100644
--- a/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/storage/StorageStatusListener.scala
@@ -19,7 +19,6 @@ package org.apache.spark.storage
import scala.collection.mutable
-import org.apache.spark.SparkContext
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.scheduler._
@@ -32,7 +31,7 @@ class StorageStatusListener extends SparkListener {
// This maintains only blocks that are cached (i.e. storage level is not StorageLevel.NONE)
private[storage] val executorIdToStorageStatus = mutable.Map[String, StorageStatus]()
- def storageStatusList = executorIdToStorageStatus.values.toSeq
+ def storageStatusList: Seq[StorageStatus] = executorIdToStorageStatus.values.toSeq
/** Update storage status list to reflect updated block statuses */
private def updateStorageStatus(execId: String, updatedBlocks: Seq[(BlockId, BlockStatus)]) {
@@ -56,7 +55,7 @@ class StorageStatusListener extends SparkListener {
}
}
- override def onTaskEnd(taskEnd: SparkListenerTaskEnd) = synchronized {
+ override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = synchronized {
val info = taskEnd.taskInfo
val metrics = taskEnd.taskMetrics
if (info != null && metrics != null) {
@@ -67,7 +66,7 @@ class StorageStatusListener extends SparkListener {
}
}
- override def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD) = synchronized {
+ override def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit = synchronized {
updateStorageStatus(unpersistRDD.rddId)
}
diff --git a/core/src/main/scala/org/apache/spark/storage/TachyonFileSegment.scala b/core/src/main/scala/org/apache/spark/storage/TachyonFileSegment.scala
index b86abbda1d3e74c63574e55afde80cbe4d93ee6b..65fa81704c3658a7537d54e7d383559d91bc99cd 100644
--- a/core/src/main/scala/org/apache/spark/storage/TachyonFileSegment.scala
+++ b/core/src/main/scala/org/apache/spark/storage/TachyonFileSegment.scala
@@ -24,5 +24,7 @@ import tachyon.client.TachyonFile
* a length.
*/
private[spark] class TachyonFileSegment(val file: TachyonFile, val offset: Long, val length: Long) {
- override def toString = "(name=%s, offset=%d, length=%d)".format(file.getPath(), offset, length)
+ override def toString: String = {
+ "(name=%s, offset=%d, length=%d)".format(file.getPath(), offset, length)
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index 0c24ad2760e085d70f5ed5447db39eba678c342d..adfa6bbada256d4a0b7935193a62ed537891d48f 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -60,7 +60,7 @@ private[spark] class SparkUI private (
}
initialize()
- def getAppName = appName
+ def getAppName: String = appName
/** Set the app name for this UI. */
def setAppName(name: String) {
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index b5022fe853c496aed4e7308901cef7bb97b49e4e..f07864141a21cfcbdd6506aa467bbced5eb09a78 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -149,9 +149,11 @@ private[spark] object UIUtils extends Logging {
}
}
- def prependBaseUri(basePath: String = "", resource: String = "") = uiRoot + basePath + resource
+ def prependBaseUri(basePath: String = "", resource: String = ""): String = {
+ uiRoot + basePath + resource
+ }
- def commonHeaderNodes = {
+ def commonHeaderNodes: Seq[Node] = {
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
<link rel="stylesheet" href={prependBaseUri("/static/bootstrap.min.css")}
type="text/css" />
diff --git a/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala b/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala
index fc1844600f1cbe72f22fc44a217b0cb9ea110413..19ac7a826e306116fe7c757ce90c68bf29488bc8 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala
@@ -51,7 +51,7 @@ private[spark] object UIWorkloadGenerator {
val nJobSet = args(2).toInt
val sc = new SparkContext(conf)
- def setProperties(s: String) = {
+ def setProperties(s: String): Unit = {
if(schedulingMode == SchedulingMode.FAIR) {
sc.setLocalProperty("spark.scheduler.pool", s)
}
@@ -59,7 +59,7 @@ private[spark] object UIWorkloadGenerator {
}
val baseData = sc.makeRDD(1 to NUM_PARTITIONS * 10, NUM_PARTITIONS)
- def nextFloat() = new Random().nextFloat()
+ def nextFloat(): Float = new Random().nextFloat()
val jobs = Seq[(String, () => Long)](
("Count", baseData.count),
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
index 3afd7ef07d7c94b68fb533d3cf5afd1e79edac3c..69053fe44d7e48e2f488eccddd625a8d1ba6ce3f 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
@@ -22,7 +22,7 @@ import scala.collection.mutable.HashMap
import org.apache.spark.ExceptionFailure
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.scheduler._
-import org.apache.spark.storage.StorageStatusListener
+import org.apache.spark.storage.{StorageStatus, StorageStatusListener}
import org.apache.spark.ui.{SparkUI, SparkUITab}
private[ui] class ExecutorsTab(parent: SparkUI) extends SparkUITab(parent, "executors") {
@@ -55,19 +55,19 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener) extends Sp
val executorToShuffleWrite = HashMap[String, Long]()
val executorToLogUrls = HashMap[String, Map[String, String]]()
- def storageStatusList = storageStatusListener.storageStatusList
+ def storageStatusList: Seq[StorageStatus] = storageStatusListener.storageStatusList
- override def onExecutorAdded(executorAdded: SparkListenerExecutorAdded) = synchronized {
+ override def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit = synchronized {
val eid = executorAdded.executorId
executorToLogUrls(eid) = executorAdded.executorInfo.logUrlMap
}
- override def onTaskStart(taskStart: SparkListenerTaskStart) = synchronized {
+ override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = synchronized {
val eid = taskStart.taskInfo.executorId
executorToTasksActive(eid) = executorToTasksActive.getOrElse(eid, 0) + 1
}
- override def onTaskEnd(taskEnd: SparkListenerTaskEnd) = synchronized {
+ override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = synchronized {
val info = taskEnd.taskInfo
if (info != null) {
val eid = info.executorId
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
index 937d95a934b590ded3d5eeb122a33897a0dbcd93..949e80d30f5eb608a922f23e1e8664a9ccf2990b 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
@@ -73,7 +73,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
// Misc:
val executorIdToBlockManagerId = HashMap[ExecutorId, BlockManagerId]()
- def blockManagerIds = executorIdToBlockManagerId.values.toSeq
+ def blockManagerIds: Seq[BlockManagerId] = executorIdToBlockManagerId.values.toSeq
var schedulingMode: Option[SchedulingMode] = None
@@ -146,7 +146,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
}
}
- override def onJobStart(jobStart: SparkListenerJobStart) = synchronized {
+ override def onJobStart(jobStart: SparkListenerJobStart): Unit = synchronized {
val jobGroup = for (
props <- Option(jobStart.properties);
group <- Option(props.getProperty(SparkContext.SPARK_JOB_GROUP_ID))
@@ -182,7 +182,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
}
}
- override def onJobEnd(jobEnd: SparkListenerJobEnd) = synchronized {
+ override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = synchronized {
val jobData = activeJobs.remove(jobEnd.jobId).getOrElse {
logWarning(s"Job completed for unknown job ${jobEnd.jobId}")
new JobUIData(jobId = jobEnd.jobId)
@@ -219,7 +219,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
}
}
- override def onStageCompleted(stageCompleted: SparkListenerStageCompleted) = synchronized {
+ override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = synchronized {
val stage = stageCompleted.stageInfo
stageIdToInfo(stage.stageId) = stage
val stageData = stageIdToData.getOrElseUpdate((stage.stageId, stage.attemptId), {
@@ -260,7 +260,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
}
/** For FIFO, all stages are contained by "default" pool but "default" pool here is meaningless */
- override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted) = synchronized {
+ override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = synchronized {
val stage = stageSubmitted.stageInfo
activeStages(stage.stageId) = stage
pendingStages.remove(stage.stageId)
@@ -288,7 +288,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
}
}
- override def onTaskStart(taskStart: SparkListenerTaskStart) = synchronized {
+ override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = synchronized {
val taskInfo = taskStart.taskInfo
if (taskInfo != null) {
val stageData = stageIdToData.getOrElseUpdate((taskStart.stageId, taskStart.stageAttemptId), {
@@ -312,7 +312,7 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
// stageToTaskInfos already has the updated status.
}
- override def onTaskEnd(taskEnd: SparkListenerTaskEnd) = synchronized {
+ override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = synchronized {
val info = taskEnd.taskInfo
// If stage attempt id is -1, it means the DAGScheduler had no idea which attempt this task
// completion event is for. Let's just drop it here. This means we might have some speculation
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala
index b2bbfdee569460c62306cbd8c5feb5f422d2e34e..7ffcf291b5cc670df7f3f61144dcc6829c083b92 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala
@@ -24,7 +24,7 @@ import org.apache.spark.ui.{SparkUI, SparkUITab}
private[ui] class JobsTab(parent: SparkUI) extends SparkUITab(parent, "jobs") {
val sc = parent.sc
val killEnabled = parent.killEnabled
- def isFairScheduler = listener.schedulingMode.exists(_ == SchedulingMode.FAIR)
+ def isFairScheduler: Boolean = listener.schedulingMode.exists(_ == SchedulingMode.FAIR)
val listener = parent.jobProgressListener
attachPage(new AllJobsPage(this))
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 110f8780a9a1275610459efd75365b9f66fafee8..e03442894c5cc60f5efc12e22a3eafa709abfd3d 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -20,7 +20,7 @@ package org.apache.spark.ui.jobs
import java.util.Date
import javax.servlet.http.HttpServletRequest
-import scala.xml.{Node, Unparsed}
+import scala.xml.{Elem, Node, Unparsed}
import org.apache.commons.lang3.StringEscapeUtils
@@ -170,7 +170,8 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
</div>
val accumulableHeaders: Seq[String] = Seq("Accumulable", "Value")
- def accumulableRow(acc: AccumulableInfo) = <tr><td>{acc.name}</td><td>{acc.value}</td></tr>
+ def accumulableRow(acc: AccumulableInfo): Elem =
+ <tr><td>{acc.name}</td><td>{acc.value}</td></tr>
val accumulableTable = UIUtils.listingTable(accumulableHeaders, accumulableRow,
accumulables.values.toSeq)
@@ -293,10 +294,11 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
val schedulerDelayQuantiles = schedulerDelayTitle +:
getFormattedTimeQuantiles(schedulerDelays)
- def getFormattedSizeQuantiles(data: Seq[Double]) =
+ def getFormattedSizeQuantiles(data: Seq[Double]): Seq[Elem] =
getDistributionQuantiles(data).map(d => <td>{Utils.bytesToString(d.toLong)}</td>)
- def getFormattedSizeQuantilesWithRecords(data: Seq[Double], records: Seq[Double]) = {
+ def getFormattedSizeQuantilesWithRecords(data: Seq[Double], records: Seq[Double])
+ : Seq[Elem] = {
val recordDist = getDistributionQuantiles(records).iterator
getDistributionQuantiles(data).map(d =>
<td>{s"${Utils.bytesToString(d.toLong)} / ${recordDist.next().toLong}"}</td>
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagesTab.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagesTab.scala
index 937261de00e3ad4a193ca98585562ba87c09f341..1bd2d87e007969df0eaa3aafb406f9f7c7a05d3a 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagesTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagesTab.scala
@@ -32,10 +32,10 @@ private[ui] class StagesTab(parent: SparkUI) extends SparkUITab(parent, "stages"
attachPage(new StagePage(this))
attachPage(new PoolPage(this))
- def isFairScheduler = listener.schedulingMode.exists(_ == SchedulingMode.FAIR)
+ def isFairScheduler: Boolean = listener.schedulingMode.exists(_ == SchedulingMode.FAIR)
- def handleKillRequest(request: HttpServletRequest) = {
- if ((killEnabled) && (parent.securityManager.checkModifyPermissions(request.getRemoteUser))) {
+ def handleKillRequest(request: HttpServletRequest): Unit = {
+ if (killEnabled && parent.securityManager.checkModifyPermissions(request.getRemoteUser)) {
val killFlag = Option(request.getParameter("terminate")).getOrElse("false").toBoolean
val stageId = Option(request.getParameter("id")).getOrElse("-1").toInt
if (stageId >= 0 && killFlag && listener.activeStages.contains(stageId)) {
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
index dbf1ceeda187895255e3548ab9326d8638b93541..711a3697bda15d3016ab4e0688c7e647ae0df53a 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
@@ -94,11 +94,11 @@ private[jobs] object UIData {
var taskData = new HashMap[Long, TaskUIData]
var executorSummary = new HashMap[String, ExecutorSummary]
- def hasInput = inputBytes > 0
- def hasOutput = outputBytes > 0
- def hasShuffleRead = shuffleReadTotalBytes > 0
- def hasShuffleWrite = shuffleWriteBytes > 0
- def hasBytesSpilled = memoryBytesSpilled > 0 && diskBytesSpilled > 0
+ def hasInput: Boolean = inputBytes > 0
+ def hasOutput: Boolean = outputBytes > 0
+ def hasShuffleRead: Boolean = shuffleReadTotalBytes > 0
+ def hasShuffleWrite: Boolean = shuffleWriteBytes > 0
+ def hasBytesSpilled: Boolean = memoryBytesSpilled > 0 && diskBytesSpilled > 0
}
/**
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala
index a81291d50558390c51dcacb50eafbf0dfddd7503..045bd784990d152d18ae1ddda4bfcd7260e88266 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala
@@ -40,10 +40,10 @@ private[ui] class StorageTab(parent: SparkUI) extends SparkUITab(parent, "storag
class StorageListener(storageStatusListener: StorageStatusListener) extends SparkListener {
private[ui] val _rddInfoMap = mutable.Map[Int, RDDInfo]() // exposed for testing
- def storageStatusList = storageStatusListener.storageStatusList
+ def storageStatusList: Seq[StorageStatus] = storageStatusListener.storageStatusList
/** Filter RDD info to include only those with cached partitions */
- def rddInfoList = _rddInfoMap.values.filter(_.numCachedPartitions > 0).toSeq
+ def rddInfoList: Seq[RDDInfo] = _rddInfoMap.values.filter(_.numCachedPartitions > 0).toSeq
/** Update the storage info of the RDDs whose blocks are among the given updated blocks */
private def updateRDDInfo(updatedBlocks: Seq[(BlockId, BlockStatus)]): Unit = {
@@ -56,19 +56,19 @@ class StorageListener(storageStatusListener: StorageStatusListener) extends Spar
* Assumes the storage status list is fully up-to-date. This implies the corresponding
* StorageStatusSparkListener must process the SparkListenerTaskEnd event before this listener.
*/
- override def onTaskEnd(taskEnd: SparkListenerTaskEnd) = synchronized {
+ override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = synchronized {
val metrics = taskEnd.taskMetrics
if (metrics != null && metrics.updatedBlocks.isDefined) {
updateRDDInfo(metrics.updatedBlocks.get)
}
}
- override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted) = synchronized {
+ override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = synchronized {
val rddInfos = stageSubmitted.stageInfo.rddInfos
rddInfos.foreach { info => _rddInfoMap.getOrElseUpdate(info.id, info) }
}
- override def onStageCompleted(stageCompleted: SparkListenerStageCompleted) = synchronized {
+ override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = synchronized {
// Remove all partitions that are no longer cached in current completed stage
val completedRddIds = stageCompleted.stageInfo.rddInfos.map(r => r.id).toSet
_rddInfoMap.retain { case (id, info) =>
@@ -76,7 +76,7 @@ class StorageListener(storageStatusListener: StorageStatusListener) extends Spar
}
}
- override def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD) = synchronized {
+ override def onUnpersistRDD(unpersistRDD: SparkListenerUnpersistRDD): Unit = synchronized {
_rddInfoMap.remove(unpersistRDD.rddId)
}
}
diff --git a/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala b/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala
index 390310243ee0a02536780e8fbc366450b9808e57..9044aaeef2d489eab88349f3542310ea2cb9f05e 100644
--- a/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala
+++ b/core/src/main/scala/org/apache/spark/util/CompletionIterator.scala
@@ -27,8 +27,8 @@ abstract class CompletionIterator[ +A, +I <: Iterator[A]](sub: I) extends Iterat
// scalastyle:on
private[this] var completed = false
- def next() = sub.next()
- def hasNext = {
+ def next(): A = sub.next()
+ def hasNext: Boolean = {
val r = sub.hasNext
if (!r && !completed) {
completed = true
@@ -37,13 +37,13 @@ abstract class CompletionIterator[ +A, +I <: Iterator[A]](sub: I) extends Iterat
r
}
- def completion()
+ def completion(): Unit
}
private[spark] object CompletionIterator {
- def apply[A, I <: Iterator[A]](sub: I, completionFunction: => Unit) : CompletionIterator[A,I] = {
+ def apply[A, I <: Iterator[A]](sub: I, completionFunction: => Unit) : CompletionIterator[A, I] = {
new CompletionIterator[A,I](sub) {
- def completion() = completionFunction
+ def completion(): Unit = completionFunction
}
}
}
diff --git a/core/src/main/scala/org/apache/spark/util/Distribution.scala b/core/src/main/scala/org/apache/spark/util/Distribution.scala
index a465298c8c5ab927c1719a2115348fd9fb5e0eda..9aea8efa38c7a189a58339fcc996bccc85d862f7 100644
--- a/core/src/main/scala/org/apache/spark/util/Distribution.scala
+++ b/core/src/main/scala/org/apache/spark/util/Distribution.scala
@@ -57,7 +57,7 @@ private[spark] class Distribution(val data: Array[Double], val startIdx: Int, va
out.println
}
- def statCounter = StatCounter(data.slice(startIdx, endIdx))
+ def statCounter: StatCounter = StatCounter(data.slice(startIdx, endIdx))
/**
* print a summary of this distribution to the given PrintStream.
diff --git a/core/src/main/scala/org/apache/spark/util/ManualClock.scala b/core/src/main/scala/org/apache/spark/util/ManualClock.scala
index cf89c1782fd67f4e3cbe46ceb9a83ae33b093fd3..1718554061985110b98db5a395a029a853dc5fdc 100644
--- a/core/src/main/scala/org/apache/spark/util/ManualClock.scala
+++ b/core/src/main/scala/org/apache/spark/util/ManualClock.scala
@@ -39,31 +39,27 @@ private[spark] class ManualClock(private var time: Long) extends Clock {
/**
* @param timeToSet new time (in milliseconds) that the clock should represent
*/
- def setTime(timeToSet: Long) =
- synchronized {
- time = timeToSet
- notifyAll()
- }
+ def setTime(timeToSet: Long): Unit = synchronized {
+ time = timeToSet
+ notifyAll()
+ }
/**
* @param timeToAdd time (in milliseconds) to add to the clock's time
*/
- def advance(timeToAdd: Long) =
- synchronized {
- time += timeToAdd
- notifyAll()
- }
+ def advance(timeToAdd: Long): Unit = synchronized {
+ time += timeToAdd
+ notifyAll()
+ }
/**
* @param targetTime block until the clock time is set or advanced to at least this time
* @return current time reported by the clock when waiting finishes
*/
- def waitTillTime(targetTime: Long): Long =
- synchronized {
- while (time < targetTime) {
- wait(100)
- }
- getTimeMillis()
+ def waitTillTime(targetTime: Long): Long = synchronized {
+ while (time < targetTime) {
+ wait(100)
}
-
+ getTimeMillis()
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/util/MetadataCleaner.scala b/core/src/main/scala/org/apache/spark/util/MetadataCleaner.scala
index ac40f19ed679954a947087f5102db04b2d701e34..375ed430bde45e443409f1c651f7f32a9c2ec598 100644
--- a/core/src/main/scala/org/apache/spark/util/MetadataCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/util/MetadataCleaner.scala
@@ -67,14 +67,15 @@ private[spark] object MetadataCleanerType extends Enumeration {
type MetadataCleanerType = Value
- def systemProperty(which: MetadataCleanerType.MetadataCleanerType) =
- "spark.cleaner.ttl." + which.toString
+ def systemProperty(which: MetadataCleanerType.MetadataCleanerType): String = {
+ "spark.cleaner.ttl." + which.toString
+ }
}
// TODO: This mutates a Conf to set properties right now, which is kind of ugly when used in the
// initialization of StreamingContext. It's okay for users trying to configure stuff themselves.
private[spark] object MetadataCleaner {
- def getDelaySeconds(conf: SparkConf) = {
+ def getDelaySeconds(conf: SparkConf): Int = {
conf.getInt("spark.cleaner.ttl", -1)
}
diff --git a/core/src/main/scala/org/apache/spark/util/MutablePair.scala b/core/src/main/scala/org/apache/spark/util/MutablePair.scala
index 74fa77b68de0b18761921858f144a164a99dbaa6..dad888548ed10c4ed1e72d187123cbc78bdc10b6 100644
--- a/core/src/main/scala/org/apache/spark/util/MutablePair.scala
+++ b/core/src/main/scala/org/apache/spark/util/MutablePair.scala
@@ -43,7 +43,7 @@ case class MutablePair[@specialized(Int, Long, Double, Char, Boolean/* , AnyRef
this
}
- override def toString = "(" + _1 + "," + _2 + ")"
+ override def toString: String = "(" + _1 + "," + _2 + ")"
override def canEqual(that: Any): Boolean = that.isInstanceOf[MutablePair[_,_]]
}
diff --git a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala b/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
index 6d8d9e8da36789a634d76f5f0ad0793bd0e00100..73d126ff6254e12b650a399904334491b02ea97b 100644
--- a/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
+++ b/core/src/main/scala/org/apache/spark/util/ParentClassLoader.scala
@@ -22,7 +22,7 @@ package org.apache.spark.util
*/
private[spark] class ParentClassLoader(parent: ClassLoader) extends ClassLoader(parent) {
- override def findClass(name: String) = {
+ override def findClass(name: String): Class[_] = {
super.findClass(name)
}
diff --git a/core/src/main/scala/org/apache/spark/util/SerializableBuffer.scala b/core/src/main/scala/org/apache/spark/util/SerializableBuffer.scala
index 770ff9d5ad6ae863214911e964c498fbf3b62ac1..a06b6f84ef11bbcf4ed1a0f4878b8deff01aa7d6 100644
--- a/core/src/main/scala/org/apache/spark/util/SerializableBuffer.scala
+++ b/core/src/main/scala/org/apache/spark/util/SerializableBuffer.scala
@@ -27,7 +27,7 @@ import java.nio.channels.Channels
*/
private[spark]
class SerializableBuffer(@transient var buffer: ByteBuffer) extends Serializable {
- def value = buffer
+ def value: ByteBuffer = buffer
private def readObject(in: ObjectInputStream): Unit = Utils.tryOrIOException {
val length = in.readInt()
diff --git a/core/src/main/scala/org/apache/spark/util/StatCounter.scala b/core/src/main/scala/org/apache/spark/util/StatCounter.scala
index d80eed455c42755eb00e6da598fb8837039ca290..8586da1996cf30a2987e24428bf4a1956d3d6c32 100644
--- a/core/src/main/scala/org/apache/spark/util/StatCounter.scala
+++ b/core/src/main/scala/org/apache/spark/util/StatCounter.scala
@@ -141,8 +141,8 @@ class StatCounter(values: TraversableOnce[Double]) extends Serializable {
object StatCounter {
/** Build a StatCounter from a list of values. */
- def apply(values: TraversableOnce[Double]) = new StatCounter(values)
+ def apply(values: TraversableOnce[Double]): StatCounter = new StatCounter(values)
/** Build a StatCounter from a list of values passed as variable-length arguments. */
- def apply(values: Double*) = new StatCounter(values)
+ def apply(values: Double*): StatCounter = new StatCounter(values)
}
diff --git a/core/src/main/scala/org/apache/spark/util/TimeStampedWeakValueHashMap.scala b/core/src/main/scala/org/apache/spark/util/TimeStampedWeakValueHashMap.scala
index f5be5856c2109be66ed6e4616c4a3b8491ddc699..310c0c109416cca89897006d53acc589df0f9a44 100644
--- a/core/src/main/scala/org/apache/spark/util/TimeStampedWeakValueHashMap.scala
+++ b/core/src/main/scala/org/apache/spark/util/TimeStampedWeakValueHashMap.scala
@@ -82,7 +82,7 @@ private[spark] class TimeStampedWeakValueHashMap[A, B](updateTimeStampOnGet: Boo
this
}
- override def update(key: A, value: B) = this += ((key, value))
+ override def update(key: A, value: B): Unit = this += ((key, value))
override def apply(key: A): B = internalMap.apply(key)
@@ -92,14 +92,14 @@ private[spark] class TimeStampedWeakValueHashMap[A, B](updateTimeStampOnGet: Boo
override def size: Int = internalMap.size
- override def foreach[U](f: ((A, B)) => U) = nonNullReferenceMap.foreach(f)
+ override def foreach[U](f: ((A, B)) => U): Unit = nonNullReferenceMap.foreach(f)
def putIfAbsent(key: A, value: B): Option[B] = internalMap.putIfAbsent(key, value)
def toMap: Map[A, B] = iterator.toMap
/** Remove old key-value pairs with timestamps earlier than `threshTime`. */
- def clearOldValues(threshTime: Long) = internalMap.clearOldValues(threshTime)
+ def clearOldValues(threshTime: Long): Unit = internalMap.clearOldValues(threshTime)
/** Remove entries with values that are no longer strongly reachable. */
def clearNullValues() {
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index fa56bb09e4e5cd25a71b94969f77aad3d3953a0b..d9a671687aad0e19b1552f9e96e2633119658984 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -85,7 +85,7 @@ private[spark] object Utils extends Logging {
def deserialize[T](bytes: Array[Byte], loader: ClassLoader): T = {
val bis = new ByteArrayInputStream(bytes)
val ois = new ObjectInputStream(bis) {
- override def resolveClass(desc: ObjectStreamClass) =
+ override def resolveClass(desc: ObjectStreamClass): Class[_] =
Class.forName(desc.getName, false, loader)
}
ois.readObject.asInstanceOf[T]
@@ -106,11 +106,10 @@ private[spark] object Utils extends Logging {
/** Serialize via nested stream using specific serializer */
def serializeViaNestedStream(os: OutputStream, ser: SerializerInstance)(
- f: SerializationStream => Unit) = {
+ f: SerializationStream => Unit): Unit = {
val osWrapper = ser.serializeStream(new OutputStream {
- def write(b: Int) = os.write(b)
-
- override def write(b: Array[Byte], off: Int, len: Int) = os.write(b, off, len)
+ override def write(b: Int): Unit = os.write(b)
+ override def write(b: Array[Byte], off: Int, len: Int): Unit = os.write(b, off, len)
})
try {
f(osWrapper)
@@ -121,10 +120,9 @@ private[spark] object Utils extends Logging {
/** Deserialize via nested stream using specific serializer */
def deserializeViaNestedStream(is: InputStream, ser: SerializerInstance)(
- f: DeserializationStream => Unit) = {
+ f: DeserializationStream => Unit): Unit = {
val isWrapper = ser.deserializeStream(new InputStream {
- def read(): Int = is.read()
-
+ override def read(): Int = is.read()
override def read(b: Array[Byte], off: Int, len: Int): Int = is.read(b, off, len)
})
try {
@@ -137,7 +135,7 @@ private[spark] object Utils extends Logging {
/**
* Get the ClassLoader which loaded Spark.
*/
- def getSparkClassLoader = getClass.getClassLoader
+ def getSparkClassLoader: ClassLoader = getClass.getClassLoader
/**
* Get the Context ClassLoader on this thread or, if not present, the ClassLoader that
@@ -146,7 +144,7 @@ private[spark] object Utils extends Logging {
* This should be used whenever passing a ClassLoader to Class.ForName or finding the currently
* active loader when setting up ClassLoader delegation chains.
*/
- def getContextOrSparkClassLoader =
+ def getContextOrSparkClassLoader: ClassLoader =
Option(Thread.currentThread().getContextClassLoader).getOrElse(getSparkClassLoader)
/** Determines whether the provided class is loadable in the current thread. */
@@ -155,12 +153,14 @@ private[spark] object Utils extends Logging {
}
/** Preferred alternative to Class.forName(className) */
- def classForName(className: String) = Class.forName(className, true, getContextOrSparkClassLoader)
+ def classForName(className: String): Class[_] = {
+ Class.forName(className, true, getContextOrSparkClassLoader)
+ }
/**
* Primitive often used when writing [[java.nio.ByteBuffer]] to [[java.io.DataOutput]]
*/
- def writeByteBuffer(bb: ByteBuffer, out: ObjectOutput) = {
+ def writeByteBuffer(bb: ByteBuffer, out: ObjectOutput): Unit = {
if (bb.hasArray) {
out.write(bb.array(), bb.arrayOffset() + bb.position(), bb.remaining())
} else {
@@ -1557,7 +1557,7 @@ private[spark] object Utils extends Logging {
/** Return the class name of the given object, removing all dollar signs */
- def getFormattedClassName(obj: AnyRef) = {
+ def getFormattedClassName(obj: AnyRef): String = {
obj.getClass.getSimpleName.replace("$", "")
}
@@ -1570,7 +1570,7 @@ private[spark] object Utils extends Logging {
}
/** Return an empty JSON object */
- def emptyJson = JObject(List[JField]())
+ def emptyJson: JsonAST.JObject = JObject(List[JField]())
/**
* Return a Hadoop FileSystem with the scheme encoded in the given path.
@@ -1618,7 +1618,7 @@ private[spark] object Utils extends Logging {
/**
* Indicates whether Spark is currently running unit tests.
*/
- def isTesting = {
+ def isTesting: Boolean = {
sys.env.contains("SPARK_TESTING") || sys.props.contains("spark.testing")
}
diff --git a/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala b/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala
index af1f64649f354da23985d548504b4a484104377d..f79e8e0491ea1149b32753b010f5c982bbf19d69 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/BitSet.scala
@@ -156,10 +156,10 @@ class BitSet(numBits: Int) extends Serializable {
/**
* Get an iterator over the set bits.
*/
- def iterator = new Iterator[Int] {
+ def iterator: Iterator[Int] = new Iterator[Int] {
var ind = nextSetBit(0)
override def hasNext: Boolean = ind >= 0
- override def next() = {
+ override def next(): Int = {
val tmp = ind
ind = nextSetBit(ind + 1)
tmp
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
index 8a0f5a602de12bfa61ddf980290181eeba6a9b6c..9ff4744593d4dc9a91ad88dd6f4b17f86dde37cb 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
@@ -159,7 +159,7 @@ class ExternalAppendOnlyMap[K, V, C](
val batchSizes = new ArrayBuffer[Long]
// Flush the disk writer's contents to disk, and update relevant variables
- def flush() = {
+ def flush(): Unit = {
val w = writer
writer = null
w.commitAndClose()
@@ -355,7 +355,7 @@ class ExternalAppendOnlyMap[K, V, C](
val pairs: ArrayBuffer[(K, C)])
extends Comparable[StreamBuffer] {
- def isEmpty = pairs.length == 0
+ def isEmpty: Boolean = pairs.length == 0
// Invalid if there are no more pairs in this stream
def minKeyHash: Int = {
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
index d69f2d9048055bf14444d401feec5cd385bc6430..3262e670c203067e07bb38d7c2618949d28e419b 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
@@ -283,7 +283,7 @@ private[spark] class ExternalSorter[K, V, C](
// Flush the disk writer's contents to disk, and update relevant variables.
// The writer is closed at the end of this process, and cannot be reused.
- def flush() = {
+ def flush(): Unit = {
val w = writer
writer = null
w.commitAndClose()
diff --git a/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala b/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala
index b8de4ff9aa494f3efbf5f910793633005b740d5c..c52591b35234026ed76fd04e6261a1093a5e9db4 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/OpenHashMap.scala
@@ -109,7 +109,7 @@ class OpenHashMap[K : ClassTag, @specialized(Long, Int, Double) V: ClassTag](
}
}
- override def iterator = new Iterator[(K, V)] {
+ override def iterator: Iterator[(K, V)] = new Iterator[(K, V)] {
var pos = -1
var nextPair: (K, V) = computeNextPair()
@@ -132,9 +132,9 @@ class OpenHashMap[K : ClassTag, @specialized(Long, Int, Double) V: ClassTag](
}
}
- def hasNext = nextPair != null
+ def hasNext: Boolean = nextPair != null
- def next() = {
+ def next(): (K, V) = {
val pair = nextPair
nextPair = computeNextPair()
pair
diff --git a/core/src/main/scala/org/apache/spark/util/collection/OpenHashSet.scala b/core/src/main/scala/org/apache/spark/util/collection/OpenHashSet.scala
index 4e363b74f4bef60b579c51a3366ecfc0b38e0f92..c80057f95e0b206327592c8bd0ff54f7cacfe079 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/OpenHashSet.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/OpenHashSet.scala
@@ -85,7 +85,7 @@ class OpenHashSet[@specialized(Long, Int) T: ClassTag](
protected var _bitset = new BitSet(_capacity)
- def getBitSet = _bitset
+ def getBitSet: BitSet = _bitset
// Init of the array in constructor (instead of in declaration) to work around a Scala compiler
// specialization bug that would generate two arrays (one for Object and one for specialized T).
@@ -183,7 +183,7 @@ class OpenHashSet[@specialized(Long, Int) T: ClassTag](
/** Return the value at the specified position. */
def getValue(pos: Int): T = _data(pos)
- def iterator = new Iterator[T] {
+ def iterator: Iterator[T] = new Iterator[T] {
var pos = nextPos(0)
override def hasNext: Boolean = pos != INVALID_POS
override def next(): T = {
diff --git a/core/src/main/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMap.scala b/core/src/main/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMap.scala
index 2e1ef06cbc4e11df051a53f73e5c6facb9c5c5a3..61e22642761f0415788c378c726644b1ea36b81c 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMap.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/PrimitiveKeyOpenHashMap.scala
@@ -46,7 +46,7 @@ class PrimitiveKeyOpenHashMap[@specialized(Long, Int) K: ClassTag,
private var _oldValues: Array[V] = null
- override def size = _keySet.size
+ override def size: Int = _keySet.size
/** Get the value for a given key */
def apply(k: K): V = {
@@ -87,7 +87,7 @@ class PrimitiveKeyOpenHashMap[@specialized(Long, Int) K: ClassTag,
}
}
- override def iterator = new Iterator[(K, V)] {
+ override def iterator: Iterator[(K, V)] = new Iterator[(K, V)] {
var pos = 0
var nextPair: (K, V) = computeNextPair()
@@ -103,9 +103,9 @@ class PrimitiveKeyOpenHashMap[@specialized(Long, Int) K: ClassTag,
}
}
- def hasNext = nextPair != null
+ def hasNext: Boolean = nextPair != null
- def next() = {
+ def next(): (K, V) = {
val pair = nextPair
nextPair = computeNextPair()
pair
diff --git a/core/src/main/scala/org/apache/spark/util/collection/Utils.scala b/core/src/main/scala/org/apache/spark/util/collection/Utils.scala
index c5268c0fae0ef9f86e26597222bd396e2c6b2677..bdbca00a0062219c23dc5d95b1f9f848267d5ede 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/Utils.scala
@@ -32,7 +32,7 @@ private[spark] object Utils {
*/
def takeOrdered[T](input: Iterator[T], num: Int)(implicit ord: Ordering[T]): Iterator[T] = {
val ordering = new GuavaOrdering[T] {
- override def compare(l: T, r: T) = ord.compare(l, r)
+ override def compare(l: T, r: T): Int = ord.compare(l, r)
}
collectionAsScalaIterable(ordering.leastOf(asJavaIterator(input), num)).iterator
}
diff --git a/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala b/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
index 1d5467060623c820a462fbdfcb8bc054492224f9..14b6ba4af489a6ad5a49761ac01ed0e2ff341fbe 100644
--- a/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
+++ b/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
@@ -121,7 +121,7 @@ private[spark] object FileAppender extends Logging {
val rollingSizeBytes = conf.get(SIZE_PROPERTY, STRATEGY_DEFAULT)
val rollingInterval = conf.get(INTERVAL_PROPERTY, INTERVAL_DEFAULT)
- def createTimeBasedAppender() = {
+ def createTimeBasedAppender(): FileAppender = {
val validatedParams: Option[(Long, String)] = rollingInterval match {
case "daily" =>
logInfo(s"Rolling executor logs enabled for $file with daily rolling")
@@ -149,7 +149,7 @@ private[spark] object FileAppender extends Logging {
}
}
- def createSizeBasedAppender() = {
+ def createSizeBasedAppender(): FileAppender = {
rollingSizeBytes match {
case IntParam(bytes) =>
logInfo(s"Rolling executor logs enabled for $file with rolling every $bytes bytes")
diff --git a/core/src/main/scala/org/apache/spark/util/random/RandomSampler.scala b/core/src/main/scala/org/apache/spark/util/random/RandomSampler.scala
index 76e7a2760bcd14c5fd13645e4c1cb5be608beef9..786b97ad7b9ece4e5c2d1c646286db415ca26b8a 100644
--- a/core/src/main/scala/org/apache/spark/util/random/RandomSampler.scala
+++ b/core/src/main/scala/org/apache/spark/util/random/RandomSampler.scala
@@ -105,7 +105,7 @@ class BernoulliCellSampler[T](lb: Double, ub: Double, complement: Boolean = fals
private val rng: Random = new XORShiftRandom
- override def setSeed(seed: Long) = rng.setSeed(seed)
+ override def setSeed(seed: Long): Unit = rng.setSeed(seed)
override def sample(items: Iterator[T]): Iterator[T] = {
if (ub - lb <= 0.0) {
@@ -131,7 +131,7 @@ class BernoulliCellSampler[T](lb: Double, ub: Double, complement: Boolean = fals
def cloneComplement(): BernoulliCellSampler[T] =
new BernoulliCellSampler[T](lb, ub, !complement)
- override def clone = new BernoulliCellSampler[T](lb, ub, complement)
+ override def clone: BernoulliCellSampler[T] = new BernoulliCellSampler[T](lb, ub, complement)
}
@@ -153,7 +153,7 @@ class BernoulliSampler[T: ClassTag](fraction: Double) extends RandomSampler[T, T
private val rng: Random = RandomSampler.newDefaultRNG
- override def setSeed(seed: Long) = rng.setSeed(seed)
+ override def setSeed(seed: Long): Unit = rng.setSeed(seed)
override def sample(items: Iterator[T]): Iterator[T] = {
if (fraction <= 0.0) {
@@ -167,7 +167,7 @@ class BernoulliSampler[T: ClassTag](fraction: Double) extends RandomSampler[T, T
}
}
- override def clone = new BernoulliSampler[T](fraction)
+ override def clone: BernoulliSampler[T] = new BernoulliSampler[T](fraction)
}
@@ -209,7 +209,7 @@ class PoissonSampler[T: ClassTag](fraction: Double) extends RandomSampler[T, T]
}
}
- override def clone = new PoissonSampler[T](fraction)
+ override def clone: PoissonSampler[T] = new PoissonSampler[T](fraction)
}
@@ -228,15 +228,18 @@ class GapSamplingIterator[T: ClassTag](
val arrayClass = Array.empty[T].iterator.getClass
val arrayBufferClass = ArrayBuffer.empty[T].iterator.getClass
data.getClass match {
- case `arrayClass` => ((n: Int) => { data = data.drop(n) })
- case `arrayBufferClass` => ((n: Int) => { data = data.drop(n) })
- case _ => ((n: Int) => {
+ case `arrayClass` =>
+ (n: Int) => { data = data.drop(n) }
+ case `arrayBufferClass` =>
+ (n: Int) => { data = data.drop(n) }
+ case _ =>
+ (n: Int) => {
var j = 0
while (j < n && data.hasNext) {
data.next()
j += 1
}
- })
+ }
}
}
@@ -244,21 +247,21 @@ class GapSamplingIterator[T: ClassTag](
override def next(): T = {
val r = data.next()
- advance
+ advance()
r
}
private val lnq = math.log1p(-f)
/** skip elements that won't be sampled, according to geometric dist P(k) = (f)(1-f)^k. */
- private def advance: Unit = {
+ private def advance(): Unit = {
val u = math.max(rng.nextDouble(), epsilon)
val k = (math.log(u) / lnq).toInt
iterDrop(k)
}
/** advance to first sample as part of object construction. */
- advance
+ advance()
// Attempting to invoke this closer to the top with other object initialization
// was causing it to break in strange ways, so I'm invoking it last, which seems to
// work reliably.
@@ -279,15 +282,18 @@ class GapSamplingReplacementIterator[T: ClassTag](
val arrayClass = Array.empty[T].iterator.getClass
val arrayBufferClass = ArrayBuffer.empty[T].iterator.getClass
data.getClass match {
- case `arrayClass` => ((n: Int) => { data = data.drop(n) })
- case `arrayBufferClass` => ((n: Int) => { data = data.drop(n) })
- case _ => ((n: Int) => {
+ case `arrayClass` =>
+ (n: Int) => { data = data.drop(n) }
+ case `arrayBufferClass` =>
+ (n: Int) => { data = data.drop(n) }
+ case _ =>
+ (n: Int) => {
var j = 0
while (j < n && data.hasNext) {
data.next()
j += 1
}
- })
+ }
}
}
@@ -300,7 +306,7 @@ class GapSamplingReplacementIterator[T: ClassTag](
override def next(): T = {
val r = v
rep -= 1
- if (rep <= 0) advance
+ if (rep <= 0) advance()
r
}
@@ -309,7 +315,7 @@ class GapSamplingReplacementIterator[T: ClassTag](
* Samples 'k' from geometric distribution P(k) = (1-q)(q)^k, where q = e^(-f), that is
* q is the probabililty of Poisson(0; f)
*/
- private def advance: Unit = {
+ private def advance(): Unit = {
val u = math.max(rng.nextDouble(), epsilon)
val k = (math.log(u) / (-f)).toInt
iterDrop(k)
@@ -343,7 +349,7 @@ class GapSamplingReplacementIterator[T: ClassTag](
}
/** advance to first sample as part of object construction. */
- advance
+ advance()
// Attempting to invoke this closer to the top with other object initialization
// was causing it to break in strange ways, so I'm invoking it last, which seems to
// work reliably.
diff --git a/core/src/main/scala/org/apache/spark/util/random/StratifiedSamplingUtils.scala b/core/src/main/scala/org/apache/spark/util/random/StratifiedSamplingUtils.scala
index 2ae308dacf1ae357e6ff3d688e621629c96d0ddb..9e29bf9d61f177cc4c9e2715238f4bcadf63562c 100644
--- a/core/src/main/scala/org/apache/spark/util/random/StratifiedSamplingUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/random/StratifiedSamplingUtils.scala
@@ -311,7 +311,7 @@ private[random] class AcceptanceResult(var numItems: Long = 0L, var numAccepted:
var acceptBound: Double = Double.NaN // upper bound for accepting item instantly
var waitListBound: Double = Double.NaN // upper bound for adding item to waitlist
- def areBoundsEmpty = acceptBound.isNaN || waitListBound.isNaN
+ def areBoundsEmpty: Boolean = acceptBound.isNaN || waitListBound.isNaN
def merge(other: Option[AcceptanceResult]): Unit = {
if (other.isDefined) {
diff --git a/core/src/main/scala/org/apache/spark/util/random/XORShiftRandom.scala b/core/src/main/scala/org/apache/spark/util/random/XORShiftRandom.scala
index 467b890fb4bb92cb05f70bf1086a79e89f751296..c4a7b4441c85cf5c8914fc6e1a46b7c51f4ea261 100644
--- a/core/src/main/scala/org/apache/spark/util/random/XORShiftRandom.scala
+++ b/core/src/main/scala/org/apache/spark/util/random/XORShiftRandom.scala
@@ -83,7 +83,7 @@ private[spark] object XORShiftRandom {
* @return Map of execution times for {@link java.util.Random java.util.Random}
* and XORShift
*/
- def benchmark(numIters: Int) = {
+ def benchmark(numIters: Int): Map[String, Long] = {
val seed = 1L
val million = 1e6.toInt
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 328d59485a7319f33bbff6641d3bc7008d36e083..56f5dbe53fad4f381fcf633e14a92f2b8bef3097 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -44,7 +44,13 @@ object MimaExcludes {
// the maven-generated artifacts in 1.3.
excludePackage("org.spark-project.jetty"),
MimaBuild.excludeSparkPackage("unused"),
- ProblemFilters.exclude[MissingClassProblem]("com.google.common.base.Optional")
+ ProblemFilters.exclude[MissingClassProblem]("com.google.common.base.Optional"),
+ ProblemFilters.exclude[IncompatibleResultTypeProblem](
+ "org.apache.spark.rdd.JdbcRDD.compute"),
+ ProblemFilters.exclude[IncompatibleResultTypeProblem](
+ "org.apache.spark.broadcast.HttpBroadcastFactory.newBroadcast"),
+ ProblemFilters.exclude[IncompatibleResultTypeProblem](
+ "org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast")
)
case v if v.startsWith("1.3") =>
diff --git a/scalastyle-config.xml b/scalastyle-config.xml
index 0ff521706c71aa3269ae3422d6a54e546a38ac9b..459a5035d4984a0958bbcf810ce39cf45ff54d2e 100644
--- a/scalastyle-config.xml
+++ b/scalastyle-config.xml
@@ -137,9 +137,9 @@
<!-- <parameter name="maxMethods"><![CDATA[30]]></parameter> -->
<!-- </parameters> -->
<!-- </check> -->
- <!-- <check level="error" class="org.scalastyle.scalariform.PublicMethodsHaveTypeChecker" enabled="true"></check> -->
+ <check level="error" class="org.scalastyle.scalariform.PublicMethodsHaveTypeChecker" enabled="false"></check>
<check level="error" class="org.scalastyle.file.NewLineAtEofChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.file.NoNewLineAtEofChecker" enabled="false"></check>
- <check level="error" class="org.apache.spark.scalastyle.NonASCIICharacterChecker" enabled="true"></check>
+ <check level="error" class="org.scalastyle.scalariform.NonASCIICharacterChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.SpaceAfterCommentStartChecker" enabled="true"></check>
</scalastyle>