diff --git a/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleWriter.java
index 2389c28b2839532f98cf9ce8d926ef5b2d9fb768..fdb309e365f696ad62dfbf68674690c93d6e4945 100644
--- a/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/unsafe/UnsafeShuffleWriter.java
@@ -24,7 +24,7 @@ import java.util.Iterator;
import scala.Option;
import scala.Product2;
-import scala.collection.JavaConversions;
+import scala.collection.JavaConverters;
import scala.collection.immutable.Map;
import scala.reflect.ClassTag;
import scala.reflect.ClassTag$;
@@ -160,7 +160,7 @@ public class UnsafeShuffleWriter<K, V> extends ShuffleWriter<K, V> {
*/
@VisibleForTesting
public void write(Iterator<Product2<K, V>> records) throws IOException {
- write(JavaConversions.asScalaIterator(records));
+ write(JavaConverters.asScalaIteratorConverter(records).asScala());
}
@Override
diff --git a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
index 92218832d256f1c05926204993792822f12f2db7..a3875927838507ac940bd40e152d7b96979a8340 100644
--- a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
+++ b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
@@ -21,8 +21,8 @@ import java.io._
import java.util.concurrent.ConcurrentHashMap
import java.util.zip.{GZIPInputStream, GZIPOutputStream}
+import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, Map}
-import scala.collection.JavaConversions._
import scala.reflect.ClassTag
import org.apache.spark.rpc.{RpcEndpointRef, RpcEnv, RpcCallContext, RpcEndpoint}
@@ -398,7 +398,7 @@ private[spark] class MapOutputTrackerMaster(conf: SparkConf)
*/
private[spark] class MapOutputTrackerWorker(conf: SparkConf) extends MapOutputTracker(conf) {
protected val mapStatuses: Map[Int, Array[MapStatus]] =
- new ConcurrentHashMap[Int, Array[MapStatus]]
+ new ConcurrentHashMap[Int, Array[MapStatus]]().asScala
}
private[spark] object MapOutputTracker extends Logging {
diff --git a/core/src/main/scala/org/apache/spark/SSLOptions.scala b/core/src/main/scala/org/apache/spark/SSLOptions.scala
index 32df42d57dbd6aa7b6bc16bb0a7fb6cd52e55b18..3b9c885bf97a7a72cbda408a13890238d101fbd7 100644
--- a/core/src/main/scala/org/apache/spark/SSLOptions.scala
+++ b/core/src/main/scala/org/apache/spark/SSLOptions.scala
@@ -17,9 +17,11 @@
package org.apache.spark
-import java.io.{File, FileInputStream}
-import java.security.{KeyStore, NoSuchAlgorithmException}
-import javax.net.ssl.{KeyManager, KeyManagerFactory, SSLContext, TrustManager, TrustManagerFactory}
+import java.io.File
+import java.security.NoSuchAlgorithmException
+import javax.net.ssl.SSLContext
+
+import scala.collection.JavaConverters._
import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory}
import org.eclipse.jetty.util.ssl.SslContextFactory
@@ -79,7 +81,6 @@ private[spark] case class SSLOptions(
* object. It can be used then to compose the ultimate Akka configuration.
*/
def createAkkaConfig: Option[Config] = {
- import scala.collection.JavaConversions._
if (enabled) {
Some(ConfigFactory.empty()
.withValue("akka.remote.netty.tcp.security.key-store",
@@ -97,7 +98,7 @@ private[spark] case class SSLOptions(
.withValue("akka.remote.netty.tcp.security.protocol",
ConfigValueFactory.fromAnyRef(protocol.getOrElse("")))
.withValue("akka.remote.netty.tcp.security.enabled-algorithms",
- ConfigValueFactory.fromIterable(supportedAlgorithms.toSeq))
+ ConfigValueFactory.fromIterable(supportedAlgorithms.asJava))
.withValue("akka.remote.netty.tcp.enable-ssl",
ConfigValueFactory.fromAnyRef(true)))
} else {
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 9849aff85d72e556b3309f16056e231f05da7c5a..f3da04a7f55d04cbae7cd12b5cd3f1f40291ac96 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -26,8 +26,8 @@ import java.util.{Arrays, Properties, UUID}
import java.util.concurrent.atomic.{AtomicReference, AtomicBoolean, AtomicInteger}
import java.util.UUID.randomUUID
+import scala.collection.JavaConverters._
import scala.collection.{Map, Set}
-import scala.collection.JavaConversions._
import scala.collection.generic.Growable
import scala.collection.mutable.HashMap
import scala.reflect.{ClassTag, classTag}
@@ -1546,7 +1546,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
def getAllPools: Seq[Schedulable] = {
assertNotStopped()
// TODO(xiajunluan): We should take nested pools into account
- taskScheduler.rootPool.schedulableQueue.toSeq
+ taskScheduler.rootPool.schedulableQueue.asScala.toSeq
}
/**
diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala
index a1ebbecf93b7bc211be220defd4f72ccf780f4be..888763a3e8ebf1002ea60ef899cc0a69020aa40f 100644
--- a/core/src/main/scala/org/apache/spark/TestUtils.scala
+++ b/core/src/main/scala/org/apache/spark/TestUtils.scala
@@ -19,11 +19,12 @@ package org.apache.spark
import java.io.{ByteArrayInputStream, File, FileInputStream, FileOutputStream}
import java.net.{URI, URL}
+import java.nio.charset.StandardCharsets
+import java.util.Arrays
import java.util.jar.{JarEntry, JarOutputStream}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
-import com.google.common.base.Charsets.UTF_8
import com.google.common.io.{ByteStreams, Files}
import javax.tools.{JavaFileObject, SimpleJavaFileObject, ToolProvider}
@@ -71,7 +72,7 @@ private[spark] object TestUtils {
files.foreach { case (k, v) =>
val entry = new JarEntry(k)
jarStream.putNextEntry(entry)
- ByteStreams.copy(new ByteArrayInputStream(v.getBytes(UTF_8)), jarStream)
+ ByteStreams.copy(new ByteArrayInputStream(v.getBytes(StandardCharsets.UTF_8)), jarStream)
}
jarStream.close()
jarFile.toURI.toURL
@@ -125,7 +126,7 @@ private[spark] object TestUtils {
} else {
Seq()
}
- compiler.getTask(null, null, null, options, null, Seq(sourceFile)).call()
+ compiler.getTask(null, null, null, options.asJava, null, Arrays.asList(sourceFile)).call()
val fileName = className + ".class"
val result = new File(fileName)
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaHadoopRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaHadoopRDD.scala
index 0ae0b4ec042e2a2dfbdf3f0117e68ecad55b7ab0..891bcddeac2865c30faf94f02322b884b76b2238 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaHadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaHadoopRDD.scala
@@ -17,7 +17,7 @@
package org.apache.spark.api.java
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import org.apache.hadoop.mapred.InputSplit
@@ -37,7 +37,7 @@ class JavaHadoopRDD[K, V](rdd: HadoopRDD[K, V])
def mapPartitionsWithInputSplit[R](
f: JFunction2[InputSplit, java.util.Iterator[(K, V)], java.util.Iterator[R]],
preservesPartitioning: Boolean = false): JavaRDD[R] = {
- new JavaRDD(rdd.mapPartitionsWithInputSplit((a, b) => f.call(a, asJavaIterator(b)),
+ new JavaRDD(rdd.mapPartitionsWithInputSplit((a, b) => f.call(a, b.asJava).asScala,
preservesPartitioning)(fakeClassTag))(fakeClassTag)
}
}
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaNewHadoopRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaNewHadoopRDD.scala
index ec4f3964d75e0131cca4e2b21179a1d79b4ffe8c..0f49279f3e647d5b8fcd984daecc5f509255c113 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaNewHadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaNewHadoopRDD.scala
@@ -17,7 +17,7 @@
package org.apache.spark.api.java
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import org.apache.hadoop.mapreduce.InputSplit
@@ -37,7 +37,7 @@ class JavaNewHadoopRDD[K, V](rdd: NewHadoopRDD[K, V])
def mapPartitionsWithInputSplit[R](
f: JFunction2[InputSplit, java.util.Iterator[(K, V)], java.util.Iterator[R]],
preservesPartitioning: Boolean = false): JavaRDD[R] = {
- new JavaRDD(rdd.mapPartitionsWithInputSplit((a, b) => f.call(a, asJavaIterator(b)),
+ new JavaRDD(rdd.mapPartitionsWithInputSplit((a, b) => f.call(a, b.asJava).asScala,
preservesPartitioning)(fakeClassTag))(fakeClassTag)
}
}
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
index 8441bb3a3047effcbda023f7007d633f1782a6e3..fb787979c18209e7d69c582b9f7a2f77ac43b3b0 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
@@ -20,7 +20,7 @@ package org.apache.spark.api.java
import java.util.{Comparator, List => JList, Map => JMap}
import java.lang.{Iterable => JIterable}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.language.implicitConversions
import scala.reflect.ClassTag
@@ -142,7 +142,7 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])
def sampleByKey(withReplacement: Boolean,
fractions: JMap[K, Double],
seed: Long): JavaPairRDD[K, V] =
- new JavaPairRDD[K, V](rdd.sampleByKey(withReplacement, fractions, seed))
+ new JavaPairRDD[K, V](rdd.sampleByKey(withReplacement, fractions.asScala, seed))
/**
* Return a subset of this RDD sampled by key (via stratified sampling).
@@ -173,7 +173,7 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])
def sampleByKeyExact(withReplacement: Boolean,
fractions: JMap[K, Double],
seed: Long): JavaPairRDD[K, V] =
- new JavaPairRDD[K, V](rdd.sampleByKeyExact(withReplacement, fractions, seed))
+ new JavaPairRDD[K, V](rdd.sampleByKeyExact(withReplacement, fractions.asScala, seed))
/**
* ::Experimental::
@@ -768,7 +768,7 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])
* Return the list of values in the RDD for key `key`. This operation is done efficiently if the
* RDD has a known partitioner by only searching the partition that the key maps to.
*/
- def lookup(key: K): JList[V] = seqAsJavaList(rdd.lookup(key))
+ def lookup(key: K): JList[V] = rdd.lookup(key).asJava
/** Output the RDD to any Hadoop-supported file system. */
def saveAsHadoopFile[F <: OutputFormat[_, _]](
@@ -987,30 +987,27 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])
object JavaPairRDD {
private[spark]
def groupByResultToJava[K: ClassTag, T](rdd: RDD[(K, Iterable[T])]): RDD[(K, JIterable[T])] = {
- rddToPairRDDFunctions(rdd).mapValues(asJavaIterable)
+ rddToPairRDDFunctions(rdd).mapValues(_.asJava)
}
private[spark]
def cogroupResultToJava[K: ClassTag, V, W](
rdd: RDD[(K, (Iterable[V], Iterable[W]))]): RDD[(K, (JIterable[V], JIterable[W]))] = {
- rddToPairRDDFunctions(rdd).mapValues(x => (asJavaIterable(x._1), asJavaIterable(x._2)))
+ rddToPairRDDFunctions(rdd).mapValues(x => (x._1.asJava, x._2.asJava))
}
private[spark]
def cogroupResult2ToJava[K: ClassTag, V, W1, W2](
rdd: RDD[(K, (Iterable[V], Iterable[W1], Iterable[W2]))])
: RDD[(K, (JIterable[V], JIterable[W1], JIterable[W2]))] = {
- rddToPairRDDFunctions(rdd)
- .mapValues(x => (asJavaIterable(x._1), asJavaIterable(x._2), asJavaIterable(x._3)))
+ rddToPairRDDFunctions(rdd).mapValues(x => (x._1.asJava, x._2.asJava, x._3.asJava))
}
private[spark]
def cogroupResult3ToJava[K: ClassTag, V, W1, W2, W3](
rdd: RDD[(K, (Iterable[V], Iterable[W1], Iterable[W2], Iterable[W3]))])
: RDD[(K, (JIterable[V], JIterable[W1], JIterable[W2], JIterable[W3]))] = {
- rddToPairRDDFunctions(rdd)
- .mapValues(x =>
- (asJavaIterable(x._1), asJavaIterable(x._2), asJavaIterable(x._3), asJavaIterable(x._4)))
+ rddToPairRDDFunctions(rdd).mapValues(x => (x._1.asJava, x._2.asJava, x._3.asJava, x._4.asJava))
}
def fromRDD[K: ClassTag, V: ClassTag](rdd: RDD[(K, V)]): JavaPairRDD[K, V] = {
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
index c582488f16fe72e74c638679eca5bc54d26fc8d9..fc817cdd6a3f80ac06de742f48aede35d77d82a4 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
@@ -21,7 +21,6 @@ import java.{lang => jl}
import java.lang.{Iterable => JIterable, Long => JLong}
import java.util.{Comparator, List => JList, Iterator => JIterator}
-import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
import scala.reflect.ClassTag
@@ -59,10 +58,10 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
def rdd: RDD[T]
@deprecated("Use partitions() instead.", "1.1.0")
- def splits: JList[Partition] = new java.util.ArrayList(rdd.partitions.toSeq)
+ def splits: JList[Partition] = rdd.partitions.toSeq.asJava
/** Set of partitions in this RDD. */
- def partitions: JList[Partition] = new java.util.ArrayList(rdd.partitions.toSeq)
+ def partitions: JList[Partition] = rdd.partitions.toSeq.asJava
/** The partitioner of this RDD. */
def partitioner: Optional[Partitioner] = JavaUtils.optionToOptional(rdd.partitioner)
@@ -82,7 +81,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
* subclasses of RDD.
*/
def iterator(split: Partition, taskContext: TaskContext): java.util.Iterator[T] =
- asJavaIterator(rdd.iterator(split, taskContext))
+ rdd.iterator(split, taskContext).asJava
// Transformations (return a new RDD)
@@ -99,7 +98,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
def mapPartitionsWithIndex[R](
f: JFunction2[jl.Integer, java.util.Iterator[T], java.util.Iterator[R]],
preservesPartitioning: Boolean = false): JavaRDD[R] =
- new JavaRDD(rdd.mapPartitionsWithIndex(((a, b) => f(a, asJavaIterator(b))),
+ new JavaRDD(rdd.mapPartitionsWithIndex((a, b) => f.call(a, b.asJava).asScala,
preservesPartitioning)(fakeClassTag))(fakeClassTag)
/**
@@ -153,7 +152,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
*/
def mapPartitions[U](f: FlatMapFunction[java.util.Iterator[T], U]): JavaRDD[U] = {
def fn: (Iterator[T]) => Iterator[U] = {
- (x: Iterator[T]) => asScalaIterator(f.call(asJavaIterator(x)).iterator())
+ (x: Iterator[T]) => f.call(x.asJava).iterator().asScala
}
JavaRDD.fromRDD(rdd.mapPartitions(fn)(fakeClassTag[U]))(fakeClassTag[U])
}
@@ -164,7 +163,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
def mapPartitions[U](f: FlatMapFunction[java.util.Iterator[T], U],
preservesPartitioning: Boolean): JavaRDD[U] = {
def fn: (Iterator[T]) => Iterator[U] = {
- (x: Iterator[T]) => asScalaIterator(f.call(asJavaIterator(x)).iterator())
+ (x: Iterator[T]) => f.call(x.asJava).iterator().asScala
}
JavaRDD.fromRDD(
rdd.mapPartitions(fn, preservesPartitioning)(fakeClassTag[U]))(fakeClassTag[U])
@@ -175,7 +174,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
*/
def mapPartitionsToDouble(f: DoubleFlatMapFunction[java.util.Iterator[T]]): JavaDoubleRDD = {
def fn: (Iterator[T]) => Iterator[jl.Double] = {
- (x: Iterator[T]) => asScalaIterator(f.call(asJavaIterator(x)).iterator())
+ (x: Iterator[T]) => f.call(x.asJava).iterator().asScala
}
new JavaDoubleRDD(rdd.mapPartitions(fn).map((x: jl.Double) => x.doubleValue()))
}
@@ -186,7 +185,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
def mapPartitionsToPair[K2, V2](f: PairFlatMapFunction[java.util.Iterator[T], K2, V2]):
JavaPairRDD[K2, V2] = {
def fn: (Iterator[T]) => Iterator[(K2, V2)] = {
- (x: Iterator[T]) => asScalaIterator(f.call(asJavaIterator(x)).iterator())
+ (x: Iterator[T]) => f.call(x.asJava).iterator().asScala
}
JavaPairRDD.fromRDD(rdd.mapPartitions(fn))(fakeClassTag[K2], fakeClassTag[V2])
}
@@ -197,7 +196,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
def mapPartitionsToDouble(f: DoubleFlatMapFunction[java.util.Iterator[T]],
preservesPartitioning: Boolean): JavaDoubleRDD = {
def fn: (Iterator[T]) => Iterator[jl.Double] = {
- (x: Iterator[T]) => asScalaIterator(f.call(asJavaIterator(x)).iterator())
+ (x: Iterator[T]) => f.call(x.asJava).iterator().asScala
}
new JavaDoubleRDD(rdd.mapPartitions(fn, preservesPartitioning)
.map(x => x.doubleValue()))
@@ -209,7 +208,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
def mapPartitionsToPair[K2, V2](f: PairFlatMapFunction[java.util.Iterator[T], K2, V2],
preservesPartitioning: Boolean): JavaPairRDD[K2, V2] = {
def fn: (Iterator[T]) => Iterator[(K2, V2)] = {
- (x: Iterator[T]) => asScalaIterator(f.call(asJavaIterator(x)).iterator())
+ (x: Iterator[T]) => f.call(x.asJava).iterator().asScala
}
JavaPairRDD.fromRDD(
rdd.mapPartitions(fn, preservesPartitioning))(fakeClassTag[K2], fakeClassTag[V2])
@@ -219,14 +218,14 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
* Applies a function f to each partition of this RDD.
*/
def foreachPartition(f: VoidFunction[java.util.Iterator[T]]) {
- rdd.foreachPartition((x => f.call(asJavaIterator(x))))
+ rdd.foreachPartition((x => f.call(x.asJava)))
}
/**
* Return an RDD created by coalescing all elements within each partition into an array.
*/
def glom(): JavaRDD[JList[T]] =
- new JavaRDD(rdd.glom().map(x => new java.util.ArrayList[T](x.toSeq)))
+ new JavaRDD(rdd.glom().map(_.toSeq.asJava))
/**
* Return the Cartesian product of this RDD and another one, that is, the RDD of all pairs of
@@ -266,13 +265,13 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
* Return an RDD created by piping elements to a forked external process.
*/
def pipe(command: JList[String]): JavaRDD[String] =
- rdd.pipe(asScalaBuffer(command))
+ rdd.pipe(command.asScala)
/**
* Return an RDD created by piping elements to a forked external process.
*/
def pipe(command: JList[String], env: java.util.Map[String, String]): JavaRDD[String] =
- rdd.pipe(asScalaBuffer(command), mapAsScalaMap(env))
+ rdd.pipe(command.asScala, env.asScala)
/**
* Zips this RDD with another one, returning key-value pairs with the first element in each RDD,
@@ -294,8 +293,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
other: JavaRDDLike[U, _],
f: FlatMapFunction2[java.util.Iterator[T], java.util.Iterator[U], V]): JavaRDD[V] = {
def fn: (Iterator[T], Iterator[U]) => Iterator[V] = {
- (x: Iterator[T], y: Iterator[U]) => asScalaIterator(
- f.call(asJavaIterator(x), asJavaIterator(y)).iterator())
+ (x: Iterator[T], y: Iterator[U]) => f.call(x.asJava, y.asJava).iterator().asScala
}
JavaRDD.fromRDD(
rdd.zipPartitions(other.rdd)(fn)(other.classTag, fakeClassTag[V]))(fakeClassTag[V])
@@ -333,22 +331,16 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
/**
* Return an array that contains all of the elements in this RDD.
*/
- def collect(): JList[T] = {
- import scala.collection.JavaConversions._
- val arr: java.util.Collection[T] = rdd.collect().toSeq
- new java.util.ArrayList(arr)
- }
+ def collect(): JList[T] =
+ rdd.collect().toSeq.asJava
/**
* Return an iterator that contains all of the elements in this RDD.
*
* The iterator will consume as much memory as the largest partition in this RDD.
*/
- def toLocalIterator(): JIterator[T] = {
- import scala.collection.JavaConversions._
- rdd.toLocalIterator
- }
-
+ def toLocalIterator(): JIterator[T] =
+ asJavaIteratorConverter(rdd.toLocalIterator).asJava
/**
* Return an array that contains all of the elements in this RDD.
@@ -363,9 +355,8 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
def collectPartitions(partitionIds: Array[Int]): Array[JList[T]] = {
// This is useful for implementing `take` from other language frontends
// like Python where the data is serialized.
- import scala.collection.JavaConversions._
val res = context.runJob(rdd, (it: Iterator[T]) => it.toArray, partitionIds)
- res.map(x => new java.util.ArrayList(x.toSeq)).toArray
+ res.map(_.toSeq.asJava)
}
/**
@@ -489,20 +480,14 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
* it will be slow if a lot of partitions are required. In that case, use collect() to get the
* whole RDD instead.
*/
- def take(num: Int): JList[T] = {
- import scala.collection.JavaConversions._
- val arr: java.util.Collection[T] = rdd.take(num).toSeq
- new java.util.ArrayList(arr)
- }
+ def take(num: Int): JList[T] =
+ rdd.take(num).toSeq.asJava
def takeSample(withReplacement: Boolean, num: Int): JList[T] =
takeSample(withReplacement, num, Utils.random.nextLong)
- def takeSample(withReplacement: Boolean, num: Int, seed: Long): JList[T] = {
- import scala.collection.JavaConversions._
- val arr: java.util.Collection[T] = rdd.takeSample(withReplacement, num, seed).toSeq
- new java.util.ArrayList(arr)
- }
+ def takeSample(withReplacement: Boolean, num: Int, seed: Long): JList[T] =
+ rdd.takeSample(withReplacement, num, seed).toSeq.asJava
/**
* Return the first element in this RDD.
@@ -582,10 +567,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
* @return an array of top elements
*/
def top(num: Int, comp: Comparator[T]): JList[T] = {
- import scala.collection.JavaConversions._
- val topElems = rdd.top(num)(Ordering.comparatorToOrdering(comp))
- val arr: java.util.Collection[T] = topElems.toSeq
- new java.util.ArrayList(arr)
+ rdd.top(num)(Ordering.comparatorToOrdering(comp)).toSeq.asJava
}
/**
@@ -607,10 +589,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
* @return an array of top elements
*/
def takeOrdered(num: Int, comp: Comparator[T]): JList[T] = {
- import scala.collection.JavaConversions._
- val topElems = rdd.takeOrdered(num)(Ordering.comparatorToOrdering(comp))
- val arr: java.util.Collection[T] = topElems.toSeq
- new java.util.ArrayList(arr)
+ rdd.takeOrdered(num)(Ordering.comparatorToOrdering(comp)).toSeq.asJava
}
/**
@@ -696,7 +675,7 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable {
* applies a function f to each partition of this RDD.
*/
def foreachPartitionAsync(f: VoidFunction[java.util.Iterator[T]]): JavaFutureAction[Void] = {
- new JavaFutureActionWrapper[Unit, Void](rdd.foreachPartitionAsync(x => f.call(x)),
+ new JavaFutureActionWrapper[Unit, Void](rdd.foreachPartitionAsync(x => f.call(x.asJava)),
{ x => null.asInstanceOf[Void] })
}
}
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
index 02e49a853c5f78e6304c0d9654f75d81e63cbf8c..609496ccdfef15ee564a75ed15698115ebb47d4d 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
@@ -21,8 +21,7 @@ import java.io.Closeable
import java.util
import java.util.{Map => JMap}
-import scala.collection.JavaConversions
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.language.implicitConversions
import scala.reflect.ClassTag
@@ -104,7 +103,7 @@ class JavaSparkContext(val sc: SparkContext)
*/
def this(master: String, appName: String, sparkHome: String, jars: Array[String],
environment: JMap[String, String]) =
- this(new SparkContext(master, appName, sparkHome, jars.toSeq, environment, Map()))
+ this(new SparkContext(master, appName, sparkHome, jars.toSeq, environment.asScala, Map()))
private[spark] val env = sc.env
@@ -118,7 +117,7 @@ class JavaSparkContext(val sc: SparkContext)
def appName: String = sc.appName
- def jars: util.List[String] = sc.jars
+ def jars: util.List[String] = sc.jars.asJava
def startTime: java.lang.Long = sc.startTime
@@ -142,7 +141,7 @@ class JavaSparkContext(val sc: SparkContext)
/** Distribute a local Scala collection to form an RDD. */
def parallelize[T](list: java.util.List[T], numSlices: Int): JavaRDD[T] = {
implicit val ctag: ClassTag[T] = fakeClassTag
- sc.parallelize(JavaConversions.asScalaBuffer(list), numSlices)
+ sc.parallelize(list.asScala, numSlices)
}
/** Get an RDD that has no partitions or elements. */
@@ -161,7 +160,7 @@ class JavaSparkContext(val sc: SparkContext)
: JavaPairRDD[K, V] = {
implicit val ctagK: ClassTag[K] = fakeClassTag
implicit val ctagV: ClassTag[V] = fakeClassTag
- JavaPairRDD.fromRDD(sc.parallelize(JavaConversions.asScalaBuffer(list), numSlices))
+ JavaPairRDD.fromRDD(sc.parallelize(list.asScala, numSlices))
}
/** Distribute a local Scala collection to form an RDD. */
@@ -170,8 +169,7 @@ class JavaSparkContext(val sc: SparkContext)
/** Distribute a local Scala collection to form an RDD. */
def parallelizeDoubles(list: java.util.List[java.lang.Double], numSlices: Int): JavaDoubleRDD =
- JavaDoubleRDD.fromRDD(sc.parallelize(JavaConversions.asScalaBuffer(list).map(_.doubleValue()),
- numSlices))
+ JavaDoubleRDD.fromRDD(sc.parallelize(list.asScala.map(_.doubleValue()), numSlices))
/** Distribute a local Scala collection to form an RDD. */
def parallelizeDoubles(list: java.util.List[java.lang.Double]): JavaDoubleRDD =
@@ -519,7 +517,7 @@ class JavaSparkContext(val sc: SparkContext)
/** Build the union of two or more RDDs. */
override def union[T](first: JavaRDD[T], rest: java.util.List[JavaRDD[T]]): JavaRDD[T] = {
- val rdds: Seq[RDD[T]] = (Seq(first) ++ asScalaBuffer(rest)).map(_.rdd)
+ val rdds: Seq[RDD[T]] = (Seq(first) ++ rest.asScala).map(_.rdd)
implicit val ctag: ClassTag[T] = first.classTag
sc.union(rdds)
}
@@ -527,7 +525,7 @@ class JavaSparkContext(val sc: SparkContext)
/** Build the union of two or more RDDs. */
override def union[K, V](first: JavaPairRDD[K, V], rest: java.util.List[JavaPairRDD[K, V]])
: JavaPairRDD[K, V] = {
- val rdds: Seq[RDD[(K, V)]] = (Seq(first) ++ asScalaBuffer(rest)).map(_.rdd)
+ val rdds: Seq[RDD[(K, V)]] = (Seq(first) ++ rest.asScala).map(_.rdd)
implicit val ctag: ClassTag[(K, V)] = first.classTag
implicit val ctagK: ClassTag[K] = first.kClassTag
implicit val ctagV: ClassTag[V] = first.vClassTag
@@ -536,7 +534,7 @@ class JavaSparkContext(val sc: SparkContext)
/** Build the union of two or more RDDs. */
override def union(first: JavaDoubleRDD, rest: java.util.List[JavaDoubleRDD]): JavaDoubleRDD = {
- val rdds: Seq[RDD[Double]] = (Seq(first) ++ asScalaBuffer(rest)).map(_.srdd)
+ val rdds: Seq[RDD[Double]] = (Seq(first) ++ rest.asScala).map(_.srdd)
new JavaDoubleRDD(sc.union(rdds))
}
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonHadoopUtil.scala b/core/src/main/scala/org/apache/spark/api/python/PythonHadoopUtil.scala
index b959b683d16742566835b7b44eb1175ff7b43f1c..a7dfa1d257cf2a67d577825009e3a7b79de4f815 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonHadoopUtil.scala
@@ -17,15 +17,17 @@
package org.apache.spark.api.python
-import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.rdd.RDD
-import org.apache.spark.util.{SerializableConfiguration, Utils}
-import org.apache.spark.{Logging, SparkException}
+import scala.collection.JavaConverters._
+import scala.util.{Failure, Success, Try}
+
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io._
-import scala.util.{Failure, Success, Try}
-import org.apache.spark.annotation.Experimental
+import org.apache.spark.{Logging, SparkException}
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.rdd.RDD
+import org.apache.spark.util.{SerializableConfiguration, Utils}
/**
* :: Experimental ::
@@ -68,7 +70,6 @@ private[python] class WritableToJavaConverter(
* object representation
*/
private def convertWritable(writable: Writable): Any = {
- import collection.JavaConversions._
writable match {
case iw: IntWritable => iw.get()
case dw: DoubleWritable => dw.get()
@@ -89,9 +90,7 @@ private[python] class WritableToJavaConverter(
aw.get().map(convertWritable(_))
case mw: MapWritable =>
val map = new java.util.HashMap[Any, Any]()
- mw.foreach { case (k, v) =>
- map.put(convertWritable(k), convertWritable(v))
- }
+ mw.asScala.foreach { case (k, v) => map.put(convertWritable(k), convertWritable(v)) }
map
case w: Writable => WritableUtils.clone(w, conf.value.value)
case other => other
@@ -122,7 +121,6 @@ private[python] class JavaToWritableConverter extends Converter[Any, Writable] {
* supported out-of-the-box.
*/
private def convertToWritable(obj: Any): Writable = {
- import collection.JavaConversions._
obj match {
case i: java.lang.Integer => new IntWritable(i)
case d: java.lang.Double => new DoubleWritable(d)
@@ -134,7 +132,7 @@ private[python] class JavaToWritableConverter extends Converter[Any, Writable] {
case null => NullWritable.get()
case map: java.util.Map[_, _] =>
val mapWritable = new MapWritable()
- map.foreach { case (k, v) =>
+ map.asScala.foreach { case (k, v) =>
mapWritable.put(convertToWritable(k), convertToWritable(v))
}
mapWritable
@@ -161,9 +159,8 @@ private[python] object PythonHadoopUtil {
* Convert a [[java.util.Map]] of properties to a [[org.apache.hadoop.conf.Configuration]]
*/
def mapToConf(map: java.util.Map[String, String]): Configuration = {
- import collection.JavaConversions._
val conf = new Configuration()
- map.foreach{ case (k, v) => conf.set(k, v) }
+ map.asScala.foreach { case (k, v) => conf.set(k, v) }
conf
}
@@ -172,9 +169,8 @@ private[python] object PythonHadoopUtil {
* any matching keys in left
*/
def mergeConfs(left: Configuration, right: Configuration): Configuration = {
- import collection.JavaConversions._
val copy = new Configuration(left)
- right.iterator().foreach(entry => copy.set(entry.getKey, entry.getValue))
+ right.asScala.foreach(entry => copy.set(entry.getKey, entry.getValue))
copy
}
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
index 2a56bf28d70270693375e9e360a8005a9326b86f..b4d152b3366022534296e2bb2f62273ed752e0cd 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
@@ -21,7 +21,7 @@ import java.io._
import java.net._
import java.util.{Collections, ArrayList => JArrayList, List => JList, Map => JMap}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.language.existentials
@@ -66,11 +66,11 @@ private[spark] class PythonRDD(
val env = SparkEnv.get
val localdir = env.blockManager.diskBlockManager.localDirs.map(
f => f.getPath()).mkString(",")
- envVars += ("SPARK_LOCAL_DIRS" -> localdir) // it's also used in monitor thread
+ envVars.put("SPARK_LOCAL_DIRS", localdir) // it's also used in monitor thread
if (reuse_worker) {
- envVars += ("SPARK_REUSE_WORKER" -> "1")
+ envVars.put("SPARK_REUSE_WORKER", "1")
}
- val worker: Socket = env.createPythonWorker(pythonExec, envVars.toMap)
+ val worker: Socket = env.createPythonWorker(pythonExec, envVars.asScala.toMap)
// Whether is the worker released into idle pool
@volatile var released = false
@@ -150,7 +150,7 @@ private[spark] class PythonRDD(
// Check whether the worker is ready to be re-used.
if (stream.readInt() == SpecialLengths.END_OF_STREAM) {
if (reuse_worker) {
- env.releasePythonWorker(pythonExec, envVars.toMap, worker)
+ env.releasePythonWorker(pythonExec, envVars.asScala.toMap, worker)
released = true
}
}
@@ -217,13 +217,13 @@ private[spark] class PythonRDD(
// sparkFilesDir
PythonRDD.writeUTF(SparkFiles.getRootDirectory, dataOut)
// Python includes (*.zip and *.egg files)
- dataOut.writeInt(pythonIncludes.length)
- for (include <- pythonIncludes) {
+ dataOut.writeInt(pythonIncludes.size())
+ for (include <- pythonIncludes.asScala) {
PythonRDD.writeUTF(include, dataOut)
}
// Broadcast variables
val oldBids = PythonRDD.getWorkerBroadcasts(worker)
- val newBids = broadcastVars.map(_.id).toSet
+ val newBids = broadcastVars.asScala.map(_.id).toSet
// number of different broadcasts
val toRemove = oldBids.diff(newBids)
val cnt = toRemove.size + newBids.diff(oldBids).size
@@ -233,7 +233,7 @@ private[spark] class PythonRDD(
dataOut.writeLong(- bid - 1) // bid >= 0
oldBids.remove(bid)
}
- for (broadcast <- broadcastVars) {
+ for (broadcast <- broadcastVars.asScala) {
if (!oldBids.contains(broadcast.id)) {
// send new broadcast
dataOut.writeLong(broadcast.id)
@@ -287,7 +287,7 @@ private[spark] class PythonRDD(
if (!context.isCompleted) {
try {
logWarning("Incomplete task interrupted: Attempting to kill Python Worker")
- env.destroyPythonWorker(pythonExec, envVars.toMap, worker)
+ env.destroyPythonWorker(pythonExec, envVars.asScala.toMap, worker)
} catch {
case e: Exception =>
logError("Exception when trying to kill worker", e)
@@ -358,10 +358,10 @@ private[spark] object PythonRDD extends Logging {
type ByteArray = Array[Byte]
type UnrolledPartition = Array[ByteArray]
val allPartitions: Array[UnrolledPartition] =
- sc.runJob(rdd, (x: Iterator[ByteArray]) => x.toArray, partitions)
+ sc.runJob(rdd, (x: Iterator[ByteArray]) => x.toArray, partitions.asScala)
val flattenedPartition: UnrolledPartition = Array.concat(allPartitions: _*)
serveIterator(flattenedPartition.iterator,
- s"serve RDD ${rdd.id} with partitions ${partitions.mkString(",")}")
+ s"serve RDD ${rdd.id} with partitions ${partitions.asScala.mkString(",")}")
}
/**
@@ -819,7 +819,7 @@ private class PythonAccumulatorParam(@transient serverHost: String, serverPort:
val in = socket.getInputStream
val out = new DataOutputStream(new BufferedOutputStream(socket.getOutputStream, bufferSize))
out.writeInt(val2.size)
- for (array <- val2) {
+ for (array <- val2.asScala) {
out.writeInt(array.length)
out.write(array)
}
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
index 90dacaeb9342996854d46f498d6208595f97344e..31e534f160eeb8195ad48669541c45c344d173d6 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
@@ -17,10 +17,10 @@
package org.apache.spark.api.python
-import java.io.{File}
+import java.io.File
import java.util.{List => JList}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.SparkContext
@@ -51,7 +51,14 @@ private[spark] object PythonUtils {
* Convert list of T into seq of T (for calling API with varargs)
*/
def toSeq[T](vs: JList[T]): Seq[T] = {
- vs.toList.toSeq
+ vs.asScala
+ }
+
+ /**
+ * Convert list of T into a (Scala) List of T
+ */
+ def toList[T](vs: JList[T]): List[T] = {
+ vs.asScala.toList
}
/**
@@ -65,6 +72,6 @@ private[spark] object PythonUtils {
* Convert java map of K, V into Map of K, V (for calling API with varargs)
*/
def toScalaMap[K, V](jm: java.util.Map[K, V]): Map[K, V] = {
- jm.toMap
+ jm.asScala.toMap
}
}
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala b/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
index e314408c067e9ee42b4d69843aaef9a6b9cf2951..7039b734d2e4015f42fc0177f53f1470e33bc5ef 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonWorkerFactory.scala
@@ -19,9 +19,10 @@ package org.apache.spark.api.python
import java.io.{DataOutputStream, DataInputStream, InputStream, OutputStreamWriter}
import java.net.{InetAddress, ServerSocket, Socket, SocketException}
+import java.util.Arrays
import scala.collection.mutable
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark._
import org.apache.spark.util.{RedirectThread, Utils}
@@ -108,9 +109,9 @@ private[spark] class PythonWorkerFactory(pythonExec: String, envVars: Map[String
serverSocket = new ServerSocket(0, 1, InetAddress.getByAddress(Array(127, 0, 0, 1)))
// Create and start the worker
- val pb = new ProcessBuilder(Seq(pythonExec, "-m", "pyspark.worker"))
+ val pb = new ProcessBuilder(Arrays.asList(pythonExec, "-m", "pyspark.worker"))
val workerEnv = pb.environment()
- workerEnv.putAll(envVars)
+ workerEnv.putAll(envVars.asJava)
workerEnv.put("PYTHONPATH", pythonPath)
// This is equivalent to setting the -u flag; we use it because ipython doesn't support -u:
workerEnv.put("PYTHONUNBUFFERED", "YES")
@@ -151,9 +152,9 @@ private[spark] class PythonWorkerFactory(pythonExec: String, envVars: Map[String
try {
// Create and start the daemon
- val pb = new ProcessBuilder(Seq(pythonExec, "-m", "pyspark.daemon"))
+ val pb = new ProcessBuilder(Arrays.asList(pythonExec, "-m", "pyspark.daemon"))
val workerEnv = pb.environment()
- workerEnv.putAll(envVars)
+ workerEnv.putAll(envVars.asJava)
workerEnv.put("PYTHONPATH", pythonPath)
// This is equivalent to setting the -u flag; we use it because ipython doesn't support -u:
workerEnv.put("PYTHONUNBUFFERED", "YES")
diff --git a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
index 1f1debcf84ad4fd17ce63bd02f2dc7cbbb438711..fd27276e70bfe106c24a43c20aa4516fbfb27425 100644
--- a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala
@@ -22,7 +22,6 @@ import java.util.{ArrayList => JArrayList}
import org.apache.spark.api.java.JavaRDD
-import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.util.Failure
@@ -214,7 +213,7 @@ private[spark] object SerDeUtil extends Logging {
new AutoBatchedPickler(cleaned)
} else {
val pickle = new Pickler
- cleaned.grouped(batchSize).map(batched => pickle.dumps(seqAsJavaList(batched)))
+ cleaned.grouped(batchSize).map(batched => pickle.dumps(batched.asJava))
}
}
}
diff --git a/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala b/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala
index 8f30ff9202c838d1587494df7d591b75f8a987fc..ee1fb056f0d9636e20c97c72d935e4dce981d47a 100644
--- a/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/WriteInputFormatTestDataGenerator.scala
@@ -20,6 +20,8 @@ package org.apache.spark.api.python
import java.io.{DataOutput, DataInput}
import java.{util => ju}
+import scala.collection.JavaConverters._
+
import com.google.common.base.Charsets.UTF_8
import org.apache.hadoop.io._
@@ -62,10 +64,9 @@ private[python] class TestInputKeyConverter extends Converter[Any, Any] {
}
private[python] class TestInputValueConverter extends Converter[Any, Any] {
- import collection.JavaConversions._
override def convert(obj: Any): ju.List[Double] = {
val m = obj.asInstanceOf[MapWritable]
- seqAsJavaList(m.keySet.map(w => w.asInstanceOf[DoubleWritable].get()).toSeq)
+ m.keySet.asScala.map(_.asInstanceOf[DoubleWritable].get()).toSeq.asJava
}
}
@@ -76,9 +77,8 @@ private[python] class TestOutputKeyConverter extends Converter[Any, Any] {
}
private[python] class TestOutputValueConverter extends Converter[Any, Any] {
- import collection.JavaConversions._
override def convert(obj: Any): DoubleWritable = {
- new DoubleWritable(obj.asInstanceOf[java.util.Map[Double, _]].keySet().head)
+ new DoubleWritable(obj.asInstanceOf[java.util.Map[Double, _]].keySet().iterator().next())
}
}
diff --git a/core/src/main/scala/org/apache/spark/api/r/RRDD.scala b/core/src/main/scala/org/apache/spark/api/r/RRDD.scala
index 1cf2824f862ee0a7d645783045d4c603cf1c63aa..9d5bbb5d609f37f4020c9e0a5da3c287df273982 100644
--- a/core/src/main/scala/org/apache/spark/api/r/RRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/RRDD.scala
@@ -19,9 +19,10 @@ package org.apache.spark.api.r
import java.io._
import java.net.{InetAddress, ServerSocket}
+import java.util.Arrays
import java.util.{Map => JMap}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.io.Source
import scala.reflect.ClassTag
import scala.util.Try
@@ -365,11 +366,11 @@ private[r] object RRDD {
sparkConf.setIfMissing("spark.master", "local")
}
- for ((name, value) <- sparkEnvirMap) {
- sparkConf.set(name.asInstanceOf[String], value.asInstanceOf[String])
+ for ((name, value) <- sparkEnvirMap.asScala) {
+ sparkConf.set(name.toString, value.toString)
}
- for ((name, value) <- sparkExecutorEnvMap) {
- sparkConf.setExecutorEnv(name.asInstanceOf[String], value.asInstanceOf[String])
+ for ((name, value) <- sparkExecutorEnvMap.asScala) {
+ sparkConf.setExecutorEnv(name.toString, value.toString)
}
val jsc = new JavaSparkContext(sparkConf)
@@ -395,7 +396,7 @@ private[r] object RRDD {
val rOptions = "--vanilla"
val rLibDir = RUtils.sparkRPackagePath(isDriver = false)
val rExecScript = rLibDir + "/SparkR/worker/" + script
- val pb = new ProcessBuilder(List(rCommand, rOptions, rExecScript))
+ val pb = new ProcessBuilder(Arrays.asList(rCommand, rOptions, rExecScript))
// Unset the R_TESTS environment variable for workers.
// This is set by R CMD check as startup.Rs
// (http://svn.r-project.org/R/trunk/src/library/tools/R/testing.R)
diff --git a/core/src/main/scala/org/apache/spark/api/r/RUtils.scala b/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
index 427b2bc7cbcbb90fa031e54f5fe0e22789e13759..9e807cc52f18ceecb85892789bde22166e948bf0 100644
--- a/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
@@ -18,8 +18,7 @@
package org.apache.spark.api.r
import java.io.File
-
-import scala.collection.JavaConversions._
+import java.util.Arrays
import org.apache.spark.{SparkEnv, SparkException}
@@ -68,7 +67,7 @@ private[spark] object RUtils {
/** Check if R is installed before running tests that use R commands. */
def isRInstalled: Boolean = {
try {
- val builder = new ProcessBuilder(Seq("R", "--version"))
+ val builder = new ProcessBuilder(Arrays.asList("R", "--version"))
builder.start().waitFor() == 0
} catch {
case e: Exception => false
diff --git a/core/src/main/scala/org/apache/spark/api/r/SerDe.scala b/core/src/main/scala/org/apache/spark/api/r/SerDe.scala
index 3c89f24473744137474f5d998e5faae0bdd3a386..dbbbcf40c1e963c97110e84b7d9d8aa71f50c177 100644
--- a/core/src/main/scala/org/apache/spark/api/r/SerDe.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/SerDe.scala
@@ -20,7 +20,7 @@ package org.apache.spark.api.r
import java.io.{DataInputStream, DataOutputStream}
import java.sql.{Timestamp, Date, Time}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
/**
* Utility functions to serialize, deserialize objects to / from R
@@ -165,7 +165,7 @@ private[spark] object SerDe {
val valueType = readObjectType(in)
readTypedObject(in, valueType)
})
- mapAsJavaMap(keys.zip(values).toMap)
+ keys.zip(values).toMap.asJava
} else {
new java.util.HashMap[Object, Object]()
}
diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
index a0c9b5e63c7441eb63e331e0ea215d84afcc36a4..7e3764d802fe11eaff9d0de5c3715f15cdea8849 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
@@ -20,7 +20,7 @@ package org.apache.spark.broadcast
import java.io._
import java.nio.ByteBuffer
-import scala.collection.JavaConversions.asJavaEnumeration
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import scala.util.Random
@@ -210,7 +210,7 @@ private object TorrentBroadcast extends Logging {
compressionCodec: Option[CompressionCodec]): T = {
require(blocks.nonEmpty, "Cannot unblockify an empty array of blocks")
val is = new SequenceInputStream(
- asJavaEnumeration(blocks.iterator.map(block => new ByteBufferInputStream(block))))
+ blocks.iterator.map(new ByteBufferInputStream(_)).asJavaEnumeration)
val in: InputStream = compressionCodec.map(c => c.compressedInputStream(is)).getOrElse(is)
val ser = serializer.newInstance()
val serIn = ser.deserializeStream(in)
diff --git a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
index 22ef701d833b2ece8c14b71ea21cb12161946be4..6840a3ae831f0d0bf8ba0e640760878d15fe7008 100644
--- a/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/ExternalShuffleService.scala
@@ -19,13 +19,13 @@ package org.apache.spark.deploy
import java.util.concurrent.CountDownLatch
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.{Logging, SparkConf, SecurityManager}
import org.apache.spark.network.TransportContext
import org.apache.spark.network.netty.SparkTransportConf
import org.apache.spark.network.sasl.SaslServerBootstrap
-import org.apache.spark.network.server.TransportServer
+import org.apache.spark.network.server.{TransportServerBootstrap, TransportServer}
import org.apache.spark.network.shuffle.ExternalShuffleBlockHandler
import org.apache.spark.network.util.TransportConf
import org.apache.spark.util.Utils
@@ -67,13 +67,13 @@ class ExternalShuffleService(sparkConf: SparkConf, securityManager: SecurityMana
def start() {
require(server == null, "Shuffle server already started")
logInfo(s"Starting shuffle service on port $port with useSasl = $useSasl")
- val bootstraps =
+ val bootstraps: Seq[TransportServerBootstrap] =
if (useSasl) {
Seq(new SaslServerBootstrap(transportConf, securityManager))
} else {
Nil
}
- server = transportContext.createServer(port, bootstraps)
+ server = transportContext.createServer(port, bootstraps.asJava)
}
/** Clean up all shuffle files associated with an application that has exited. */
diff --git a/core/src/main/scala/org/apache/spark/deploy/PythonRunner.scala b/core/src/main/scala/org/apache/spark/deploy/PythonRunner.scala
index 23d01e9cbb9f990b1b8a9cbbbcb485fdee9c948f..d85327603f64df6213d07adb9059ca27be744c66 100644
--- a/core/src/main/scala/org/apache/spark/deploy/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/PythonRunner.scala
@@ -21,7 +21,7 @@ import java.net.URI
import java.io.File
import scala.collection.mutable.ArrayBuffer
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.util.Try
import org.apache.spark.SparkUserAppException
@@ -71,7 +71,7 @@ object PythonRunner {
val pythonPath = PythonUtils.mergePythonPaths(pathElements: _*)
// Launch Python process
- val builder = new ProcessBuilder(Seq(pythonExec, formattedPythonFile) ++ otherArgs)
+ val builder = new ProcessBuilder((Seq(pythonExec, formattedPythonFile) ++ otherArgs).asJava)
val env = builder.environment()
env.put("PYTHONPATH", pythonPath)
// This is equivalent to setting the -u flag; we use it because ipython doesn't support -u:
diff --git a/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala b/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala
index ed1e972955679cb44cf2bede7a18856b33942f30..4b28866dcaa7ca2594901c19352a58363b29074f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/RPackageUtils.scala
@@ -22,7 +22,7 @@ import java.util.jar.JarFile
import java.util.logging.Level
import java.util.zip.{ZipEntry, ZipOutputStream}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import com.google.common.io.{ByteStreams, Files}
@@ -110,7 +110,7 @@ private[deploy] object RPackageUtils extends Logging {
print(s"Building R package with the command: $installCmd", printStream)
}
try {
- val builder = new ProcessBuilder(installCmd)
+ val builder = new ProcessBuilder(installCmd.asJava)
builder.redirectErrorStream(true)
val env = builder.environment()
env.clear()
diff --git a/core/src/main/scala/org/apache/spark/deploy/RRunner.scala b/core/src/main/scala/org/apache/spark/deploy/RRunner.scala
index c0cab22fa8252409ab549dc7a3e83beb6125c90d..05b954ce36998fceaf0dfddeeab34a0953f7d271 100644
--- a/core/src/main/scala/org/apache/spark/deploy/RRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/RRunner.scala
@@ -20,7 +20,7 @@ package org.apache.spark.deploy
import java.io._
import java.util.concurrent.{Semaphore, TimeUnit}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.hadoop.fs.Path
@@ -68,7 +68,7 @@ object RRunner {
if (initialized.tryAcquire(backendTimeout, TimeUnit.SECONDS)) {
// Launch R
val returnCode = try {
- val builder = new ProcessBuilder(Seq(rCommand, rFileNormalized) ++ otherArgs)
+ val builder = new ProcessBuilder((Seq(rCommand, rFileNormalized) ++ otherArgs).asJava)
val env = builder.environment()
env.put("EXISTING_SPARKR_BACKEND_PORT", sparkRBackendPort.toString)
val rPackageDir = RUtils.sparkRPackagePath(isDriver = true)
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkCuratorUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkCuratorUtil.scala
index b8d399354022028a7f76a45222ef820a5fe5144e..8d5e716e6aea4140ac895cd52285317ef3e4efb9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkCuratorUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkCuratorUtil.scala
@@ -17,7 +17,7 @@
package org.apache.spark.deploy
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory}
import org.apache.curator.retry.ExponentialBackoffRetry
@@ -57,7 +57,7 @@ private[spark] object SparkCuratorUtil extends Logging {
def deleteRecursive(zk: CuratorFramework, path: String) {
if (zk.checkExists().forPath(path) != null) {
- for (child <- zk.getChildren.forPath(path)) {
+ for (child <- zk.getChildren.forPath(path).asScala) {
zk.delete().forPath(path + "/" + child)
}
zk.delete().forPath(path)
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index dda4216c7efe2a935e63feea9036eca1d05f242b..f7723ef5bde4c1d2e513095170bdcc30e7cbec30 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -22,7 +22,7 @@ import java.lang.reflect.Method
import java.security.PrivilegedExceptionAction
import java.util.{Arrays, Comparator}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.control.NonFatal
@@ -71,7 +71,7 @@ class SparkHadoopUtil extends Logging {
}
def transferCredentials(source: UserGroupInformation, dest: UserGroupInformation) {
- for (token <- source.getTokens()) {
+ for (token <- source.getTokens.asScala) {
dest.addToken(token)
}
}
@@ -175,8 +175,8 @@ class SparkHadoopUtil extends Logging {
}
private def getFileSystemThreadStatistics(): Seq[AnyRef] = {
- val stats = FileSystem.getAllStatistics()
- stats.map(Utils.invoke(classOf[Statistics], _, "getThreadStatistics"))
+ FileSystem.getAllStatistics.asScala.map(
+ Utils.invoke(classOf[Statistics], _, "getThreadStatistics"))
}
private def getFileSystemThreadStatisticsMethod(methodName: String): Method = {
@@ -306,12 +306,13 @@ class SparkHadoopUtil extends Logging {
val renewalInterval =
sparkConf.getLong("spark.yarn.token.renewal.interval", (24 hours).toMillis)
- credentials.getAllTokens.filter(_.getKind == DelegationTokenIdentifier.HDFS_DELEGATION_KIND)
+ credentials.getAllTokens.asScala
+ .filter(_.getKind == DelegationTokenIdentifier.HDFS_DELEGATION_KIND)
.map { t =>
- val identifier = new DelegationTokenIdentifier()
- identifier.readFields(new DataInputStream(new ByteArrayInputStream(t.getIdentifier)))
- (identifier.getIssueDate + fraction * renewalInterval).toLong - now
- }.foldLeft(0L)(math.max)
+ val identifier = new DelegationTokenIdentifier()
+ identifier.readFields(new DataInputStream(new ByteArrayInputStream(t.getIdentifier)))
+ (identifier.getIssueDate + fraction * renewalInterval).toLong - now
+ }.foldLeft(0L)(math.max)
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index 3f3c6627c21fbcc5bc8f651715119f7485b28833..18a1c52ae53fb0c0a75ebcb9747971c3471d1e5c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -23,7 +23,7 @@ import java.net.URI
import java.util.{List => JList}
import java.util.jar.JarFile
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, HashMap}
import scala.io.Source
@@ -94,7 +94,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
// Set parameters from command line arguments
try {
- parse(args.toList)
+ parse(args.asJava)
} catch {
case e: IllegalArgumentException =>
SparkSubmit.printErrorAndExit(e.getMessage())
@@ -458,7 +458,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
}
override protected def handleExtraArgs(extra: JList[String]): Unit = {
- childArgs ++= extra
+ childArgs ++= extra.asScala
}
private def printUsageAndExit(exitCode: Int, unknownParam: Any = null): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
index 563831cc6b8dd532fda848e6a683b1cf3254965e..540e802420ce0f5933784e0ca045253b94d30e3f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.master
import java.nio.ByteBuffer
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import org.apache.curator.framework.CuratorFramework
@@ -49,8 +49,8 @@ private[master] class ZooKeeperPersistenceEngine(conf: SparkConf, val serializer
}
override def read[T: ClassTag](prefix: String): Seq[T] = {
- val file = zk.getChildren.forPath(WORKING_DIR).filter(_.startsWith(prefix))
- file.map(deserializeFromFile[T]).flatten
+ zk.getChildren.forPath(WORKING_DIR).asScala
+ .filter(_.startsWith(prefix)).map(deserializeFromFile[T]).flatten
}
override def close() {
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
index 45a3f430454372cb132610513f6065489b1855a6..ce02ee203a4bd4c86f73e52d9163a9ded270544a 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
@@ -18,9 +18,8 @@
package org.apache.spark.deploy.worker
import java.io.{File, FileOutputStream, InputStream, IOException}
-import java.lang.System._
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.Map
import org.apache.spark.Logging
@@ -62,7 +61,7 @@ object CommandUtils extends Logging {
// SPARK-698: do not call the run.cmd script, as process.destroy()
// fails to kill a process tree on Windows
val cmd = new WorkerCommandBuilder(sparkHome, memory, command).buildCommand()
- cmd.toSeq ++ Seq(command.mainClass) ++ command.arguments
+ cmd.asScala ++ Seq(command.mainClass) ++ command.arguments
}
/**
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
index ec51c3d935d8ed185a8dd2686ecab10c6d7354cb..89159ff5e2b3ce4eb8ec66c014f44bc1ef8f10b5 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.worker
import java.io._
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import com.google.common.base.Charsets.UTF_8
import com.google.common.io.Files
@@ -172,8 +172,8 @@ private[deploy] class DriverRunner(
CommandUtils.redirectStream(process.getInputStream, stdout)
val stderr = new File(baseDir, "stderr")
- val header = "Launch Command: %s\n%s\n\n".format(
- builder.command.mkString("\"", "\" \"", "\""), "=" * 40)
+ val formattedCommand = builder.command.asScala.mkString("\"", "\" \"", "\"")
+ val header = "Launch Command: %s\n%s\n\n".format(formattedCommand, "=" * 40)
Files.append(header, stderr, UTF_8)
CommandUtils.redirectStream(process.getErrorStream, stderr)
}
@@ -229,6 +229,6 @@ private[deploy] trait ProcessBuilderLike {
private[deploy] object ProcessBuilderLike {
def apply(processBuilder: ProcessBuilder): ProcessBuilderLike = new ProcessBuilderLike {
override def start(): Process = processBuilder.start()
- override def command: Seq[String] = processBuilder.command()
+ override def command: Seq[String] = processBuilder.command().asScala
}
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
index ab3fea475c2a58ead624d637e74b21f2551e08f2..3aef0515cbf6e91b70fd41c495735486b5b952e9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.worker
import java.io._
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import com.google.common.base.Charsets.UTF_8
import com.google.common.io.Files
@@ -129,7 +129,8 @@ private[deploy] class ExecutorRunner(
val builder = CommandUtils.buildProcessBuilder(appDesc.command, new SecurityManager(conf),
memory, sparkHome.getAbsolutePath, substituteVariables)
val command = builder.command()
- logInfo("Launch command: " + command.mkString("\"", "\" \"", "\""))
+ val formattedCommand = command.asScala.mkString("\"", "\" \"", "\"")
+ logInfo(s"Launch command: $formattedCommand")
builder.directory(executorDir)
builder.environment.put("SPARK_EXECUTOR_DIRS", appLocalDirs.mkString(File.pathSeparator))
@@ -145,7 +146,7 @@ private[deploy] class ExecutorRunner(
process = builder.start()
val header = "Spark Executor Command: %s\n%s\n\n".format(
- command.mkString("\"", "\" \"", "\""), "=" * 40)
+ formattedCommand, "=" * 40)
// Redirect its stdout and stderr to files
val stdout = new File(executorDir, "stdout")
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
index 79b1536d94016a3852eb3cd4454ea17e983b8ff6..770927c80f7a4ed4b1b6b75f08ed2fee8076e9a0 100755
--- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
@@ -24,7 +24,6 @@ import java.util.{UUID, Date}
import java.util.concurrent._
import java.util.concurrent.{Future => JFuture, ScheduledFuture => JScheduledFuture}
-import scala.collection.JavaConversions._
import scala.collection.mutable.{HashMap, HashSet, LinkedHashMap}
import scala.concurrent.ExecutionContext
import scala.util.Random
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index 42a85e42ea2b6de485af160d48ca20f85bfe6c21..c3491bb8b1cf31b74ca44d8a86cf2bf026b5bcd5 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -23,7 +23,7 @@ import java.net.URL
import java.nio.ByteBuffer
import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, HashMap}
import scala.util.control.NonFatal
@@ -147,7 +147,7 @@ private[spark] class Executor(
/** Returns the total amount of time this JVM process has spent in garbage collection. */
private def computeTotalGcTime(): Long = {
- ManagementFactory.getGarbageCollectorMXBeans.map(_.getCollectionTime).sum
+ ManagementFactory.getGarbageCollectorMXBeans.asScala.map(_.getCollectionTime).sum
}
class TaskRunner(
@@ -425,7 +425,7 @@ private[spark] class Executor(
val tasksMetrics = new ArrayBuffer[(Long, TaskMetrics)]()
val curGCTime = computeTotalGcTime()
- for (taskRunner <- runningTasks.values()) {
+ for (taskRunner <- runningTasks.values().asScala) {
if (taskRunner.task != null) {
taskRunner.task.metrics.foreach { metrics =>
metrics.updateShuffleReadMetrics()
diff --git a/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala b/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala
index 293c512f8b70c0f6dbe98071087d52c163d5a7a1..d16f4a1fc4e3b42969e1d2acbd591d3f1f056f27 100644
--- a/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ExecutorSource.scala
@@ -19,7 +19,7 @@ package org.apache.spark.executor
import java.util.concurrent.ThreadPoolExecutor
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import com.codahale.metrics.{Gauge, MetricRegistry}
import org.apache.hadoop.fs.FileSystem
@@ -30,7 +30,7 @@ private[spark]
class ExecutorSource(threadPool: ThreadPoolExecutor, executorId: String) extends Source {
private def fileStats(scheme: String) : Option[FileSystem.Statistics] =
- FileSystem.getAllStatistics().find(s => s.getScheme.equals(scheme))
+ FileSystem.getAllStatistics.asScala.find(s => s.getScheme.equals(scheme))
private def registerFileSystemStat[T](
scheme: String, name: String, f: FileSystem.Statistics => T, defaultValue: T) = {
diff --git a/core/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala
index cfd672e1d8a973ee34f9e32a10514529dc20fee8..0474fd2ccc12ee0bd39658e8a0e180bc871e3fb3 100644
--- a/core/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala
+++ b/core/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala
@@ -19,7 +19,7 @@ package org.apache.spark.executor
import java.nio.ByteBuffer
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.mesos.protobuf.ByteString
import org.apache.mesos.{Executor => MesosExecutor, ExecutorDriver, MesosExecutorDriver}
@@ -28,7 +28,7 @@ import org.apache.mesos.Protos.{TaskStatus => MesosTaskStatus, _}
import org.apache.spark.{Logging, TaskState, SparkConf, SparkEnv}
import org.apache.spark.TaskState.TaskState
import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.scheduler.cluster.mesos.{MesosTaskLaunchData}
+import org.apache.spark.scheduler.cluster.mesos.MesosTaskLaunchData
import org.apache.spark.util.{SignalLogger, Utils}
private[spark] class MesosExecutorBackend
@@ -55,7 +55,7 @@ private[spark] class MesosExecutorBackend
slaveInfo: SlaveInfo) {
// Get num cores for this task from ExecutorInfo, created in MesosSchedulerBackend.
- val cpusPerTask = executorInfo.getResourcesList
+ val cpusPerTask = executorInfo.getResourcesList.asScala
.find(_.getName == "cpus")
.map(_.getScalar.getValue.toInt)
.getOrElse(0)
diff --git a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
index 6cda7772f77bce1b2e865866a719e2575c74a3c0..a5ad47293f1c2d1f2b3f3601bbfd661e08efa9c2 100644
--- a/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
+++ b/core/src/main/scala/org/apache/spark/input/PortableDataStream.scala
@@ -19,7 +19,7 @@ package org.apache.spark.input
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, DataOutputStream}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import com.google.common.io.ByteStreams
import org.apache.hadoop.conf.Configuration
@@ -44,12 +44,9 @@ private[spark] abstract class StreamFileInputFormat[T]
* which is set through setMaxSplitSize
*/
def setMinPartitions(context: JobContext, minPartitions: Int) {
- val files = listStatus(context)
- val totalLen = files.map { file =>
- if (file.isDir) 0L else file.getLen
- }.sum
-
- val maxSplitSize = Math.ceil(totalLen * 1.0 / files.length).toLong
+ val files = listStatus(context).asScala
+ val totalLen = files.map(file => if (file.isDir) 0L else file.getLen).sum
+ val maxSplitSize = Math.ceil(totalLen * 1.0 / files.size).toLong
super.setMaxSplitSize(maxSplitSize)
}
diff --git a/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala b/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala
index aaef7c74eea33ee6f6c6180129662ae484758e71..1ba34a11414a21702a8371a1718d45ddb9509a3e 100644
--- a/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala
+++ b/core/src/main/scala/org/apache/spark/input/WholeTextFileInputFormat.scala
@@ -17,7 +17,7 @@
package org.apache.spark.input
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapreduce.InputSplit
@@ -52,10 +52,8 @@ private[spark] class WholeTextFileInputFormat
* which is set through setMaxSplitSize
*/
def setMinPartitions(context: JobContext, minPartitions: Int) {
- val files = listStatus(context)
- val totalLen = files.map { file =>
- if (file.isDir) 0L else file.getLen
- }.sum
+ val files = listStatus(context).asScala
+ val totalLen = files.map(file => if (file.isDir) 0L else file.getLen).sum
val maxSplitSize = Math.ceil(totalLen * 1.0 /
(if (minPartitions == 0) 1 else minPartitions)).toLong
super.setMaxSplitSize(maxSplitSize)
diff --git a/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala b/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala
index 9be98723aed14c9182f6f279aa659f8370433792..0c096656f9236daccbb70b288f3317d1f856631c 100644
--- a/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala
@@ -20,7 +20,7 @@ package org.apache.spark.launcher
import java.io.File
import java.util.{HashMap => JHashMap, List => JList, Map => JMap}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.deploy.Command
@@ -32,7 +32,7 @@ import org.apache.spark.deploy.Command
private[spark] class WorkerCommandBuilder(sparkHome: String, memoryMb: Int, command: Command)
extends AbstractCommandBuilder {
- childEnv.putAll(command.environment)
+ childEnv.putAll(command.environment.asJava)
childEnv.put(CommandBuilderUtils.ENV_SPARK_HOME, sparkHome)
override def buildCommand(env: JMap[String, String]): JList[String] = {
diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala b/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
index d7495551ad2333cd1c32be90ad7d987ac9453c5e..dd2d325d8703411cb60408e712f5a17b6464e4bc 100644
--- a/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/MetricsConfig.scala
@@ -20,6 +20,7 @@ package org.apache.spark.metrics
import java.io.{FileInputStream, InputStream}
import java.util.Properties
+import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.util.matching.Regex
@@ -58,25 +59,20 @@ private[spark] class MetricsConfig(conf: SparkConf) extends Logging {
propertyCategories = subProperties(properties, INSTANCE_REGEX)
if (propertyCategories.contains(DEFAULT_PREFIX)) {
- import scala.collection.JavaConversions._
-
- val defaultProperty = propertyCategories(DEFAULT_PREFIX)
- for { (inst, prop) <- propertyCategories
- if (inst != DEFAULT_PREFIX)
- (k, v) <- defaultProperty
- if (prop.getProperty(k) == null) } {
- prop.setProperty(k, v)
+ val defaultProperty = propertyCategories(DEFAULT_PREFIX).asScala
+ for((inst, prop) <- propertyCategories if (inst != DEFAULT_PREFIX);
+ (k, v) <- defaultProperty if (prop.get(k) == null)) {
+ prop.put(k, v)
}
}
}
def subProperties(prop: Properties, regex: Regex): mutable.HashMap[String, Properties] = {
val subProperties = new mutable.HashMap[String, Properties]
- import scala.collection.JavaConversions._
- prop.foreach { kv =>
- if (regex.findPrefixOf(kv._1).isDefined) {
- val regex(prefix, suffix) = kv._1
- subProperties.getOrElseUpdate(prefix, new Properties).setProperty(suffix, kv._2)
+ prop.asScala.foreach { kv =>
+ if (regex.findPrefixOf(kv._1.toString).isDefined) {
+ val regex(prefix, suffix) = kv._1.toString
+ subProperties.getOrElseUpdate(prefix, new Properties).setProperty(suffix, kv._2.toString)
}
}
subProperties
diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
index b089da8596e2b7a19bf44f3eb7b47bf8a789c3e0..7c170a742fb6436ad33b3633877f0060a985e2b0 100644
--- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
+++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockRpcServer.scala
@@ -19,7 +19,7 @@ package org.apache.spark.network.netty
import java.nio.ByteBuffer
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.Logging
import org.apache.spark.network.BlockDataManager
@@ -55,7 +55,7 @@ class NettyBlockRpcServer(
case openBlocks: OpenBlocks =>
val blocks: Seq[ManagedBuffer] =
openBlocks.blockIds.map(BlockId.apply).map(blockManager.getBlockData)
- val streamId = streamManager.registerStream(blocks.iterator)
+ val streamId = streamManager.registerStream(blocks.iterator.asJava)
logTrace(s"Registered streamId $streamId with ${blocks.size} buffers")
responseContext.onSuccess(new StreamHandle(streamId, blocks.size).toByteArray)
diff --git a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
index d650d5fe73087abc0e67939912dbac29a2730d54..ff8aae9ebe9f0ceeadce96372e0ee2a4cec08d1e 100644
--- a/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
+++ b/core/src/main/scala/org/apache/spark/network/netty/NettyBlockTransferService.scala
@@ -17,7 +17,7 @@
package org.apache.spark.network.netty
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.concurrent.{Future, Promise}
import org.apache.spark.{SecurityManager, SparkConf}
@@ -58,7 +58,7 @@ class NettyBlockTransferService(conf: SparkConf, securityManager: SecurityManage
securityManager.isSaslEncryptionEnabled()))
}
transportContext = new TransportContext(transportConf, rpcHandler)
- clientFactory = transportContext.createClientFactory(clientBootstrap.toList)
+ clientFactory = transportContext.createClientFactory(clientBootstrap.toSeq.asJava)
server = createServer(serverBootstrap.toList)
appId = conf.getAppId
logInfo("Server created on " + server.getPort)
@@ -67,7 +67,7 @@ class NettyBlockTransferService(conf: SparkConf, securityManager: SecurityManage
/** Creates and binds the TransportServer, possibly trying multiple ports. */
private def createServer(bootstraps: List[TransportServerBootstrap]): TransportServer = {
def startService(port: Int): (TransportServer, Int) = {
- val server = transportContext.createServer(port, bootstraps)
+ val server = transportContext.createServer(port, bootstraps.asJava)
(server, server.getPort)
}
diff --git a/core/src/main/scala/org/apache/spark/network/nio/Connection.scala b/core/src/main/scala/org/apache/spark/network/nio/Connection.scala
index 1499da07bb83b1f38132c852f14912020263cc0b..8d9ebadaf79d485f77f600e3cbc23afc413c4923 100644
--- a/core/src/main/scala/org/apache/spark/network/nio/Connection.scala
+++ b/core/src/main/scala/org/apache/spark/network/nio/Connection.scala
@@ -23,7 +23,7 @@ import java.nio.channels._
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.LinkedList
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, HashMap}
import scala.util.control.NonFatal
@@ -145,7 +145,7 @@ abstract class Connection(val channel: SocketChannel, val selector: Selector,
}
def callOnExceptionCallbacks(e: Throwable) {
- onExceptionCallbacks foreach {
+ onExceptionCallbacks.asScala.foreach {
callback =>
try {
callback(this, e)
diff --git a/core/src/main/scala/org/apache/spark/partial/GroupedCountEvaluator.scala b/core/src/main/scala/org/apache/spark/partial/GroupedCountEvaluator.scala
index 91b07ce3af1b6ab363b9f07ad7cddca90402b613..5afce75680f942547bf0b604060eabe305558ef0 100644
--- a/core/src/main/scala/org/apache/spark/partial/GroupedCountEvaluator.scala
+++ b/core/src/main/scala/org/apache/spark/partial/GroupedCountEvaluator.scala
@@ -19,7 +19,7 @@ package org.apache.spark.partial
import java.util.{HashMap => JHashMap}
-import scala.collection.JavaConversions.mapAsScalaMap
+import scala.collection.JavaConverters._
import scala.collection.Map
import scala.collection.mutable.HashMap
import scala.reflect.ClassTag
@@ -48,9 +48,9 @@ private[spark] class GroupedCountEvaluator[T : ClassTag](totalOutputs: Int, conf
if (outputsMerged == totalOutputs) {
val result = new JHashMap[T, BoundedDouble](sums.size)
sums.foreach { case (key, sum) =>
- result(key) = new BoundedDouble(sum, 1.0, sum, sum)
+ result.put(key, new BoundedDouble(sum, 1.0, sum, sum))
}
- result
+ result.asScala
} else if (outputsMerged == 0) {
new HashMap[T, BoundedDouble]
} else {
@@ -64,9 +64,9 @@ private[spark] class GroupedCountEvaluator[T : ClassTag](totalOutputs: Int, conf
val stdev = math.sqrt(variance)
val low = mean - confFactor * stdev
val high = mean + confFactor * stdev
- result(key) = new BoundedDouble(mean, confidence, low, high)
+ result.put(key, new BoundedDouble(mean, confidence, low, high))
}
- result
+ result.asScala
}
}
}
diff --git a/core/src/main/scala/org/apache/spark/partial/GroupedMeanEvaluator.scala b/core/src/main/scala/org/apache/spark/partial/GroupedMeanEvaluator.scala
index af26c3d59ac025814bbf63edaa5b42647a9d77ed..a1640406848038da254cbce15bc42b5e42145242 100644
--- a/core/src/main/scala/org/apache/spark/partial/GroupedMeanEvaluator.scala
+++ b/core/src/main/scala/org/apache/spark/partial/GroupedMeanEvaluator.scala
@@ -19,7 +19,7 @@ package org.apache.spark.partial
import java.util.{HashMap => JHashMap}
-import scala.collection.JavaConversions.mapAsScalaMap
+import scala.collection.JavaConverters._
import scala.collection.Map
import scala.collection.mutable.HashMap
@@ -55,9 +55,9 @@ private[spark] class GroupedMeanEvaluator[T](totalOutputs: Int, confidence: Doub
while (iter.hasNext) {
val entry = iter.next()
val mean = entry.getValue.mean
- result(entry.getKey) = new BoundedDouble(mean, 1.0, mean, mean)
+ result.put(entry.getKey, new BoundedDouble(mean, 1.0, mean, mean))
}
- result
+ result.asScala
} else if (outputsMerged == 0) {
new HashMap[T, BoundedDouble]
} else {
@@ -72,9 +72,9 @@ private[spark] class GroupedMeanEvaluator[T](totalOutputs: Int, confidence: Doub
val confFactor = studentTCacher.get(counter.count)
val low = mean - confFactor * stdev
val high = mean + confFactor * stdev
- result(entry.getKey) = new BoundedDouble(mean, confidence, low, high)
+ result.put(entry.getKey, new BoundedDouble(mean, confidence, low, high))
}
- result
+ result.asScala
}
}
}
diff --git a/core/src/main/scala/org/apache/spark/partial/GroupedSumEvaluator.scala b/core/src/main/scala/org/apache/spark/partial/GroupedSumEvaluator.scala
index 442fb86227d8699e2a4c253204659e7dae9f38c9..54a1beab3514b91e981c5c8ffbef620d2eedaefd 100644
--- a/core/src/main/scala/org/apache/spark/partial/GroupedSumEvaluator.scala
+++ b/core/src/main/scala/org/apache/spark/partial/GroupedSumEvaluator.scala
@@ -19,7 +19,7 @@ package org.apache.spark.partial
import java.util.{HashMap => JHashMap}
-import scala.collection.JavaConversions.mapAsScalaMap
+import scala.collection.JavaConverters._
import scala.collection.Map
import scala.collection.mutable.HashMap
@@ -55,9 +55,9 @@ private[spark] class GroupedSumEvaluator[T](totalOutputs: Int, confidence: Doubl
while (iter.hasNext) {
val entry = iter.next()
val sum = entry.getValue.sum
- result(entry.getKey) = new BoundedDouble(sum, 1.0, sum, sum)
+ result.put(entry.getKey, new BoundedDouble(sum, 1.0, sum, sum))
}
- result
+ result.asScala
} else if (outputsMerged == 0) {
new HashMap[T, BoundedDouble]
} else {
@@ -80,9 +80,9 @@ private[spark] class GroupedSumEvaluator[T](totalOutputs: Int, confidence: Doubl
val confFactor = studentTCacher.get(counter.count)
val low = sumEstimate - confFactor * sumStdev
val high = sumEstimate + confFactor * sumStdev
- result(entry.getKey) = new BoundedDouble(sumEstimate, confidence, low, high)
+ result.put(entry.getKey, new BoundedDouble(sumEstimate, confidence, low, high))
}
- result
+ result.asScala
}
}
}
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
index 326fafb230a402bff11a0c1432105b60965a7938..4e5f2e8a5d4678c2f4e9c4c045c053e8f44821fa 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala
@@ -22,7 +22,7 @@ import java.text.SimpleDateFormat
import java.util.{Date, HashMap => JHashMap}
import scala.collection.{Map, mutable}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import scala.reflect.ClassTag
import scala.util.DynamicVariable
@@ -312,14 +312,14 @@ class PairRDDFunctions[K, V](self: RDD[(K, V)])
} : Iterator[JHashMap[K, V]]
val mergeMaps = (m1: JHashMap[K, V], m2: JHashMap[K, V]) => {
- m2.foreach { pair =>
+ m2.asScala.foreach { pair =>
val old = m1.get(pair._1)
m1.put(pair._1, if (old == null) pair._2 else cleanedF(old, pair._2))
}
m1
} : JHashMap[K, V]
- self.mapPartitions(reducePartition).reduce(mergeMaps)
+ self.mapPartitions(reducePartition).reduce(mergeMaps).asScala
}
/** Alias for reduceByKeyLocally */
diff --git a/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
index 3bb9998e1db44350426d4fc17ad6b07abf84909a..afbe566b76566b8bea73d8cf95c71a4ea8424657 100644
--- a/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/PipedRDD.scala
@@ -23,7 +23,7 @@ import java.io.IOException
import java.io.PrintWriter
import java.util.StringTokenizer
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.Map
import scala.collection.mutable.ArrayBuffer
import scala.io.Source
@@ -72,7 +72,7 @@ private[spark] class PipedRDD[T: ClassTag](
}
override def compute(split: Partition, context: TaskContext): Iterator[String] = {
- val pb = new ProcessBuilder(command)
+ val pb = new ProcessBuilder(command.asJava)
// Add the environmental variables to the process.
val currentEnvVars = pb.environment()
envVars.foreach { case (variable, value) => currentEnvVars.put(variable, value) }
@@ -81,7 +81,7 @@ private[spark] class PipedRDD[T: ClassTag](
// so the user code can access the input filename
if (split.isInstanceOf[HadoopPartition]) {
val hadoopSplit = split.asInstanceOf[HadoopPartition]
- currentEnvVars.putAll(hadoopSplit.getPipeEnvVars())
+ currentEnvVars.putAll(hadoopSplit.getPipeEnvVars().asJava)
}
// When spark.worker.separated.working.directory option is turned on, each
diff --git a/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
index f7cb1791d4ac6012c30beddb3a7da29aaa2cff52..9a4fa301b06e319d3254394fd38ea76fa834e7c5 100644
--- a/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
@@ -19,7 +19,7 @@ package org.apache.spark.rdd
import java.util.{HashMap => JHashMap}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import scala.reflect.ClassTag
@@ -125,7 +125,7 @@ private[spark] class SubtractedRDD[K: ClassTag, V: ClassTag, W: ClassTag](
integrate(0, t => getSeq(t._1) += t._2)
// the second dep is rdd2; remove all of its keys
integrate(1, t => map.remove(t._1))
- map.iterator.map { t => t._2.iterator.map { (t._1, _) } }.flatten
+ map.asScala.iterator.map(t => t._2.iterator.map((t._1, _))).flatten
}
override def clearDependencies() {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/InputFormatInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/InputFormatInfo.scala
index bac37bfdaa23fd89c43e9e19f4a182078339bb32..0e438ab4366d92e96060ab2f3c6c2a2548bcaae3 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/InputFormatInfo.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/InputFormatInfo.scala
@@ -17,7 +17,7 @@
package org.apache.spark.scheduler
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.immutable.Set
import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
@@ -107,7 +107,7 @@ class InputFormatInfo(val configuration: Configuration, val inputFormatClazz: Cl
val retval = new ArrayBuffer[SplitInfo]()
val list = instance.getSplits(job)
- for (split <- list) {
+ for (split <- list.asScala) {
retval ++= SplitInfo.toSplitInfo(inputFormatClazz, path, split)
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/Pool.scala b/core/src/main/scala/org/apache/spark/scheduler/Pool.scala
index 174b73221afc0038599270d25514f7e3336a7e5e..5821afea9898256f5d307207b12682a24d0754c1 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/Pool.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/Pool.scala
@@ -19,7 +19,7 @@ package org.apache.spark.scheduler
import java.util.concurrent.{ConcurrentHashMap, ConcurrentLinkedQueue}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.Logging
@@ -74,7 +74,7 @@ private[spark] class Pool(
if (schedulableNameToSchedulable.containsKey(schedulableName)) {
return schedulableNameToSchedulable.get(schedulableName)
}
- for (schedulable <- schedulableQueue) {
+ for (schedulable <- schedulableQueue.asScala) {
val sched = schedulable.getSchedulableByName(schedulableName)
if (sched != null) {
return sched
@@ -84,12 +84,12 @@ private[spark] class Pool(
}
override def executorLost(executorId: String, host: String) {
- schedulableQueue.foreach(_.executorLost(executorId, host))
+ schedulableQueue.asScala.foreach(_.executorLost(executorId, host))
}
override def checkSpeculatableTasks(): Boolean = {
var shouldRevive = false
- for (schedulable <- schedulableQueue) {
+ for (schedulable <- schedulableQueue.asScala) {
shouldRevive |= schedulable.checkSpeculatableTasks()
}
shouldRevive
@@ -98,7 +98,7 @@ private[spark] class Pool(
override def getSortedTaskSetQueue: ArrayBuffer[TaskSetManager] = {
var sortedTaskSetQueue = new ArrayBuffer[TaskSetManager]
val sortedSchedulableQueue =
- schedulableQueue.toSeq.sortWith(taskSetSchedulingAlgorithm.comparator)
+ schedulableQueue.asScala.toSeq.sortWith(taskSetSchedulingAlgorithm.comparator)
for (schedulable <- sortedSchedulableQueue) {
sortedTaskSetQueue ++= schedulable.getSortedTaskSetQueue
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
index d6e1e9e5bebc2ce53d23317e7b87232e8b00e5ee..452c32d5411cddcdf32b4385f8f1c73c1ce4a143 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala
@@ -21,7 +21,7 @@ import java.io.File
import java.util.concurrent.locks.ReentrantLock
import java.util.{Collections, List => JList}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.{HashMap, HashSet}
import com.google.common.collect.HashBiMap
@@ -233,7 +233,7 @@ private[spark] class CoarseMesosSchedulerBackend(
override def resourceOffers(d: SchedulerDriver, offers: JList[Offer]) {
stateLock.synchronized {
val filters = Filters.newBuilder().setRefuseSeconds(5).build()
- for (offer <- offers) {
+ for (offer <- offers.asScala) {
val offerAttributes = toAttributeMap(offer.getAttributesList)
val meetsConstraints = matchesAttributeRequirements(slaveOfferConstraints, offerAttributes)
val slaveId = offer.getSlaveId.getValue
@@ -251,21 +251,21 @@ private[spark] class CoarseMesosSchedulerBackend(
val cpusToUse = math.min(cpus, maxCores - totalCoresAcquired)
totalCoresAcquired += cpusToUse
val taskId = newMesosTaskId()
- taskIdToSlaveId(taskId) = slaveId
+ taskIdToSlaveId.put(taskId, slaveId)
slaveIdsWithExecutors += slaveId
coresByTaskId(taskId) = cpusToUse
// Gather cpu resources from the available resources and use them in the task.
val (remainingResources, cpuResourcesToUse) =
partitionResources(offer.getResourcesList, "cpus", cpusToUse)
val (_, memResourcesToUse) =
- partitionResources(remainingResources, "mem", calculateTotalMemory(sc))
+ partitionResources(remainingResources.asJava, "mem", calculateTotalMemory(sc))
val taskBuilder = MesosTaskInfo.newBuilder()
.setTaskId(TaskID.newBuilder().setValue(taskId.toString).build())
.setSlaveId(offer.getSlaveId)
.setCommand(createCommand(offer, cpusToUse + extraCoresPerSlave, taskId))
.setName("Task " + taskId)
- .addAllResources(cpuResourcesToUse)
- .addAllResources(memResourcesToUse)
+ .addAllResources(cpuResourcesToUse.asJava)
+ .addAllResources(memResourcesToUse.asJava)
sc.conf.getOption("spark.mesos.executor.docker.image").foreach { image =>
MesosSchedulerBackendUtil
@@ -314,9 +314,9 @@ private[spark] class CoarseMesosSchedulerBackend(
}
if (TaskState.isFinished(TaskState.fromMesos(state))) {
- val slaveId = taskIdToSlaveId(taskId)
+ val slaveId = taskIdToSlaveId.get(taskId)
slaveIdsWithExecutors -= slaveId
- taskIdToSlaveId -= taskId
+ taskIdToSlaveId.remove(taskId)
// Remove the cores we have remembered for this task, if it's in the hashmap
for (cores <- coresByTaskId.get(taskId)) {
totalCoresAcquired -= cores
@@ -361,7 +361,7 @@ private[spark] class CoarseMesosSchedulerBackend(
stateLock.synchronized {
if (slaveIdsWithExecutors.contains(slaveId)) {
val slaveIdToTaskId = taskIdToSlaveId.inverse()
- if (slaveIdToTaskId.contains(slaveId)) {
+ if (slaveIdToTaskId.containsKey(slaveId)) {
val taskId: Int = slaveIdToTaskId.get(slaveId)
taskIdToSlaveId.remove(taskId)
removeExecutor(sparkExecutorId(slaveId, taskId.toString), reason)
@@ -411,7 +411,7 @@ private[spark] class CoarseMesosSchedulerBackend(
val slaveIdToTaskId = taskIdToSlaveId.inverse()
for (executorId <- executorIds) {
val slaveId = executorId.split("/")(0)
- if (slaveIdToTaskId.contains(slaveId)) {
+ if (slaveIdToTaskId.containsKey(slaveId)) {
mesosDriver.killTask(
TaskID.newBuilder().setValue(slaveIdToTaskId.get(slaveId).toString).build())
pendingRemovedSlaveIds += slaveId
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterPersistenceEngine.scala
index 3efc536f1456ce45416053569241cd3d4eccef26..e0c547dce6d07a8ad9e7f346ac9939e2df82fe41 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterPersistenceEngine.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterPersistenceEngine.scala
@@ -17,7 +17,7 @@
package org.apache.spark.scheduler.cluster.mesos
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.curator.framework.CuratorFramework
import org.apache.zookeeper.CreateMode
@@ -129,6 +129,6 @@ private[spark] class ZookeeperMesosClusterPersistenceEngine(
}
override def fetchAll[T](): Iterable[T] = {
- zk.getChildren.forPath(WORKING_DIR).map(fetch[T]).flatten
+ zk.getChildren.forPath(WORKING_DIR).asScala.flatMap(fetch[T])
}
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
index 1206f184fbc8246c4ba53a609cea65c4f7bdd4d0..07da9242b992211f64b2ed45bd9771f651f07335 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
@@ -21,7 +21,7 @@ import java.io.File
import java.util.concurrent.locks.ReentrantLock
import java.util.{Collections, Date, List => JList}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
@@ -350,7 +350,7 @@ private[spark] class MesosClusterScheduler(
}
// TODO: Page the status updates to avoid trying to reconcile
// a large amount of tasks at once.
- driver.reconcileTasks(statuses)
+ driver.reconcileTasks(statuses.toSeq.asJava)
}
}
}
@@ -493,10 +493,10 @@ private[spark] class MesosClusterScheduler(
}
override def resourceOffers(driver: SchedulerDriver, offers: JList[Offer]): Unit = {
- val currentOffers = offers.map { o =>
+ val currentOffers = offers.asScala.map(o =>
new ResourceOffer(
o, getResource(o.getResourcesList, "cpus"), getResource(o.getResourcesList, "mem"))
- }.toList
+ ).toList
logTrace(s"Received offers from Mesos: \n${currentOffers.mkString("\n")}")
val tasks = new mutable.HashMap[OfferID, ArrayBuffer[TaskInfo]]()
val currentTime = new Date()
@@ -521,10 +521,10 @@ private[spark] class MesosClusterScheduler(
currentOffers,
tasks)
}
- tasks.foreach { case (offerId, tasks) =>
- driver.launchTasks(Collections.singleton(offerId), tasks)
+ tasks.foreach { case (offerId, taskInfos) =>
+ driver.launchTasks(Collections.singleton(offerId), taskInfos.asJava)
}
- offers
+ offers.asScala
.filter(o => !tasks.keySet.contains(o.getId))
.foreach(o => driver.declineOffer(o.getId))
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
index 5c20606d587151a5ba2277c79098f1221eddc6fe..2e424054be78514fead41e43e5013e89d4f9890b 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala
@@ -20,7 +20,7 @@ package org.apache.spark.scheduler.cluster.mesos
import java.io.File
import java.util.{ArrayList => JArrayList, Collections, List => JList}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.{HashMap, HashSet}
import org.apache.mesos.{Scheduler => MScheduler, _}
@@ -129,14 +129,12 @@ private[spark] class MesosSchedulerBackend(
val (resourcesAfterCpu, usedCpuResources) =
partitionResources(availableResources, "cpus", scheduler.CPUS_PER_TASK)
val (resourcesAfterMem, usedMemResources) =
- partitionResources(resourcesAfterCpu, "mem", calculateTotalMemory(sc))
+ partitionResources(resourcesAfterCpu.asJava, "mem", calculateTotalMemory(sc))
- builder.addAllResources(usedCpuResources)
- builder.addAllResources(usedMemResources)
+ builder.addAllResources(usedCpuResources.asJava)
+ builder.addAllResources(usedMemResources.asJava)
- sc.conf.getOption("spark.mesos.uris").map { uris =>
- setupUris(uris, command)
- }
+ sc.conf.getOption("spark.mesos.uris").foreach(setupUris(_, command))
val executorInfo = builder
.setExecutorId(ExecutorID.newBuilder().setValue(execId).build())
@@ -148,7 +146,7 @@ private[spark] class MesosSchedulerBackend(
.setupContainerBuilderDockerInfo(image, sc.conf, executorInfo.getContainerBuilder())
}
- (executorInfo.build(), resourcesAfterMem)
+ (executorInfo.build(), resourcesAfterMem.asJava)
}
/**
@@ -193,7 +191,7 @@ private[spark] class MesosSchedulerBackend(
private def getTasksSummary(tasks: JArrayList[MesosTaskInfo]): String = {
val builder = new StringBuilder
- tasks.foreach { t =>
+ tasks.asScala.foreach { t =>
builder.append("Task id: ").append(t.getTaskId.getValue).append("\n")
.append("Slave id: ").append(t.getSlaveId.getValue).append("\n")
.append("Task resources: ").append(t.getResourcesList).append("\n")
@@ -211,7 +209,7 @@ private[spark] class MesosSchedulerBackend(
override def resourceOffers(d: SchedulerDriver, offers: JList[Offer]) {
inClassLoader() {
// Fail-fast on offers we know will be rejected
- val (usableOffers, unUsableOffers) = offers.partition { o =>
+ val (usableOffers, unUsableOffers) = offers.asScala.partition { o =>
val mem = getResource(o.getResourcesList, "mem")
val cpus = getResource(o.getResourcesList, "cpus")
val slaveId = o.getSlaveId.getValue
@@ -323,10 +321,10 @@ private[spark] class MesosSchedulerBackend(
.setSlaveId(SlaveID.newBuilder().setValue(slaveId).build())
.setExecutor(executorInfo)
.setName(task.name)
- .addAllResources(cpuResources)
+ .addAllResources(cpuResources.asJava)
.setData(MesosTaskLaunchData(task.serializedTask, task.attemptNumber).toByteString)
.build()
- (taskInfo, finalResources)
+ (taskInfo, finalResources.asJava)
}
override def statusUpdate(d: SchedulerDriver, status: TaskStatus) {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala
index 5b854aa5c2754b3c92a19f0647ec24b2b55fae7e..860c8e097b3b958632b65a226e08ba8296c0c72d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerUtils.scala
@@ -20,7 +20,7 @@ package org.apache.spark.scheduler.cluster.mesos
import java.util.{List => JList}
import java.util.concurrent.CountDownLatch
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import scala.util.control.NonFatal
@@ -137,7 +137,7 @@ private[mesos] trait MesosSchedulerUtils extends Logging {
protected def getResource(res: JList[Resource], name: String): Double = {
// A resource can have multiple values in the offer since it can either be from
// a specific role or wildcard.
- res.filter(_.getName == name).map(_.getScalar.getValue).sum
+ res.asScala.filter(_.getName == name).map(_.getScalar.getValue).sum
}
protected def markRegistered(): Unit = {
@@ -169,7 +169,7 @@ private[mesos] trait MesosSchedulerUtils extends Logging {
amountToUse: Double): (List[Resource], List[Resource]) = {
var remain = amountToUse
var requestedResources = new ArrayBuffer[Resource]
- val remainingResources = resources.map {
+ val remainingResources = resources.asScala.map {
case r => {
if (remain > 0 &&
r.getType == Value.Type.SCALAR &&
@@ -214,7 +214,7 @@ private[mesos] trait MesosSchedulerUtils extends Logging {
* @return
*/
protected def toAttributeMap(offerAttributes: JList[Attribute]): Map[String, GeneratedMessage] = {
- offerAttributes.map(attr => {
+ offerAttributes.asScala.map(attr => {
val attrValue = attr.getType match {
case Value.Type.SCALAR => attr.getScalar
case Value.Type.RANGES => attr.getRanges
@@ -253,7 +253,7 @@ private[mesos] trait MesosSchedulerUtils extends Logging {
requiredValues.map(_.toLong).exists(offerRange.contains(_))
case Some(offeredValue: Value.Set) =>
// check if the specified required values is a subset of offered set
- requiredValues.subsetOf(offeredValue.getItemList.toSet)
+ requiredValues.subsetOf(offeredValue.getItemList.asScala.toSet)
case Some(textValue: Value.Text) =>
// check if the specified value is equal, if multiple values are specified
// we succeed if any of them match.
@@ -299,14 +299,13 @@ private[mesos] trait MesosSchedulerUtils extends Logging {
Map()
} else {
try {
- Map() ++ mapAsScalaMap(splitter.split(constraintsVal)).map {
- case (k, v) =>
- if (v == null || v.isEmpty) {
- (k, Set[String]())
- } else {
- (k, v.split(',').toSet)
- }
- }
+ splitter.split(constraintsVal).asScala.toMap.mapValues(v =>
+ if (v == null || v.isEmpty) {
+ Set[String]()
+ } else {
+ v.split(',').toSet
+ }
+ )
} catch {
case NonFatal(e) =>
throw new IllegalArgumentException(s"Bad constraint string: $constraintsVal", e)
diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index 0ff7562e912ca6d30d23ba8c9552d8d34fa4d032..048a938507277d0f8bddadd3f00eac99dcec8be7 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -21,6 +21,7 @@ import java.io.{EOFException, IOException, InputStream, OutputStream}
import java.nio.ByteBuffer
import javax.annotation.Nullable
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import com.esotericsoftware.kryo.{Kryo, KryoException}
@@ -373,16 +374,15 @@ private class JavaIterableWrapperSerializer
override def read(kryo: Kryo, in: KryoInput, clz: Class[java.lang.Iterable[_]])
: java.lang.Iterable[_] = {
kryo.readClassAndObject(in) match {
- case scalaIterable: Iterable[_] =>
- scala.collection.JavaConversions.asJavaIterable(scalaIterable)
- case javaIterable: java.lang.Iterable[_] =>
- javaIterable
+ case scalaIterable: Iterable[_] => scalaIterable.asJava
+ case javaIterable: java.lang.Iterable[_] => javaIterable
}
}
}
private object JavaIterableWrapperSerializer extends Logging {
- // The class returned by asJavaIterable (scala.collection.convert.Wrappers$IterableWrapper).
+ // The class returned by JavaConverters.asJava
+ // (scala.collection.convert.Wrappers$IterableWrapper).
val wrapperClass =
scala.collection.convert.WrapAsJava.asJavaIterable(Seq(1)).getClass
diff --git a/core/src/main/scala/org/apache/spark/shuffle/FileShuffleBlockResolver.scala b/core/src/main/scala/org/apache/spark/shuffle/FileShuffleBlockResolver.scala
index f6a96d81e7aa963663cc81a5f68ab897c7642348..c057de9b3f4df2d9b4754f0db27fd26aee5aac1e 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/FileShuffleBlockResolver.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/FileShuffleBlockResolver.scala
@@ -21,7 +21,7 @@ import java.io.File
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.atomic.AtomicInteger
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.{Logging, SparkConf, SparkEnv}
import org.apache.spark.executor.ShuffleWriteMetrics
@@ -210,11 +210,13 @@ private[spark] class FileShuffleBlockResolver(conf: SparkConf)
shuffleStates.get(shuffleId) match {
case Some(state) =>
if (consolidateShuffleFiles) {
- for (fileGroup <- state.allFileGroups; file <- fileGroup.files) {
+ for (fileGroup <- state.allFileGroups.asScala;
+ file <- fileGroup.files) {
file.delete()
}
} else {
- for (mapId <- state.completedMapTasks; reduceId <- 0 until state.numBuckets) {
+ for (mapId <- state.completedMapTasks.asScala;
+ reduceId <- 0 until state.numBuckets) {
val blockId = new ShuffleBlockId(shuffleId, mapId, reduceId)
blockManager.diskBlockManager.getFile(blockId).delete()
}
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
index 6fec5240707a6c02e8a6866260156c5e938bb886..7db6035553ae64dc7521f0c46f2bed6e23820b3d 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
@@ -21,7 +21,7 @@ import java.util.{HashMap => JHashMap}
import scala.collection.immutable.HashSet
import scala.collection.mutable
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.concurrent.{ExecutionContext, Future}
import org.apache.spark.rpc.{RpcEndpointRef, RpcEnv, RpcCallContext, ThreadSafeRpcEndpoint}
@@ -133,7 +133,7 @@ class BlockManagerMasterEndpoint(
// Find all blocks for the given RDD, remove the block from both blockLocations and
// the blockManagerInfo that is tracking the blocks.
- val blocks = blockLocations.keys.flatMap(_.asRDDId).filter(_.rddId == rddId)
+ val blocks = blockLocations.asScala.keys.flatMap(_.asRDDId).filter(_.rddId == rddId)
blocks.foreach { blockId =>
val bms: mutable.HashSet[BlockManagerId] = blockLocations.get(blockId)
bms.foreach(bm => blockManagerInfo.get(bm).foreach(_.removeBlock(blockId)))
@@ -242,7 +242,7 @@ class BlockManagerMasterEndpoint(
private def storageStatus: Array[StorageStatus] = {
blockManagerInfo.map { case (blockManagerId, info) =>
- new StorageStatus(blockManagerId, info.maxMem, info.blocks)
+ new StorageStatus(blockManagerId, info.maxMem, info.blocks.asScala)
}.toArray
}
@@ -292,7 +292,7 @@ class BlockManagerMasterEndpoint(
if (askSlaves) {
info.slaveEndpoint.ask[Seq[BlockId]](getMatchingBlockIds)
} else {
- Future { info.blocks.keys.filter(filter).toSeq }
+ Future { info.blocks.asScala.keys.filter(filter).toSeq }
}
future
}
diff --git a/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala b/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala
index 78e7ddc27d1c779f6fe12d8cee4476b58f870394..1738258a0c7941203b34d95dcf9a752ca588cac4 100644
--- a/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/AkkaUtils.scala
@@ -17,7 +17,7 @@
package org.apache.spark.util
-import scala.collection.JavaConversions.mapAsJavaMap
+import scala.collection.JavaConverters._
import akka.actor.{ActorRef, ActorSystem, ExtendedActorSystem}
import akka.pattern.ask
@@ -92,7 +92,7 @@ private[spark] object AkkaUtils extends Logging {
val akkaSslConfig = securityManager.akkaSSLOptions.createAkkaConfig
.getOrElse(ConfigFactory.empty())
- val akkaConf = ConfigFactory.parseMap(conf.getAkkaConf.toMap[String, String])
+ val akkaConf = ConfigFactory.parseMap(conf.getAkkaConf.toMap.asJava)
.withFallback(akkaSslConfig).withFallback(ConfigFactory.parseString(
s"""
|akka.daemonic = on
diff --git a/core/src/main/scala/org/apache/spark/util/ListenerBus.scala b/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
index a725767d08cc2e8a192929bf1a85ae90ebc694ef..13cb516b583e9898f7d3a97fe3397b3262365ab9 100644
--- a/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/util/ListenerBus.scala
@@ -19,12 +19,11 @@ package org.apache.spark.util
import java.util.concurrent.CopyOnWriteArrayList
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import scala.util.control.NonFatal
import org.apache.spark.Logging
-import org.apache.spark.scheduler.SparkListener
/**
* An event bus which posts events to its listeners.
@@ -46,7 +45,7 @@ private[spark] trait ListenerBus[L <: AnyRef, E] extends Logging {
* `postToAll` in the same thread for all events.
*/
final def postToAll(event: E): Unit = {
- // JavaConversions will create a JIterableWrapper if we use some Scala collection functions.
+ // JavaConverters can create a JIterableWrapper if we use asScala.
// However, this method will be called frequently. To avoid the wrapper cost, here ewe use
// Java Iterator directly.
val iter = listeners.iterator
@@ -69,7 +68,7 @@ private[spark] trait ListenerBus[L <: AnyRef, E] extends Logging {
private[spark] def findListenersByClass[T <: L : ClassTag](): Seq[T] = {
val c = implicitly[ClassTag[T]].runtimeClass
- listeners.filter(_.getClass == c).map(_.asInstanceOf[T]).toSeq
+ listeners.asScala.filter(_.getClass == c).map(_.asInstanceOf[T]).toSeq
}
}
diff --git a/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala b/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala
index 169489df6c1ea4430dc4e86a10109646d9cd8746..a1c33212cdb2b188c7427e27a8505f0ecf3c78c3 100644
--- a/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala
+++ b/core/src/main/scala/org/apache/spark/util/MutableURLClassLoader.scala
@@ -21,8 +21,6 @@ import java.net.{URLClassLoader, URL}
import java.util.Enumeration
import java.util.concurrent.ConcurrentHashMap
-import scala.collection.JavaConversions._
-
/**
* URL class loader that exposes the `addURL` and `getURLs` methods in URLClassLoader.
*/
diff --git a/core/src/main/scala/org/apache/spark/util/TimeStampedHashMap.scala b/core/src/main/scala/org/apache/spark/util/TimeStampedHashMap.scala
index 8de75ba9a9c92d7cde36941d052dd4392d226acc..d7e5143c309531adc791548fcc24821c821f616a 100644
--- a/core/src/main/scala/org/apache/spark/util/TimeStampedHashMap.scala
+++ b/core/src/main/scala/org/apache/spark/util/TimeStampedHashMap.scala
@@ -21,7 +21,8 @@ import java.util.Set
import java.util.Map.Entry
import java.util.concurrent.ConcurrentHashMap
-import scala.collection.{JavaConversions, mutable}
+import scala.collection.JavaConverters._
+import scala.collection.mutable
import org.apache.spark.Logging
@@ -50,8 +51,7 @@ private[spark] class TimeStampedHashMap[A, B](updateTimeStampOnGet: Boolean = fa
}
def iterator: Iterator[(A, B)] = {
- val jIterator = getEntrySet.iterator
- JavaConversions.asScalaIterator(jIterator).map(kv => (kv.getKey, kv.getValue.value))
+ getEntrySet.iterator.asScala.map(kv => (kv.getKey, kv.getValue.value))
}
def getEntrySet: Set[Entry[A, TimeStampedValue[B]]] = internalMap.entrySet
@@ -90,9 +90,7 @@ private[spark] class TimeStampedHashMap[A, B](updateTimeStampOnGet: Boolean = fa
}
override def filter(p: ((A, B)) => Boolean): mutable.Map[A, B] = {
- JavaConversions.mapAsScalaConcurrentMap(internalMap)
- .map { case (k, TimeStampedValue(v, t)) => (k, v) }
- .filter(p)
+ internalMap.asScala.map { case (k, TimeStampedValue(v, t)) => (k, v) }.filter(p)
}
override def empty: mutable.Map[A, B] = new TimeStampedHashMap[A, B]()
diff --git a/core/src/main/scala/org/apache/spark/util/TimeStampedHashSet.scala b/core/src/main/scala/org/apache/spark/util/TimeStampedHashSet.scala
index 7cd8f28b12dd6bcefcd501ebbf1f8c2375d77c86..65efeb1f4c19c3feba0113bc50c8d1cadc12b6d5 100644
--- a/core/src/main/scala/org/apache/spark/util/TimeStampedHashSet.scala
+++ b/core/src/main/scala/org/apache/spark/util/TimeStampedHashSet.scala
@@ -19,7 +19,7 @@ package org.apache.spark.util
import java.util.concurrent.ConcurrentHashMap
-import scala.collection.JavaConversions
+import scala.collection.JavaConverters._
import scala.collection.mutable.Set
private[spark] class TimeStampedHashSet[A] extends Set[A] {
@@ -31,7 +31,7 @@ private[spark] class TimeStampedHashSet[A] extends Set[A] {
def iterator: Iterator[A] = {
val jIterator = internalMap.entrySet().iterator()
- JavaConversions.asScalaIterator(jIterator).map(_.getKey)
+ jIterator.asScala.map(_.getKey)
}
override def + (elem: A): Set[A] = {
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 8313312226713779e0dee12de81e906b0a0ca90b..2bab4af2e73ab9aad31758dcfdbab12fc154975f 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -25,7 +25,7 @@ import java.util.{Properties, Locale, Random, UUID}
import java.util.concurrent._
import javax.net.ssl.HttpsURLConnection
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.Map
import scala.collection.mutable.ArrayBuffer
import scala.io.Source
@@ -748,12 +748,12 @@ private[spark] object Utils extends Logging {
// getNetworkInterfaces returns ifs in reverse order compared to ifconfig output order
// on unix-like system. On windows, it returns in index order.
// It's more proper to pick ip address following system output order.
- val activeNetworkIFs = NetworkInterface.getNetworkInterfaces.toList
+ val activeNetworkIFs = NetworkInterface.getNetworkInterfaces.asScala.toSeq
val reOrderedNetworkIFs = if (isWindows) activeNetworkIFs else activeNetworkIFs.reverse
for (ni <- reOrderedNetworkIFs) {
- val addresses = ni.getInetAddresses.toList
- .filterNot(addr => addr.isLinkLocalAddress || addr.isLoopbackAddress)
+ val addresses = ni.getInetAddresses.asScala
+ .filterNot(addr => addr.isLinkLocalAddress || addr.isLoopbackAddress).toSeq
if (addresses.nonEmpty) {
val addr = addresses.find(_.isInstanceOf[Inet4Address]).getOrElse(addresses.head)
// because of Inet6Address.toHostName may add interface at the end if it knows about it
@@ -1498,10 +1498,8 @@ private[spark] object Utils extends Logging {
* properties which have been set explicitly, as well as those for which only a default value
* has been defined. */
def getSystemProperties: Map[String, String] = {
- val sysProps = for (key <- System.getProperties.stringPropertyNames()) yield
- (key, System.getProperty(key))
-
- sysProps.toMap
+ System.getProperties.stringPropertyNames().asScala
+ .map(key => (key, System.getProperty(key))).toMap
}
/**
@@ -1812,7 +1810,8 @@ private[spark] object Utils extends Logging {
try {
val properties = new Properties()
properties.load(inReader)
- properties.stringPropertyNames().map(k => (k, properties(k).trim)).toMap
+ properties.stringPropertyNames().asScala.map(
+ k => (k, properties.getProperty(k).trim)).toMap
} catch {
case e: IOException =>
throw new SparkException(s"Failed when loading Spark properties from $filename", e)
@@ -1941,7 +1940,8 @@ private[spark] object Utils extends Logging {
return true
}
isBindCollision(e.getCause)
- case e: MultiException => e.getThrowables.exists(isBindCollision)
+ case e: MultiException =>
+ e.getThrowables.asScala.exists(isBindCollision)
case e: Exception => isBindCollision(e.getCause)
case _ => false
}
diff --git a/core/src/main/scala/org/apache/spark/util/collection/Utils.scala b/core/src/main/scala/org/apache/spark/util/collection/Utils.scala
index bdbca00a0062219c23dc5d95b1f9f848267d5ede..4939b600dbfbd6355e534a8cd3b1c4b2b0c1bf46 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/Utils.scala
@@ -17,7 +17,7 @@
package org.apache.spark.util.collection
-import scala.collection.JavaConversions.{collectionAsScalaIterable, asJavaIterator}
+import scala.collection.JavaConverters._
import com.google.common.collect.{Ordering => GuavaOrdering}
@@ -34,6 +34,6 @@ private[spark] object Utils {
val ordering = new GuavaOrdering[T] {
override def compare(l: T, r: T): Int = ord.compare(l, r)
}
- collectionAsScalaIterable(ordering.leastOf(asJavaIterator(input), num)).iterator
+ ordering.leastOf(input.asJava, num).iterator.asScala
}
}
diff --git a/core/src/test/java/org/apache/spark/JavaAPISuite.java b/core/src/test/java/org/apache/spark/JavaAPISuite.java
index ffe4b4baffb2a69505b74d06e202dbe485fa2237..ebd3d61ae7324e2b1a3d1ceeb637b39a1f5414bd 100644
--- a/core/src/test/java/org/apache/spark/JavaAPISuite.java
+++ b/core/src/test/java/org/apache/spark/JavaAPISuite.java
@@ -24,10 +24,10 @@ import java.net.URI;
import java.util.*;
import java.util.concurrent.*;
-import scala.collection.JavaConversions;
import scala.Tuple2;
import scala.Tuple3;
import scala.Tuple4;
+import scala.collection.JavaConverters;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
@@ -1473,7 +1473,9 @@ public class JavaAPISuite implements Serializable {
Assert.assertEquals(expected, results);
Partitioner defaultPartitioner = Partitioner.defaultPartitioner(
- combinedRDD.rdd(), JavaConversions.asScalaBuffer(Lists.<RDD<?>>newArrayList()));
+ combinedRDD.rdd(),
+ JavaConverters.collectionAsScalaIterableConverter(
+ Collections.<RDD<?>>emptyList()).asScala().toSeq());
combinedRDD = originalRDD.keyBy(keyFunction)
.combineByKey(
createCombinerFunction,
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
index 90cb7da94e88ac529373ea59b483d10be0fd5401..ff9a92cc0a421b20dfa8700e58fa9170d672dc18 100644
--- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark
import java.util.concurrent.{TimeUnit, Executors}
+import scala.collection.JavaConverters._
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.{Try, Random}
@@ -148,7 +149,6 @@ class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSyst
}
test("Thread safeness - SPARK-5425") {
- import scala.collection.JavaConversions._
val executor = Executors.newSingleThreadScheduledExecutor()
val sf = executor.scheduleAtFixedRate(new Runnable {
override def run(): Unit =
@@ -163,8 +163,9 @@ class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSyst
}
} finally {
executor.shutdownNow()
- for (key <- System.getProperties.stringPropertyNames() if key.startsWith("spark.5425."))
- System.getProperties.remove(key)
+ val sysProps = System.getProperties
+ for (key <- sysProps.stringPropertyNames().asScala if key.startsWith("spark.5425."))
+ sysProps.remove(key)
}
}
diff --git a/core/src/test/scala/org/apache/spark/deploy/LogUrlsStandaloneSuite.scala b/core/src/test/scala/org/apache/spark/deploy/LogUrlsStandaloneSuite.scala
index cbd2aee10c0e2b2573d161f1705ad1528c3cbdd4..86eb41dd7e5d7e5947527df1f902e7ed4f881e83 100644
--- a/core/src/test/scala/org/apache/spark/deploy/LogUrlsStandaloneSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/LogUrlsStandaloneSuite.scala
@@ -19,7 +19,6 @@ package org.apache.spark.deploy
import java.net.URL
-import scala.collection.JavaConversions._
import scala.collection.mutable
import scala.io.Source
diff --git a/core/src/test/scala/org/apache/spark/deploy/RPackageUtilsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/RPackageUtilsSuite.scala
index 47a64081e297e79831cb97faee4b6987d29cf5d8..1ed4bae3ca21e942089ed70a9ce6102a62a45f0e 100644
--- a/core/src/test/scala/org/apache/spark/deploy/RPackageUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/RPackageUtilsSuite.scala
@@ -21,14 +21,14 @@ import java.io.{PrintStream, OutputStream, File}
import java.net.URI
import java.util.jar.Attributes.Name
import java.util.jar.{JarFile, Manifest}
-import java.util.zip.{ZipEntry, ZipFile}
+import java.util.zip.ZipFile
-import org.scalatest.BeforeAndAfterEach
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import com.google.common.io.Files
import org.apache.commons.io.FileUtils
+import org.scalatest.BeforeAndAfterEach
import org.apache.spark.SparkFunSuite
import org.apache.spark.api.r.RUtils
@@ -142,7 +142,7 @@ class RPackageUtilsSuite extends SparkFunSuite with BeforeAndAfterEach {
IvyTestUtils.writeFile(fakePackageDir, "DESCRIPTION", "abc")
val finalZip = RPackageUtils.zipRLibraries(tempDir, "sparkr.zip")
assert(finalZip.exists())
- val entries = new ZipFile(finalZip).entries().toSeq.map(_.getName)
+ val entries = new ZipFile(finalZip).entries().asScala.map(_.getName).toSeq
assert(entries.contains("/test.R"))
assert(entries.contains("/SparkR/abc.R"))
assert(entries.contains("/SparkR/DESCRIPTION"))
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
index bed6f3ea612412fdffa772f16c8d18430ac9122f..98664dc1101e6673e430c6923c9e945d658e100a 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
@@ -19,8 +19,6 @@ package org.apache.spark.deploy.worker
import java.io.File
-import scala.collection.JavaConversions._
-
import org.apache.spark.deploy.{ApplicationDescription, Command, ExecutorState}
import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
@@ -36,6 +34,7 @@ class ExecutorRunnerTest extends SparkFunSuite {
ExecutorState.RUNNING)
val builder = CommandUtils.buildProcessBuilder(
appDesc.command, new SecurityManager(conf), 512, sparkHome, er.substituteVariables)
- assert(builder.command().last === appId)
+ val builderCommand = builder.command()
+ assert(builderCommand.get(builderCommand.size() - 1) === appId)
}
}
diff --git a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
index 730535ece7878a4de13546a99c948e5a03640d7e..a9652d7e7d0b0f555f043897556dab0ec35b14d2 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.scheduler
import java.util.concurrent.Semaphore
import scala.collection.mutable
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.scalatest.Matchers
@@ -365,10 +365,9 @@ class SparkListenerSuite extends SparkFunSuite with LocalSparkContext with Match
.set("spark.extraListeners", classOf[ListenerThatAcceptsSparkConf].getName + "," +
classOf[BasicJobCounter].getName)
sc = new SparkContext(conf)
- sc.listenerBus.listeners.collect { case x: BasicJobCounter => x}.size should be (1)
- sc.listenerBus.listeners.collect {
- case x: ListenerThatAcceptsSparkConf => x
- }.size should be (1)
+ sc.listenerBus.listeners.asScala.count(_.isInstanceOf[BasicJobCounter]) should be (1)
+ sc.listenerBus.listeners.asScala
+ .count(_.isInstanceOf[ListenerThatAcceptsSparkConf]) should be (1)
}
/**
diff --git a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala
index 5ed30f64d705f26bd9dd28bb9a5c4385a578b9b3..319b3173e7a6e8d84649788198229762a18d7b34 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendSuite.scala
@@ -18,10 +18,11 @@
package org.apache.spark.scheduler.cluster.mesos
import java.nio.ByteBuffer
-import java.util
+import java.util.Arrays
+import java.util.Collection
import java.util.Collections
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
@@ -61,7 +62,7 @@ class MesosSchedulerBackendSuite extends SparkFunSuite with LocalSparkContext wi
val mesosSchedulerBackend = new MesosSchedulerBackend(taskScheduler, sc, "master")
- val resources = List(
+ val resources = Arrays.asList(
mesosSchedulerBackend.createResource("cpus", 4),
mesosSchedulerBackend.createResource("mem", 1024))
// uri is null.
@@ -98,7 +99,7 @@ class MesosSchedulerBackendSuite extends SparkFunSuite with LocalSparkContext wi
val backend = new MesosSchedulerBackend(taskScheduler, sc, "master")
val (execInfo, _) = backend.createExecutorInfo(
- List(backend.createResource("cpus", 4)), "mockExecutor")
+ Arrays.asList(backend.createResource("cpus", 4)), "mockExecutor")
assert(execInfo.getContainer.getDocker.getImage.equals("spark/mock"))
val portmaps = execInfo.getContainer.getDocker.getPortMappingsList
assert(portmaps.get(0).getHostPort.equals(80))
@@ -179,7 +180,7 @@ class MesosSchedulerBackendSuite extends SparkFunSuite with LocalSparkContext wi
when(taskScheduler.resourceOffers(expectedWorkerOffers)).thenReturn(Seq(Seq(taskDesc)))
when(taskScheduler.CPUS_PER_TASK).thenReturn(2)
- val capture = ArgumentCaptor.forClass(classOf[util.Collection[TaskInfo]])
+ val capture = ArgumentCaptor.forClass(classOf[Collection[TaskInfo]])
when(
driver.launchTasks(
Matchers.eq(Collections.singleton(mesosOffers.get(0).getId)),
@@ -279,7 +280,7 @@ class MesosSchedulerBackendSuite extends SparkFunSuite with LocalSparkContext wi
when(taskScheduler.resourceOffers(expectedWorkerOffers)).thenReturn(Seq(Seq(taskDesc)))
when(taskScheduler.CPUS_PER_TASK).thenReturn(1)
- val capture = ArgumentCaptor.forClass(classOf[util.Collection[TaskInfo]])
+ val capture = ArgumentCaptor.forClass(classOf[Collection[TaskInfo]])
when(
driver.launchTasks(
Matchers.eq(Collections.singleton(mesosOffers.get(0).getId)),
@@ -304,7 +305,7 @@ class MesosSchedulerBackendSuite extends SparkFunSuite with LocalSparkContext wi
assert(cpusDev.getName.equals("cpus"))
assert(cpusDev.getScalar.getValue.equals(1.0))
assert(cpusDev.getRole.equals("dev"))
- val executorResources = taskInfo.getExecutor.getResourcesList
+ val executorResources = taskInfo.getExecutor.getResourcesList.asScala
assert(executorResources.exists { r =>
r.getName.equals("mem") && r.getScalar.getValue.equals(484.0) && r.getRole.equals("prod")
})
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index 23a1fdb0f50099f525d7dbdef3bb7b696e98cc43..8d1c9d17e977e9d544073d5682eb9bdb3457da8a 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.serializer
import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
+import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.reflect.ClassTag
@@ -173,7 +174,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
test("asJavaIterable") {
// Serialize a collection wrapped by asJavaIterable
val ser = new KryoSerializer(conf).newInstance()
- val a = ser.serialize(scala.collection.convert.WrapAsJava.asJavaIterable(Seq(12345)))
+ val a = ser.serialize(Seq(12345).asJava)
val b = ser.deserialize[java.lang.Iterable[Int]](a)
assert(b.iterator().next() === 12345)
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 69888b2694bae20572949718c6e909f46e804750..22e30ecaf05334c4bde546bdbd26054921b2b4e3 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -21,7 +21,6 @@ import java.net.{HttpURLConnection, URL}
import javax.servlet.http.{HttpServletResponse, HttpServletRequest}
import scala.io.Source
-import scala.collection.JavaConversions._
import scala.xml.Node
import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler
@@ -341,15 +340,15 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
// The completed jobs table should have two rows. The first row will be the most recent job:
val firstRow = find(cssSelector("tbody tr")).get.underlying
val firstRowColumns = firstRow.findElements(By.tagName("td"))
- firstRowColumns(0).getText should be ("1")
- firstRowColumns(4).getText should be ("1/1 (2 skipped)")
- firstRowColumns(5).getText should be ("8/8 (16 skipped)")
+ firstRowColumns.get(0).getText should be ("1")
+ firstRowColumns.get(4).getText should be ("1/1 (2 skipped)")
+ firstRowColumns.get(5).getText should be ("8/8 (16 skipped)")
// The second row is the first run of the job, where nothing was skipped:
val secondRow = findAll(cssSelector("tbody tr")).toSeq(1).underlying
val secondRowColumns = secondRow.findElements(By.tagName("td"))
- secondRowColumns(0).getText should be ("0")
- secondRowColumns(4).getText should be ("3/3")
- secondRowColumns(5).getText should be ("24/24")
+ secondRowColumns.get(0).getText should be ("0")
+ secondRowColumns.get(4).getText should be ("3/3")
+ secondRowColumns.get(5).getText should be ("24/24")
}
}
}
@@ -502,8 +501,8 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
for {
(row, idx) <- rows.zipWithIndex
columns = row.findElements(By.tagName("td"))
- id = columns(0).getText()
- description = columns(1).getText()
+ id = columns.get(0).getText()
+ description = columns.get(1).getText()
} {
id should be (expJobInfo(idx)._1)
description should include (expJobInfo(idx)._2)
@@ -547,8 +546,8 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
for {
(row, idx) <- rows.zipWithIndex
columns = row.findElements(By.tagName("td"))
- id = columns(0).getText()
- description = columns(1).getText()
+ id = columns.get(0).getText()
+ description = columns.get(1).getText()
} {
id should be (expStageInfo(idx)._1)
description should include (expStageInfo(idx)._2)
diff --git a/examples/src/main/scala/org/apache/spark/examples/CassandraCQLTest.scala b/examples/src/main/scala/org/apache/spark/examples/CassandraCQLTest.scala
index 36832f51d2ad4393ff709f328187055fe4de5ee5..fa07c1e5017cd2e22289fb4455791b97956e2e36 100644
--- a/examples/src/main/scala/org/apache/spark/examples/CassandraCQLTest.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/CassandraCQLTest.scala
@@ -19,10 +19,7 @@
package org.apache.spark.examples
import java.nio.ByteBuffer
-
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ListBuffer
-import scala.collection.immutable.Map
+import java.util.Collections
import org.apache.cassandra.hadoop.ConfigHelper
import org.apache.cassandra.hadoop.cql3.CqlPagingInputFormat
@@ -32,7 +29,6 @@ import org.apache.cassandra.utils.ByteBufferUtil
import org.apache.hadoop.mapreduce.Job
import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.SparkContext._
/*
@@ -121,12 +117,9 @@ object CassandraCQLTest {
val casoutputCF = aggregatedRDD.map {
case (productId, saleCount) => {
- val outColFamKey = Map("prod_id" -> ByteBufferUtil.bytes(productId))
- val outKey: java.util.Map[String, ByteBuffer] = outColFamKey
- var outColFamVal = new ListBuffer[ByteBuffer]
- outColFamVal += ByteBufferUtil.bytes(saleCount)
- val outVal: java.util.List[ByteBuffer] = outColFamVal
- (outKey, outVal)
+ val outKey = Collections.singletonMap("prod_id", ByteBufferUtil.bytes(productId))
+ val outVal = Collections.singletonList(ByteBufferUtil.bytes(saleCount))
+ (outKey, outVal)
}
}
diff --git a/examples/src/main/scala/org/apache/spark/examples/CassandraTest.scala b/examples/src/main/scala/org/apache/spark/examples/CassandraTest.scala
index 96ef3e198e38063b5340c4b0cd420e3332a48d17..2e56d24c60c334933f1594f2881d1a83fcd10b9e 100644
--- a/examples/src/main/scala/org/apache/spark/examples/CassandraTest.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/CassandraTest.scala
@@ -19,10 +19,9 @@
package org.apache.spark.examples
import java.nio.ByteBuffer
+import java.util.Arrays
import java.util.SortedMap
-import scala.collection.JavaConversions._
-
import org.apache.cassandra.db.IColumn
import org.apache.cassandra.hadoop.ColumnFamilyOutputFormat
import org.apache.cassandra.hadoop.ConfigHelper
@@ -32,7 +31,6 @@ import org.apache.cassandra.utils.ByteBufferUtil
import org.apache.hadoop.mapreduce.Job
import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.SparkContext._
/*
* This example demonstrates using Spark with Cassandra with the New Hadoop API and Cassandra
@@ -118,7 +116,7 @@ object CassandraTest {
val outputkey = ByteBufferUtil.bytes(word + "-COUNT-" + System.currentTimeMillis)
- val mutations: java.util.List[Mutation] = new Mutation() :: new Mutation() :: Nil
+ val mutations = Arrays.asList(new Mutation(), new Mutation())
mutations.get(0).setColumn_or_supercolumn(new ColumnOrSuperColumn())
mutations.get(0).column_or_supercolumn.setColumn(colWord)
mutations.get(1).setColumn_or_supercolumn(new ColumnOrSuperColumn())
diff --git a/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala b/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala
index c42df2b8845d2917e20f184e159b89707ae946a4..bec61f3cd4296872d202a3ba7dc77ac271c17cf8 100644
--- a/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala
@@ -18,7 +18,7 @@
// scalastyle:off println
package org.apache.spark.examples
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.util.Utils
@@ -36,10 +36,10 @@ object DriverSubmissionTest {
val properties = Utils.getSystemProperties
println("Environment variables containing SPARK_TEST:")
- env.filter{case (k, v) => k.contains("SPARK_TEST")}.foreach(println)
+ env.asScala.filter { case (k, _) => k.contains("SPARK_TEST")}.foreach(println)
println("System properties containing spark.test:")
- properties.filter{case (k, v) => k.toString.contains("spark.test")}.foreach(println)
+ properties.filter { case (k, _) => k.toString.contains("spark.test") }.foreach(println)
for (i <- 1 until numSecondsToSleep) {
println(s"Alive for $i out of $numSecondsToSleep seconds")
diff --git a/examples/src/main/scala/org/apache/spark/examples/pythonconverters/AvroConverters.scala b/examples/src/main/scala/org/apache/spark/examples/pythonconverters/AvroConverters.scala
index 3ebb112fc069eb3d7685f106243fff0576928c06..805184e740f0602a8f7aade1616f7c6341ef31d5 100644
--- a/examples/src/main/scala/org/apache/spark/examples/pythonconverters/AvroConverters.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/pythonconverters/AvroConverters.scala
@@ -19,7 +19,7 @@ package org.apache.spark.examples.pythonconverters
import java.util.{Collection => JCollection, Map => JMap}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.avro.generic.{GenericFixed, IndexedRecord}
import org.apache.avro.mapred.AvroWrapper
@@ -58,7 +58,7 @@ object AvroConversionUtil extends Serializable {
val map = new java.util.HashMap[String, Any]
obj match {
case record: IndexedRecord =>
- record.getSchema.getFields.zipWithIndex.foreach { case (f, i) =>
+ record.getSchema.getFields.asScala.zipWithIndex.foreach { case (f, i) =>
map.put(f.name, fromAvro(record.get(i), f.schema))
}
case other => throw new SparkException(
@@ -68,9 +68,9 @@ object AvroConversionUtil extends Serializable {
}
def unpackMap(obj: Any, schema: Schema): JMap[String, Any] = {
- obj.asInstanceOf[JMap[_, _]].map { case (key, value) =>
+ obj.asInstanceOf[JMap[_, _]].asScala.map { case (key, value) =>
(key.toString, fromAvro(value, schema.getValueType))
- }
+ }.asJava
}
def unpackFixed(obj: Any, schema: Schema): Array[Byte] = {
@@ -91,17 +91,17 @@ object AvroConversionUtil extends Serializable {
def unpackArray(obj: Any, schema: Schema): JCollection[Any] = obj match {
case c: JCollection[_] =>
- c.map(fromAvro(_, schema.getElementType))
+ c.asScala.map(fromAvro(_, schema.getElementType)).toSeq.asJava
case arr: Array[_] if arr.getClass.getComponentType.isPrimitive =>
- arr.toSeq
+ arr.toSeq.asJava.asInstanceOf[JCollection[Any]]
case arr: Array[_] =>
- arr.map(fromAvro(_, schema.getElementType)).toSeq
+ arr.map(fromAvro(_, schema.getElementType)).toSeq.asJava
case other => throw new SparkException(
s"Unknown ARRAY type ${other.getClass.getName}")
}
def unpackUnion(obj: Any, schema: Schema): Any = {
- schema.getTypes.toList match {
+ schema.getTypes.asScala.toList match {
case List(s) => fromAvro(obj, s)
case List(n, s) if n.getType == NULL => fromAvro(obj, s)
case List(s, n) if n.getType == NULL => fromAvro(obj, s)
diff --git a/examples/src/main/scala/org/apache/spark/examples/pythonconverters/CassandraConverters.scala b/examples/src/main/scala/org/apache/spark/examples/pythonconverters/CassandraConverters.scala
index 83feb5703b908fdeacf0dcf79538fdcc5656a031..00ce47af4813de162d831124976e820e512aaeed 100644
--- a/examples/src/main/scala/org/apache/spark/examples/pythonconverters/CassandraConverters.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/pythonconverters/CassandraConverters.scala
@@ -17,11 +17,13 @@
package org.apache.spark.examples.pythonconverters
-import org.apache.spark.api.python.Converter
import java.nio.ByteBuffer
+
+import scala.collection.JavaConverters._
+
import org.apache.cassandra.utils.ByteBufferUtil
-import collection.JavaConversions._
+import org.apache.spark.api.python.Converter
/**
* Implementation of [[org.apache.spark.api.python.Converter]] that converts Cassandra
@@ -30,7 +32,7 @@ import collection.JavaConversions._
class CassandraCQLKeyConverter extends Converter[Any, java.util.Map[String, Int]] {
override def convert(obj: Any): java.util.Map[String, Int] = {
val result = obj.asInstanceOf[java.util.Map[String, ByteBuffer]]
- mapAsJavaMap(result.mapValues(bb => ByteBufferUtil.toInt(bb)))
+ result.asScala.mapValues(ByteBufferUtil.toInt).asJava
}
}
@@ -41,7 +43,7 @@ class CassandraCQLKeyConverter extends Converter[Any, java.util.Map[String, Int]
class CassandraCQLValueConverter extends Converter[Any, java.util.Map[String, String]] {
override def convert(obj: Any): java.util.Map[String, String] = {
val result = obj.asInstanceOf[java.util.Map[String, ByteBuffer]]
- mapAsJavaMap(result.mapValues(bb => ByteBufferUtil.string(bb)))
+ result.asScala.mapValues(ByteBufferUtil.string).asJava
}
}
@@ -52,7 +54,7 @@ class CassandraCQLValueConverter extends Converter[Any, java.util.Map[String, St
class ToCassandraCQLKeyConverter extends Converter[Any, java.util.Map[String, ByteBuffer]] {
override def convert(obj: Any): java.util.Map[String, ByteBuffer] = {
val input = obj.asInstanceOf[java.util.Map[String, Int]]
- mapAsJavaMap(input.mapValues(i => ByteBufferUtil.bytes(i)))
+ input.asScala.mapValues(ByteBufferUtil.bytes).asJava
}
}
@@ -63,6 +65,6 @@ class ToCassandraCQLKeyConverter extends Converter[Any, java.util.Map[String, By
class ToCassandraCQLValueConverter extends Converter[Any, java.util.List[ByteBuffer]] {
override def convert(obj: Any): java.util.List[ByteBuffer] = {
val input = obj.asInstanceOf[java.util.List[String]]
- seqAsJavaList(input.map(s => ByteBufferUtil.bytes(s)))
+ input.asScala.map(ByteBufferUtil.bytes).asJava
}
}
diff --git a/examples/src/main/scala/org/apache/spark/examples/pythonconverters/HBaseConverters.scala b/examples/src/main/scala/org/apache/spark/examples/pythonconverters/HBaseConverters.scala
index 90d48a64106c7555d40788a861499ea373a94ded..0a25ee7ae56f46684661ea81fd376322e5e51d27 100644
--- a/examples/src/main/scala/org/apache/spark/examples/pythonconverters/HBaseConverters.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/pythonconverters/HBaseConverters.scala
@@ -17,7 +17,7 @@
package org.apache.spark.examples.pythonconverters
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.util.parsing.json.JSONObject
import org.apache.spark.api.python.Converter
@@ -33,7 +33,6 @@ import org.apache.hadoop.hbase.CellUtil
*/
class HBaseResultToStringConverter extends Converter[Any, String] {
override def convert(obj: Any): String = {
- import collection.JavaConverters._
val result = obj.asInstanceOf[Result]
val output = result.listCells.asScala.map(cell =>
Map(
@@ -77,7 +76,7 @@ class StringToImmutableBytesWritableConverter extends Converter[Any, ImmutableBy
*/
class StringListToPutConverter extends Converter[Any, Put] {
override def convert(obj: Any): Put = {
- val output = obj.asInstanceOf[java.util.ArrayList[String]].map(Bytes.toBytes(_)).toArray
+ val output = obj.asInstanceOf[java.util.ArrayList[String]].asScala.map(Bytes.toBytes).toArray
val put = new Put(output(0))
put.add(output(1), output(2), output(3))
}
diff --git a/external/flume-sink/src/test/scala/org/apache/spark/streaming/flume/sink/SparkSinkSuite.scala b/external/flume-sink/src/test/scala/org/apache/spark/streaming/flume/sink/SparkSinkSuite.scala
index fa43629d49771d535c3c0886cf06b94399058a3e..d2654700ea729aa7a4a8ffd83a5221816c2e1fc5 100644
--- a/external/flume-sink/src/test/scala/org/apache/spark/streaming/flume/sink/SparkSinkSuite.scala
+++ b/external/flume-sink/src/test/scala/org/apache/spark/streaming/flume/sink/SparkSinkSuite.scala
@@ -20,7 +20,7 @@ import java.net.InetSocketAddress
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.{TimeUnit, CountDownLatch, Executors}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.concurrent.{ExecutionContext, Future}
import scala.util.{Failure, Success}
@@ -166,7 +166,7 @@ class SparkSinkSuite extends FunSuite {
channelContext.put("capacity", channelCapacity.toString)
channelContext.put("transactionCapacity", 1000.toString)
channelContext.put("keep-alive", 0.toString)
- channelContext.putAll(overrides)
+ channelContext.putAll(overrides.asJava)
channel.setName(scala.util.Random.nextString(10))
channel.configure(channelContext)
diff --git a/external/flume/src/main/scala/org/apache/spark/streaming/flume/EventTransformer.scala b/external/flume/src/main/scala/org/apache/spark/streaming/flume/EventTransformer.scala
index 65c49c131518b1eb484ce567124a4f5a0680c2f5..48df27b26867f0b01d0cf241cb75454e299f695e 100644
--- a/external/flume/src/main/scala/org/apache/spark/streaming/flume/EventTransformer.scala
+++ b/external/flume/src/main/scala/org/apache/spark/streaming/flume/EventTransformer.scala
@@ -19,7 +19,7 @@ package org.apache.spark.streaming.flume
import java.io.{ObjectOutput, ObjectInput}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.util.Utils
import org.apache.spark.Logging
@@ -60,7 +60,7 @@ private[streaming] object EventTransformer extends Logging {
out.write(body)
val numHeaders = headers.size()
out.writeInt(numHeaders)
- for ((k, v) <- headers) {
+ for ((k, v) <- headers.asScala) {
val keyBuff = Utils.serialize(k.toString)
out.writeInt(keyBuff.length)
out.write(keyBuff)
diff --git a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeBatchFetcher.scala b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeBatchFetcher.scala
index 88cc2aa3bf0220116e52d6d7f7eff808e492f0cc..b9d4e762ca05d18f397522d22e64d5bd0618564a 100644
--- a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeBatchFetcher.scala
+++ b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeBatchFetcher.scala
@@ -16,7 +16,6 @@
*/
package org.apache.spark.streaming.flume
-import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
import com.google.common.base.Throwables
@@ -155,7 +154,7 @@ private[flume] class FlumeBatchFetcher(receiver: FlumePollingReceiver) extends R
val buffer = new ArrayBuffer[SparkFlumeEvent](events.size())
var j = 0
while (j < events.size()) {
- val event = events(j)
+ val event = events.get(j)
val sparkFlumeEvent = new SparkFlumeEvent()
sparkFlumeEvent.event.setBody(event.getBody)
sparkFlumeEvent.event.setHeaders(event.getHeaders)
diff --git a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeInputDStream.scala b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeInputDStream.scala
index 1e32a365a1eee361fac6b12d1e99b17b5143f716..2bf99cb3cba1fab7bb5ac496dfae076f98897761 100644
--- a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeInputDStream.scala
+++ b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeInputDStream.scala
@@ -22,7 +22,7 @@ import java.io.{ObjectInput, ObjectOutput, Externalizable}
import java.nio.ByteBuffer
import java.util.concurrent.Executors
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import org.apache.flume.source.avro.AvroSourceProtocol
@@ -99,7 +99,7 @@ class SparkFlumeEvent() extends Externalizable {
val numHeaders = event.getHeaders.size()
out.writeInt(numHeaders)
- for ((k, v) <- event.getHeaders) {
+ for ((k, v) <- event.getHeaders.asScala) {
val keyBuff = Utils.serialize(k.toString)
out.writeInt(keyBuff.length)
out.write(keyBuff)
@@ -127,8 +127,7 @@ class FlumeEventServer(receiver : FlumeReceiver) extends AvroSourceProtocol {
}
override def appendBatch(events : java.util.List[AvroFlumeEvent]) : Status = {
- events.foreach (event =>
- receiver.store(SparkFlumeEvent.fromAvroFlumeEvent(event)))
+ events.asScala.foreach(event => receiver.store(SparkFlumeEvent.fromAvroFlumeEvent(event)))
Status.OK
}
}
diff --git a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumePollingInputDStream.scala b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumePollingInputDStream.scala
index 583e7dca317ad27d19bc632f6e5115822c3a91c3..0bc46209b8369abc73718d7b9a9e56ce24f38e3e 100644
--- a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumePollingInputDStream.scala
+++ b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumePollingInputDStream.scala
@@ -20,7 +20,7 @@ package org.apache.spark.streaming.flume
import java.net.InetSocketAddress
import java.util.concurrent.{LinkedBlockingQueue, Executors}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import com.google.common.util.concurrent.ThreadFactoryBuilder
@@ -94,9 +94,7 @@ private[streaming] class FlumePollingReceiver(
override def onStop(): Unit = {
logInfo("Shutting down Flume Polling Receiver")
receiverExecutor.shutdownNow()
- connections.foreach(connection => {
- connection.transceiver.close()
- })
+ connections.asScala.foreach(_.transceiver.close())
channelFactory.releaseExternalResources()
}
diff --git a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeTestUtils.scala b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeTestUtils.scala
index 9d9c3b189415f6b24410f17d2d2a3670a93ce5fa..70018c86f92beac721e8839824b0d86c772afe37 100644
--- a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeTestUtils.scala
+++ b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeTestUtils.scala
@@ -19,9 +19,9 @@ package org.apache.spark.streaming.flume
import java.net.{InetSocketAddress, ServerSocket}
import java.nio.ByteBuffer
-import java.util.{List => JList}
+import java.util.Collections
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import com.google.common.base.Charsets.UTF_8
import org.apache.avro.ipc.NettyTransceiver
@@ -59,13 +59,13 @@ private[flume] class FlumeTestUtils {
}
/** Send data to the flume receiver */
- def writeInput(input: JList[String], enableCompression: Boolean): Unit = {
+ def writeInput(input: Seq[String], enableCompression: Boolean): Unit = {
val testAddress = new InetSocketAddress("localhost", testPort)
val inputEvents = input.map { item =>
val event = new AvroFlumeEvent
event.setBody(ByteBuffer.wrap(item.getBytes(UTF_8)))
- event.setHeaders(Map[CharSequence, CharSequence]("test" -> "header"))
+ event.setHeaders(Collections.singletonMap("test", "header"))
event
}
@@ -88,7 +88,7 @@ private[flume] class FlumeTestUtils {
}
// Send data
- val status = client.appendBatch(inputEvents.toList)
+ val status = client.appendBatch(inputEvents.asJava)
if (status != avro.Status.OK) {
throw new AssertionError("Sent events unsuccessfully")
}
diff --git a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeUtils.scala b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeUtils.scala
index a65a9b921aafaf6ef3a2530ac9e555fcfa91bdf6..c719b80aca7ed9ab07debc10b60ea1e840ab9af2 100644
--- a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeUtils.scala
+++ b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeUtils.scala
@@ -21,7 +21,7 @@ import java.net.InetSocketAddress
import java.io.{DataOutputStream, ByteArrayOutputStream}
import java.util.{List => JList, Map => JMap}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.api.java.function.PairFunction
import org.apache.spark.api.python.PythonRDD
@@ -268,8 +268,8 @@ private[flume] class FlumeUtilsPythonHelper {
maxBatchSize: Int,
parallelism: Int
): JavaPairDStream[Array[Byte], Array[Byte]] = {
- assert(hosts.length == ports.length)
- val addresses = hosts.zip(ports).map {
+ assert(hosts.size() == ports.size())
+ val addresses = hosts.asScala.zip(ports.asScala).map {
case (host, port) => new InetSocketAddress(host, port)
}
val dstream = FlumeUtils.createPollingStream(
@@ -286,7 +286,7 @@ private object FlumeUtilsPythonHelper {
val output = new DataOutputStream(byteStream)
try {
output.writeInt(map.size)
- map.foreach { kv =>
+ map.asScala.foreach { kv =>
PythonRDD.writeUTF(kv._1.toString, output)
PythonRDD.writeUTF(kv._2.toString, output)
}
diff --git a/external/flume/src/main/scala/org/apache/spark/streaming/flume/PollingFlumeTestUtils.scala b/external/flume/src/main/scala/org/apache/spark/streaming/flume/PollingFlumeTestUtils.scala
index 91d63d49dbec34d077c64c9c24382324d0e99c34..a2ab320957db322c09d0db69075d92083b2d462b 100644
--- a/external/flume/src/main/scala/org/apache/spark/streaming/flume/PollingFlumeTestUtils.scala
+++ b/external/flume/src/main/scala/org/apache/spark/streaming/flume/PollingFlumeTestUtils.scala
@@ -18,9 +18,8 @@
package org.apache.spark.streaming.flume
import java.util.concurrent._
-import java.util.{List => JList, Map => JMap}
+import java.util.{Map => JMap, Collections}
-import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
import com.google.common.base.Charsets.UTF_8
@@ -77,7 +76,7 @@ private[flume] class PollingFlumeTestUtils {
/**
* Start 2 sinks and return the ports
*/
- def startMultipleSinks(): JList[Int] = {
+ def startMultipleSinks(): Seq[Int] = {
channels.clear()
sinks.clear()
@@ -138,8 +137,7 @@ private[flume] class PollingFlumeTestUtils {
/**
* A Python-friendly method to assert the output
*/
- def assertOutput(
- outputHeaders: JList[JMap[String, String]], outputBodies: JList[String]): Unit = {
+ def assertOutput(outputHeaders: Seq[JMap[String, String]], outputBodies: Seq[String]): Unit = {
require(outputHeaders.size == outputBodies.size)
val eventSize = outputHeaders.size
if (eventSize != totalEventsPerChannel * channels.size) {
@@ -149,12 +147,12 @@ private[flume] class PollingFlumeTestUtils {
var counter = 0
for (k <- 0 until channels.size; i <- 0 until totalEventsPerChannel) {
val eventBodyToVerify = s"${channels(k).getName}-$i"
- val eventHeaderToVerify: JMap[String, String] = Map[String, String](s"test-$i" -> "header")
+ val eventHeaderToVerify: JMap[String, String] = Collections.singletonMap(s"test-$i", "header")
var found = false
var j = 0
while (j < eventSize && !found) {
- if (eventBodyToVerify == outputBodies.get(j) &&
- eventHeaderToVerify == outputHeaders.get(j)) {
+ if (eventBodyToVerify == outputBodies(j) &&
+ eventHeaderToVerify == outputHeaders(j)) {
found = true
counter += 1
}
@@ -195,7 +193,7 @@ private[flume] class PollingFlumeTestUtils {
tx.begin()
for (j <- 0 until eventsPerBatch) {
channel.put(EventBuilder.withBody(s"${channel.getName}-$t".getBytes(UTF_8),
- Map[String, String](s"test-$t" -> "header")))
+ Collections.singletonMap(s"test-$t", "header")))
t += 1
}
tx.commit()
diff --git a/external/flume/src/test/scala/org/apache/spark/streaming/flume/FlumePollingStreamSuite.scala b/external/flume/src/test/scala/org/apache/spark/streaming/flume/FlumePollingStreamSuite.scala
index d5f9a0aa38f9f210a85853456705065802de9ff4..ff2fb8eed204c79c0e128bacb2185e1d352a8a31 100644
--- a/external/flume/src/test/scala/org/apache/spark/streaming/flume/FlumePollingStreamSuite.scala
+++ b/external/flume/src/test/scala/org/apache/spark/streaming/flume/FlumePollingStreamSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.streaming.flume
import java.net.InetSocketAddress
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.{SynchronizedBuffer, ArrayBuffer}
import scala.concurrent.duration._
import scala.language.postfixOps
@@ -116,9 +116,9 @@ class FlumePollingStreamSuite extends SparkFunSuite with BeforeAndAfter with Log
// The eventually is required to ensure that all data in the batch has been processed.
eventually(timeout(10 seconds), interval(100 milliseconds)) {
val flattenOutputBuffer = outputBuffer.flatten
- val headers = flattenOutputBuffer.map(_.event.getHeaders.map {
- case kv => (kv._1.toString, kv._2.toString)
- }).map(mapAsJavaMap)
+ val headers = flattenOutputBuffer.map(_.event.getHeaders.asScala.map {
+ case (key, value) => (key.toString, value.toString)
+ }).map(_.asJava)
val bodies = flattenOutputBuffer.map(e => new String(e.event.getBody.array(), UTF_8))
utils.assertOutput(headers, bodies)
}
diff --git a/external/flume/src/test/scala/org/apache/spark/streaming/flume/FlumeStreamSuite.scala b/external/flume/src/test/scala/org/apache/spark/streaming/flume/FlumeStreamSuite.scala
index 5bc4cdf65306c9559aeec841bd30ecdb19468409..5ffb60bd602f9d62a8dca4ab9838e2caedf25520 100644
--- a/external/flume/src/test/scala/org/apache/spark/streaming/flume/FlumeStreamSuite.scala
+++ b/external/flume/src/test/scala/org/apache/spark/streaming/flume/FlumeStreamSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.streaming.flume
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, SynchronizedBuffer}
import scala.concurrent.duration._
import scala.language.postfixOps
diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaTestUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaTestUtils.scala
index 79a9db4291befe1e43c2511da14a21409edaa296..c9fd715d3d554e0f28c7a92de21f3da1af218f73 100644
--- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaTestUtils.scala
+++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaTestUtils.scala
@@ -24,6 +24,7 @@ import java.util.concurrent.TimeoutException
import java.util.{Map => JMap, Properties}
import scala.annotation.tailrec
+import scala.collection.JavaConverters._
import scala.language.postfixOps
import scala.util.control.NonFatal
@@ -159,8 +160,7 @@ private[kafka] class KafkaTestUtils extends Logging {
/** Java-friendly function for sending messages to the Kafka broker */
def sendMessages(topic: String, messageToFreq: JMap[String, JInt]): Unit = {
- import scala.collection.JavaConversions._
- sendMessages(topic, Map(messageToFreq.mapValues(_.intValue()).toSeq: _*))
+ sendMessages(topic, Map(messageToFreq.asScala.mapValues(_.intValue()).toSeq: _*))
}
/** Send the messages to the Kafka broker */
diff --git a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala
index 388dbb8184106fc9c717ca5c86953836eeec6f2a..31282220775375b9e6ec417954d912d2f094bf2a 100644
--- a/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala
+++ b/external/kafka/src/main/scala/org/apache/spark/streaming/kafka/KafkaUtils.scala
@@ -20,7 +20,7 @@ package org.apache.spark.streaming.kafka
import java.lang.{Integer => JInt, Long => JLong}
import java.util.{List => JList, Map => JMap, Set => JSet}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import kafka.common.TopicAndPartition
@@ -96,7 +96,7 @@ object KafkaUtils {
groupId: String,
topics: JMap[String, JInt]
): JavaPairReceiverInputDStream[String, String] = {
- createStream(jssc.ssc, zkQuorum, groupId, Map(topics.mapValues(_.intValue()).toSeq: _*))
+ createStream(jssc.ssc, zkQuorum, groupId, Map(topics.asScala.mapValues(_.intValue()).toSeq: _*))
}
/**
@@ -115,7 +115,7 @@ object KafkaUtils {
topics: JMap[String, JInt],
storageLevel: StorageLevel
): JavaPairReceiverInputDStream[String, String] = {
- createStream(jssc.ssc, zkQuorum, groupId, Map(topics.mapValues(_.intValue()).toSeq: _*),
+ createStream(jssc.ssc, zkQuorum, groupId, Map(topics.asScala.mapValues(_.intValue()).toSeq: _*),
storageLevel)
}
@@ -149,7 +149,10 @@ object KafkaUtils {
implicit val valueCmd: ClassTag[T] = ClassTag(valueDecoderClass)
createStream[K, V, U, T](
- jssc.ssc, kafkaParams.toMap, Map(topics.mapValues(_.intValue()).toSeq: _*), storageLevel)
+ jssc.ssc,
+ kafkaParams.asScala.toMap,
+ Map(topics.asScala.mapValues(_.intValue()).toSeq: _*),
+ storageLevel)
}
/** get leaders for the given offset ranges, or throw an exception */
@@ -275,7 +278,7 @@ object KafkaUtils {
implicit val keyDecoderCmt: ClassTag[KD] = ClassTag(keyDecoderClass)
implicit val valueDecoderCmt: ClassTag[VD] = ClassTag(valueDecoderClass)
new JavaPairRDD(createRDD[K, V, KD, VD](
- jsc.sc, Map(kafkaParams.toSeq: _*), offsetRanges))
+ jsc.sc, Map(kafkaParams.asScala.toSeq: _*), offsetRanges))
}
/**
@@ -311,9 +314,9 @@ object KafkaUtils {
implicit val keyDecoderCmt: ClassTag[KD] = ClassTag(keyDecoderClass)
implicit val valueDecoderCmt: ClassTag[VD] = ClassTag(valueDecoderClass)
implicit val recordCmt: ClassTag[R] = ClassTag(recordClass)
- val leaderMap = Map(leaders.toSeq: _*)
+ val leaderMap = Map(leaders.asScala.toSeq: _*)
createRDD[K, V, KD, VD, R](
- jsc.sc, Map(kafkaParams.toSeq: _*), offsetRanges, leaderMap, messageHandler.call _)
+ jsc.sc, Map(kafkaParams.asScala.toSeq: _*), offsetRanges, leaderMap, messageHandler.call(_))
}
/**
@@ -476,8 +479,8 @@ object KafkaUtils {
val cleanedHandler = jssc.sparkContext.clean(messageHandler.call _)
createDirectStream[K, V, KD, VD, R](
jssc.ssc,
- Map(kafkaParams.toSeq: _*),
- Map(fromOffsets.mapValues { _.longValue() }.toSeq: _*),
+ Map(kafkaParams.asScala.toSeq: _*),
+ Map(fromOffsets.asScala.mapValues(_.longValue()).toSeq: _*),
cleanedHandler
)
}
@@ -531,8 +534,8 @@ object KafkaUtils {
implicit val valueDecoderCmt: ClassTag[VD] = ClassTag(valueDecoderClass)
createDirectStream[K, V, KD, VD](
jssc.ssc,
- Map(kafkaParams.toSeq: _*),
- Set(topics.toSeq: _*)
+ Map(kafkaParams.asScala.toSeq: _*),
+ Set(topics.asScala.toSeq: _*)
)
}
}
@@ -602,10 +605,10 @@ private[kafka] class KafkaUtilsPythonHelper {
): JavaPairInputDStream[Array[Byte], Array[Byte]] = {
if (!fromOffsets.isEmpty) {
- import scala.collection.JavaConversions._
- val topicsFromOffsets = fromOffsets.keySet().map(_.topic)
- if (topicsFromOffsets != topics.toSet) {
- throw new IllegalStateException(s"The specified topics: ${topics.toSet.mkString(" ")} " +
+ val topicsFromOffsets = fromOffsets.keySet().asScala.map(_.topic)
+ if (topicsFromOffsets != topics.asScala.toSet) {
+ throw new IllegalStateException(
+ s"The specified topics: ${topics.asScala.toSet.mkString(" ")} " +
s"do not equal to the topic from offsets: ${topicsFromOffsets.mkString(" ")}")
}
}
@@ -663,6 +666,6 @@ private[kafka] class KafkaUtilsPythonHelper {
"with this RDD, please call this method only on a Kafka RDD.")
val kafkaRDD = kafkaRDDs.head.asInstanceOf[KafkaRDD[_, _, _, _, _]]
- kafkaRDD.offsetRanges.toSeq
+ kafkaRDD.offsetRanges.toSeq.asJava
}
}
diff --git a/external/zeromq/src/main/scala/org/apache/spark/streaming/zeromq/ZeroMQUtils.scala b/external/zeromq/src/main/scala/org/apache/spark/streaming/zeromq/ZeroMQUtils.scala
index 0469d0af8864ac4204b5259e181231338755de56..4ea218eaa4de1ce4548475c1d34643cd5ff56027 100644
--- a/external/zeromq/src/main/scala/org/apache/spark/streaming/zeromq/ZeroMQUtils.scala
+++ b/external/zeromq/src/main/scala/org/apache/spark/streaming/zeromq/ZeroMQUtils.scala
@@ -18,15 +18,17 @@
package org.apache.spark.streaming.zeromq
import scala.reflect.ClassTag
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
+
import akka.actor.{Props, SupervisorStrategy}
import akka.util.ByteString
import akka.zeromq.Subscribe
+
import org.apache.spark.api.java.function.{Function => JFunction}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.api.java.{JavaReceiverInputDStream, JavaStreamingContext}
-import org.apache.spark.streaming.dstream.{ReceiverInputDStream}
+import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.receiver.ActorSupervisorStrategy
object ZeroMQUtils {
@@ -75,7 +77,8 @@ object ZeroMQUtils {
): JavaReceiverInputDStream[T] = {
implicit val cm: ClassTag[T] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
- val fn = (x: Seq[ByteString]) => bytesToObjects.call(x.map(_.toArray).toArray).toIterator
+ val fn =
+ (x: Seq[ByteString]) => bytesToObjects.call(x.map(_.toArray).toArray).iterator().asScala
createStream[T](jssc.ssc, publisherUrl, subscribe, fn, storageLevel, supervisorStrategy)
}
@@ -99,7 +102,8 @@ object ZeroMQUtils {
): JavaReceiverInputDStream[T] = {
implicit val cm: ClassTag[T] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
- val fn = (x: Seq[ByteString]) => bytesToObjects.call(x.map(_.toArray).toArray).toIterator
+ val fn =
+ (x: Seq[ByteString]) => bytesToObjects.call(x.map(_.toArray).toArray).iterator().asScala
createStream[T](jssc.ssc, publisherUrl, subscribe, fn, storageLevel)
}
@@ -122,7 +126,8 @@ object ZeroMQUtils {
): JavaReceiverInputDStream[T] = {
implicit val cm: ClassTag[T] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
- val fn = (x: Seq[ByteString]) => bytesToObjects.call(x.map(_.toArray).toArray).toIterator
+ val fn =
+ (x: Seq[ByteString]) => bytesToObjects.call(x.map(_.toArray).toArray).iterator().asScala
createStream[T](jssc.ssc, publisherUrl, subscribe, fn)
}
}
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala b/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
index a003ddf325e6eeb7fabcfbb3fedf9f23393b2f58..5d32fa699ae5ba0896b1a3d577b5e372ec2ede34 100644
--- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
+++ b/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala
@@ -17,7 +17,7 @@
package org.apache.spark.streaming.kinesis
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.util.control.NonFatal
import com.amazonaws.auth.{AWSCredentials, DefaultAWSCredentialsProviderChain}
@@ -213,7 +213,7 @@ class KinesisSequenceRangeIterator(
s"getting records using shard iterator") {
client.getRecords(getRecordsRequest)
}
- (getRecordsResult.getRecords.iterator(), getRecordsResult.getNextShardIterator)
+ (getRecordsResult.getRecords.iterator().asScala, getRecordsResult.getNextShardIterator)
}
/**
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala b/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
index 22324e821ce943d2fb712dd7bccbfeeb3b1f6fe5..6e0988c1af8a12b0ca9f63f005481a5f06769c6f 100644
--- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
+++ b/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala
@@ -18,7 +18,7 @@ package org.apache.spark.streaming.kinesis
import java.util.UUID
-import scala.collection.JavaConversions.asScalaIterator
+import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.util.control.NonFatal
@@ -202,7 +202,7 @@ private[kinesis] class KinesisReceiver(
/** Add records of the given shard to the current block being generated */
private[kinesis] def addRecords(shardId: String, records: java.util.List[Record]): Unit = {
if (records.size > 0) {
- val dataIterator = records.iterator().map { record =>
+ val dataIterator = records.iterator().asScala.map { record =>
val byteBuffer = record.getData()
val byteArray = new Array[Byte](byteBuffer.remaining())
byteBuffer.get(byteArray)
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala b/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
index c8eec13ec7dc7f0890fe4126b6ea7e33dcfce197..634bf945210791b236594470236d732282e8bdb7 100644
--- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
+++ b/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala
@@ -20,6 +20,7 @@ package org.apache.spark.streaming.kinesis
import java.nio.ByteBuffer
import java.util.concurrent.TimeUnit
+import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import scala.util.{Failure, Random, Success, Try}
@@ -115,7 +116,7 @@ private[kinesis] class KinesisTestUtils extends Logging {
* Expose a Python friendly API.
*/
def pushData(testData: java.util.List[Int]): Unit = {
- pushData(scala.collection.JavaConversions.asScalaBuffer(testData))
+ pushData(testData.asScala)
}
def deleteStream(): Unit = {
diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala b/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
index ceb135e0651aa515e4ed204b82ac125a60e8685c..3d136aec2e7025fcc3a3a33b4c50fb21ca387c36 100644
--- a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
+++ b/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala
@@ -17,10 +17,10 @@
package org.apache.spark.streaming.kinesis
import java.nio.ByteBuffer
+import java.nio.charset.StandardCharsets
+import java.util.Arrays
-import scala.collection.JavaConversions.seqAsJavaList
-
-import com.amazonaws.services.kinesis.clientlibrary.exceptions.{InvalidStateException, KinesisClientLibDependencyException, ShutdownException, ThrottlingException}
+import com.amazonaws.services.kinesis.clientlibrary.exceptions._
import com.amazonaws.services.kinesis.clientlibrary.interfaces.IRecordProcessorCheckpointer
import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownReason
import com.amazonaws.services.kinesis.model.Record
@@ -47,10 +47,10 @@ class KinesisReceiverSuite extends TestSuiteBase with Matchers with BeforeAndAft
val someSeqNum = Some(seqNum)
val record1 = new Record()
- record1.setData(ByteBuffer.wrap("Spark In Action".getBytes()))
+ record1.setData(ByteBuffer.wrap("Spark In Action".getBytes(StandardCharsets.UTF_8)))
val record2 = new Record()
- record2.setData(ByteBuffer.wrap("Learning Spark".getBytes()))
- val batch = List[Record](record1, record2)
+ record2.setData(ByteBuffer.wrap("Learning Spark".getBytes(StandardCharsets.UTF_8)))
+ val batch = Arrays.asList(record1, record2)
var receiverMock: KinesisReceiver = _
var checkpointerMock: IRecordProcessorCheckpointer = _
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
index 87eeb5db05d26a44c0a35d37b2c2d8c493fefe81..7a1c7796065eefe89e5e9ba008896ded7823bd3f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
@@ -17,7 +17,7 @@
package org.apache.spark.mllib.util
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.util.Random
import com.github.fommil.netlib.BLAS.{getInstance => blas}
@@ -52,7 +52,7 @@ object LinearDataGenerator {
nPoints: Int,
seed: Int,
eps: Double): java.util.List[LabeledPoint] = {
- seqAsJavaList(generateLinearInput(intercept, weights, nPoints, seed, eps))
+ generateLinearInput(intercept, weights, nPoints, seed, eps).asJava
}
/**
diff --git a/mllib/src/test/java/org/apache/spark/ml/classification/JavaOneVsRestSuite.java b/mllib/src/test/java/org/apache/spark/ml/classification/JavaOneVsRestSuite.java
index a1ee5541523722f589d81be1269502901c7fc9f7..2744e020e9e4933beb128ac398a019fc8ec27b06 100644
--- a/mllib/src/test/java/org/apache/spark/ml/classification/JavaOneVsRestSuite.java
+++ b/mllib/src/test/java/org/apache/spark/ml/classification/JavaOneVsRestSuite.java
@@ -20,7 +20,7 @@ package org.apache.spark.ml.classification;
import java.io.Serializable;
import java.util.List;
-import static scala.collection.JavaConversions.seqAsJavaList;
+import scala.collection.JavaConverters;
import org.junit.After;
import org.junit.Assert;
@@ -55,8 +55,9 @@ public class JavaOneVsRestSuite implements Serializable {
double[] xMean = {5.843, 3.057, 3.758, 1.199};
double[] xVariance = {0.6856, 0.1899, 3.116, 0.581};
- List<LabeledPoint> points = seqAsJavaList(generateMultinomialLogisticInput(
- weights, xMean, xVariance, true, nPoints, 42));
+ List<LabeledPoint> points = JavaConverters.asJavaListConverter(
+ generateMultinomialLogisticInput(weights, xMean, xVariance, true, nPoints, 42)
+ ).asJava();
datasetRDD = jsc.parallelize(points, 2);
dataset = jsql.createDataFrame(datasetRDD, LabeledPoint.class);
}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
index 2473510e13514e401ff8aff62a3630186ed657f5..8d14bb6572155933ec16a32f051c3c69bd9a4e7f 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/classification/LogisticRegressionSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.mllib.classification
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.util.Random
import scala.util.control.Breaks._
@@ -38,7 +38,7 @@ object LogisticRegressionSuite {
scale: Double,
nPoints: Int,
seed: Int): java.util.List[LabeledPoint] = {
- seqAsJavaList(generateLogisticInput(offset, scale, nPoints, seed))
+ generateLogisticInput(offset, scale, nPoints, seed).asJava
}
// Generate input of the form Y = logistic(offset + scale*X)
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/classification/SVMSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/classification/SVMSuite.scala
index b1d78cba9e3dcf5be9faf78476ade78d47eb720a..ee3c85d09a46322ad7dfe8fda786ce83dc9276fe 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/classification/SVMSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/classification/SVMSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.mllib.classification
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.util.Random
import org.jblas.DoubleMatrix
@@ -35,7 +35,7 @@ object SVMSuite {
weights: Array[Double],
nPoints: Int,
seed: Int): java.util.List[LabeledPoint] = {
- seqAsJavaList(generateSVMInput(intercept, weights, nPoints, seed))
+ generateSVMInput(intercept, weights, nPoints, seed).asJava
}
// Generate noisy input of the form Y = signum(x.dot(weights) + intercept + noise)
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala
index 13b754a03943a9cdde05575568adff3e49c05292..36ac7d267243dc555c6a18fe467ae290676b9ed4 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/optimization/GradientDescentSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.mllib.optimization
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.util.Random
import org.scalatest.Matchers
@@ -35,7 +35,7 @@ object GradientDescentSuite {
scale: Double,
nPoints: Int,
seed: Int): java.util.List[LabeledPoint] = {
- seqAsJavaList(generateGDInput(offset, scale, nPoints, seed))
+ generateGDInput(offset, scale, nPoints, seed).asJava
}
// Generate input of the form Y = logistic(offset + scale * X)
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/recommendation/ALSSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/recommendation/ALSSuite.scala
index 05b87728d6fdb8ab35e220bbb6706053053bce83..045135f7f8d60a510806f5790a36ee62da2f05fc 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/recommendation/ALSSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.mllib.recommendation
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.math.abs
import scala.util.Random
@@ -38,7 +38,7 @@ object ALSSuite {
negativeWeights: Boolean): (java.util.List[Rating], DoubleMatrix, DoubleMatrix) = {
val (sampledRatings, trueRatings, truePrefs) =
generateRatings(users, products, features, samplingRate, implicitPrefs)
- (seqAsJavaList(sampledRatings), trueRatings, truePrefs)
+ (sampledRatings.asJava, trueRatings, truePrefs)
}
def generateRatings(
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 04e0d49b178cfad36898d140017f42416b0bb9ab..ea52bfd67944a98cb40d4ef9c4f2fb5cf6da358c 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -18,13 +18,13 @@
import java.io._
import scala.util.Properties
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import sbt._
import sbt.Classpaths.publishTask
import sbt.Keys._
import sbtunidoc.Plugin.UnidocKeys.unidocGenjavadocVersion
-import com.typesafe.sbt.pom.{loadEffectivePom, PomBuild, SbtPomKeys}
+import com.typesafe.sbt.pom.{PomBuild, SbtPomKeys}
import net.virtualvoid.sbt.graph.Plugin.graphSettings
import spray.revolver.RevolverPlugin._
@@ -120,7 +120,7 @@ object SparkBuild extends PomBuild {
case _ =>
}
- override val userPropertiesMap = System.getProperties.toMap
+ override val userPropertiesMap = System.getProperties.asScala.toMap
lazy val MavenCompile = config("m2r") extend(Compile)
lazy val publishLocalBoth = TaskKey[Unit]("publish-local", "publish local for m2 and ivy")
@@ -559,7 +559,7 @@ object TestSettings {
javaOptions in Test += "-Dspark.unsafe.exceptionOnMemoryLeak=true",
javaOptions in Test += "-Dsun.io.serialization.extendedDebugInfo=true",
javaOptions in Test += "-Dderby.system.durability=test",
- javaOptions in Test ++= System.getProperties.filter(_._1 startsWith "spark")
+ javaOptions in Test ++= System.getProperties.asScala.filter(_._1.startsWith("spark"))
.map { case (k,v) => s"-D$k=$v" }.toSeq,
javaOptions in Test += "-ea",
javaOptions in Test ++= "-Xmx3g -Xss4096k -XX:PermSize=128M -XX:MaxNewSize=256m -XX:MaxPermSize=1g"
diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py
index 8af8637cf948de4932fb393318caf7f5f3fb96bc..0948f9b27cd382cbef2fefe8a9e2249facb1a1e4 100644
--- a/python/pyspark/sql/column.py
+++ b/python/pyspark/sql/column.py
@@ -61,6 +61,18 @@ def _to_seq(sc, cols, converter=None):
return sc._jvm.PythonUtils.toSeq(cols)
+def _to_list(sc, cols, converter=None):
+ """
+ Convert a list of Column (or names) into a JVM (Scala) List of Column.
+
+ An optional `converter` could be used to convert items in `cols`
+ into JVM Column objects.
+ """
+ if converter:
+ cols = [converter(c) for c in cols]
+ return sc._jvm.PythonUtils.toList(cols)
+
+
def _unary_op(name, doc="unary operator"):
""" Create a method for given unary operator """
def _(self):
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 025811f5192935d8c17b10bbf77926738cabba9d..e269ef4304f3f3c66d7abbf708a118f1da678951 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -32,7 +32,7 @@ from pyspark.storagelevel import StorageLevel
from pyspark.traceback_utils import SCCallSiteSync
from pyspark.sql import since
from pyspark.sql.types import _parse_datatype_json_string
-from pyspark.sql.column import Column, _to_seq, _to_java_column
+from pyspark.sql.column import Column, _to_seq, _to_list, _to_java_column
from pyspark.sql.readwriter import DataFrameWriter
from pyspark.sql.types import *
@@ -494,7 +494,7 @@ class DataFrame(object):
if w < 0.0:
raise ValueError("Weights must be positive. Found weight value: %s" % w)
seed = seed if seed is not None else random.randint(0, sys.maxsize)
- rdd_array = self._jdf.randomSplit(_to_seq(self.sql_ctx._sc, weights), long(seed))
+ rdd_array = self._jdf.randomSplit(_to_list(self.sql_ctx._sc, weights), long(seed))
return [DataFrame(rdd, self.sql_ctx) for rdd in rdd_array]
@property
diff --git a/scalastyle-config.xml b/scalastyle-config.xml
index b5e2e882d22546ece18a19eb98f782d9107b58c2..68fdb4141cf27d8cfd85e23be8af6f1bf4540663 100644
--- a/scalastyle-config.xml
+++ b/scalastyle-config.xml
@@ -161,6 +161,13 @@ This file is divided into 3 sections:
]]></customMessage>
</check>
+ <!-- As of SPARK-9613 JavaConversions should be replaced with JavaConverters -->
+ <check customId="javaconversions" level="error" class="org.scalastyle.scalariform.TokenChecker" enabled="true">
+ <parameters><parameter name="regex">JavaConversions</parameter></parameters>
+ <customMessage>Instead of importing implicits in scala.collection.JavaConversions._, import
+ scala.collection.JavaConverters._ and use .asScala / .asJava methods</customMessage>
+ </check>
+
<!-- ================================================================================ -->
<!-- rules we'd like to enforce, but haven't cleaned up the codebase yet -->
<!-- ================================================================================ -->
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
index ec895af9c30370b51d59b7087d8f287e5bd3a587..cfd9cb0e6259880df15c0c14c4bf82648c9dbf38 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql
+import scala.collection.JavaConverters._
+
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.GenericRow
import org.apache.spark.sql.types.StructType
@@ -280,9 +282,8 @@ trait Row extends Serializable {
*
* @throws ClassCastException when data type does not match.
*/
- def getList[T](i: Int): java.util.List[T] = {
- scala.collection.JavaConversions.seqAsJavaList(getSeq[T](i))
- }
+ def getList[T](i: Int): java.util.List[T] =
+ getSeq[T](i).asJava
/**
* Returns the value at position i of map type as a Scala Map.
@@ -296,9 +297,8 @@ trait Row extends Serializable {
*
* @throws ClassCastException when data type does not match.
*/
- def getJavaMap[K, V](i: Int): java.util.Map[K, V] = {
- scala.collection.JavaConversions.mapAsJavaMap(getMap[K, V](i))
- }
+ def getJavaMap[K, V](i: Int): java.util.Map[K, V] =
+ getMap[K, V](i).asJava
/**
* Returns the value at position i of struct type as an [[Row]] object.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
index 503c4f4b20f38b35fed5670925c1029de39d3a0a..4cc9a5520a08509a58b8516c20c0934f6acdeb7b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Catalog.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.analysis
import java.util.concurrent.ConcurrentHashMap
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
@@ -147,7 +147,7 @@ class SimpleCatalog(val conf: CatalystConf) extends Catalog {
override def getTables(databaseName: Option[String]): Seq[(String, Boolean)] = {
val result = ArrayBuffer.empty[(String, Boolean)]
- for (name <- tables.keySet()) {
+ for (name <- tables.keySet().asScala) {
result += ((name, true))
}
result
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
index a4fd4cf3b330b36b5fae21165e0736b6ee34d13d..77a42c0873a6b2abc540df2c492e78cc024aa50a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql
import java.{lang => jl}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.annotation.Experimental
import org.apache.spark.sql.catalyst.expressions._
@@ -209,7 +209,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
*
* @since 1.3.1
*/
- def fill(valueMap: java.util.Map[String, Any]): DataFrame = fill0(valueMap.toSeq)
+ def fill(valueMap: java.util.Map[String, Any]): DataFrame = fill0(valueMap.asScala.toSeq)
/**
* (Scala-specific) Returns a new [[DataFrame]] that replaces null values.
@@ -254,7 +254,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
* @since 1.3.1
*/
def replace[T](col: String, replacement: java.util.Map[T, T]): DataFrame = {
- replace[T](col, replacement.toMap : Map[T, T])
+ replace[T](col, replacement.asScala.toMap)
}
/**
@@ -277,7 +277,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
* @since 1.3.1
*/
def replace[T](cols: Array[String], replacement: java.util.Map[T, T]): DataFrame = {
- replace(cols.toSeq, replacement.toMap)
+ replace(cols.toSeq, replacement.asScala.toMap)
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index 6dc7bfe333498777e1aebd6e30b7848040778e35..97a8b6518a8322c4787d0bbfb2045c0b58980f0a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql
import java.util.Properties
+import scala.collection.JavaConverters._
+
import org.apache.hadoop.fs.Path
import org.apache.spark.annotation.Experimental
@@ -90,7 +92,7 @@ class DataFrameReader private[sql](sqlContext: SQLContext) extends Logging {
* @since 1.4.0
*/
def options(options: java.util.Map[String, String]): DataFrameReader = {
- this.options(scala.collection.JavaConversions.mapAsScalaMap(options))
+ this.options(options.asScala)
this
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index ce8744b53175b622420dbb6fed4dc515760fc660..b2a66dd417b4c24dc7750456a5ec3b87e7f7e43b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql
import java.util.Properties
+import scala.collection.JavaConverters._
+
import org.apache.spark.annotation.Experimental
import org.apache.spark.sql.catalyst.{SqlParser, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
@@ -109,7 +111,7 @@ final class DataFrameWriter private[sql](df: DataFrame) {
* @since 1.4.0
*/
def options(options: java.util.Map[String, String]): DataFrameWriter = {
- this.options(scala.collection.JavaConversions.mapAsScalaMap(options))
+ this.options(options.asScala)
this
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/GroupedData.scala b/sql/core/src/main/scala/org/apache/spark/sql/GroupedData.scala
index 99d557b03a033fc7cfab6dd6fc98776910359fc9..ee31d83cce42ce7edfa45f054d3640e7be39d40b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/GroupedData.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/GroupedData.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.language.implicitConversions
import org.apache.spark.annotation.Experimental
@@ -188,7 +188,7 @@ class GroupedData protected[sql](
* @since 1.3.0
*/
def agg(exprs: java.util.Map[String, String]): DataFrame = {
- agg(exprs.toMap)
+ agg(exprs.asScala.toMap)
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
index e9de14f0255029c3d4f5baa8dace7d520d6eeb19..e6f7619519e6aff6448813f56eedb56a55f7b9a5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql
import java.util.Properties
import scala.collection.immutable
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.parquet.hadoop.ParquetOutputCommitter
@@ -531,7 +531,7 @@ private[sql] class SQLConf extends Serializable with CatalystConf {
/** Set Spark SQL configuration properties. */
def setConf(props: Properties): Unit = settings.synchronized {
- props.foreach { case (k, v) => setConfString(k, v) }
+ props.asScala.foreach { case (k, v) => setConfString(k, v) }
}
/** Set the given Spark SQL configuration property using a `string` value. */
@@ -601,24 +601,25 @@ private[sql] class SQLConf extends Serializable with CatalystConf {
* Return all the configuration properties that have been set (i.e. not the default).
* This creates a new copy of the config properties in the form of a Map.
*/
- def getAllConfs: immutable.Map[String, String] = settings.synchronized { settings.toMap }
+ def getAllConfs: immutable.Map[String, String] =
+ settings.synchronized { settings.asScala.toMap }
/**
* Return all the configuration definitions that have been defined in [[SQLConf]]. Each
* definition contains key, defaultValue and doc.
*/
def getAllDefinedConfs: Seq[(String, String, String)] = sqlConfEntries.synchronized {
- sqlConfEntries.values.filter(_.isPublic).map { entry =>
+ sqlConfEntries.values.asScala.filter(_.isPublic).map { entry =>
(entry.key, entry.defaultValueString, entry.doc)
}.toSeq
}
private[spark] def unsetConf(key: String): Unit = {
- settings -= key
+ settings.remove(key)
}
private[spark] def unsetConf(entry: SQLConfEntry[_]): Unit = {
- settings -= entry.key
+ settings.remove(entry.key)
}
private[spark] def clear(): Unit = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index a1eea09e0477b4cfd9eb2a77641ab44bc6b1b077..4e8414af50b44dd7d423feeb5e5e5dd14ed7fb34 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -21,7 +21,7 @@ import java.beans.Introspector
import java.util.Properties
import java.util.concurrent.atomic.AtomicReference
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.immutable
import scala.reflect.runtime.universe.TypeTag
import scala.util.control.NonFatal
@@ -225,7 +225,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
conf.setConf(properties)
// After we have populated SQLConf, we call setConf to populate other confs in the subclass
// (e.g. hiveconf in HiveContext).
- properties.foreach {
+ properties.asScala.foreach {
case (key, value) => setConf(key, value)
}
}
@@ -567,7 +567,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
tableName: String,
source: String,
options: java.util.Map[String, String]): DataFrame = {
- createExternalTable(tableName, source, options.toMap)
+ createExternalTable(tableName, source, options.asScala.toMap)
}
/**
@@ -612,7 +612,7 @@ class SQLContext(@transient val sparkContext: SparkContext)
source: String,
schema: StructType,
options: java.util.Map[String, String]): DataFrame = {
- createExternalTable(tableName, source, schema, options.toMap)
+ createExternalTable(tableName, source, schema, options.asScala.toMap)
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ResolvedDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ResolvedDataSource.scala
index 8fbaf3a3059dbdc37447428410f1bd35d911205d..011724436621d3c2bf8917af6d1f55fa2a64602a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ResolvedDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ResolvedDataSource.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources
import java.util.ServiceLoader
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.language.{existentials, implicitConversions}
import scala.util.{Success, Failure, Try}
@@ -55,7 +55,7 @@ object ResolvedDataSource extends Logging {
val loader = Utils.getContextOrSparkClassLoader
val serviceLoader = ServiceLoader.load(classOf[DataSourceRegister], loader)
- serviceLoader.iterator().filter(_.shortName().equalsIgnoreCase(provider)).toList match {
+ serviceLoader.asScala.filter(_.shortName().equalsIgnoreCase(provider)).toList match {
/** the provider format did not match any given registered aliases */
case Nil => Try(loader.loadClass(provider)).orElse(Try(loader.loadClass(provider2))) match {
case Success(dataSource) => dataSource
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystReadSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystReadSupport.scala
index 3f8353af6e2ad518f9f6b2c0be1b0a7041271bb7..0a6bb44445f6e0e003005d5d1bf7e48772f9f2c6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystReadSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystReadSupport.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources.parquet
import java.util.{Map => JMap}
-import scala.collection.JavaConversions.{iterableAsScalaIterable, mapAsJavaMap, mapAsScalaMap}
+import scala.collection.JavaConverters._
import org.apache.hadoop.conf.Configuration
import org.apache.parquet.hadoop.api.ReadSupport.ReadContext
@@ -44,7 +44,7 @@ private[parquet] class CatalystReadSupport extends ReadSupport[InternalRow] with
val parquetRequestedSchema = readContext.getRequestedSchema
val catalystRequestedSchema =
- Option(readContext.getReadSupportMetadata).map(_.toMap).flatMap { metadata =>
+ Option(readContext.getReadSupportMetadata).map(_.asScala).flatMap { metadata =>
metadata
// First tries to read requested schema, which may result from projections
.get(CatalystReadSupport.SPARK_ROW_REQUESTED_SCHEMA)
@@ -123,7 +123,7 @@ private[parquet] class CatalystReadSupport extends ReadSupport[InternalRow] with
maybeRequestedSchema.fold(context.getFileSchema) { schemaString =>
val toParquet = new CatalystSchemaConverter(conf)
val fileSchema = context.getFileSchema.asGroupType()
- val fileFieldNames = fileSchema.getFields.map(_.getName).toSet
+ val fileFieldNames = fileSchema.getFields.asScala.map(_.getName).toSet
StructType
// Deserializes the Catalyst schema of requested columns
@@ -152,7 +152,7 @@ private[parquet] class CatalystReadSupport extends ReadSupport[InternalRow] with
maybeRequestedSchema.map(CatalystReadSupport.SPARK_ROW_REQUESTED_SCHEMA -> _) ++
maybeRowSchema.map(RowWriteSupport.SPARK_ROW_SCHEMA -> _)
- new ReadContext(parquetRequestedSchema, metadata)
+ new ReadContext(parquetRequestedSchema, metadata.asJava)
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystRowConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystRowConverter.scala
index cbf0704c4a9a4bf092566cae431ff616c456749c..f682ca0d8ff4fd0b2c0a8def0a393be11b5249ae 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystRowConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystRowConverter.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.datasources.parquet
import java.math.{BigDecimal, BigInteger}
import java.nio.ByteOrder
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import org.apache.parquet.column.Dictionary
@@ -183,7 +183,7 @@ private[parquet] class CatalystRowConverter(
// those missing fields and create converters for them, although values of these fields are
// always null.
val paddedParquetFields = {
- val parquetFields = parquetType.getFields
+ val parquetFields = parquetType.getFields.asScala
val parquetFieldNames = parquetFields.map(_.getName).toSet
val missingFields = catalystType.filterNot(f => parquetFieldNames.contains(f.name))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystSchemaConverter.scala
index 535f0684e97f98db27373ec16c9de5cc52871117..be6c0545f5a0a80457b24ad35d67f6ccadf7e941 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystSchemaConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystSchemaConverter.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.execution.datasources.parquet
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.hadoop.conf.Configuration
import org.apache.parquet.schema.OriginalType._
@@ -82,7 +82,7 @@ private[parquet] class CatalystSchemaConverter(
def convert(parquetSchema: MessageType): StructType = convert(parquetSchema.asGroupType())
private def convert(parquetSchema: GroupType): StructType = {
- val fields = parquetSchema.getFields.map { field =>
+ val fields = parquetSchema.getFields.asScala.map { field =>
field.getRepetition match {
case OPTIONAL =>
StructField(field.getName, convertField(field), nullable = true)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRelation.scala
index bbf682aec0f9d252cd91c4cb05ac9872d9b3f18e..64982f37cf872399163fa0b6983c56e987442529 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRelation.scala
@@ -21,7 +21,7 @@ import java.net.URI
import java.util.logging.{Logger => JLogger}
import java.util.{List => JList}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.util.{Failure, Try}
@@ -336,7 +336,7 @@ private[sql] class ParquetRelation(
override def getPartitions: Array[SparkPartition] = {
val inputFormat = new ParquetInputFormat[InternalRow] {
override def listStatus(jobContext: JobContext): JList[FileStatus] = {
- if (cacheMetadata) cachedStatuses else super.listStatus(jobContext)
+ if (cacheMetadata) cachedStatuses.asJava else super.listStatus(jobContext)
}
}
@@ -344,7 +344,8 @@ private[sql] class ParquetRelation(
val rawSplits = inputFormat.getSplits(jobContext)
Array.tabulate[SparkPartition](rawSplits.size) { i =>
- new SqlNewHadoopPartition(id, i, rawSplits(i).asInstanceOf[InputSplit with Writable])
+ new SqlNewHadoopPartition(
+ id, i, rawSplits.get(i).asInstanceOf[InputSplit with Writable])
}
}
}.asInstanceOf[RDD[Row]] // type erasure hack to pass RDD[InternalRow] as RDD[Row]
@@ -588,7 +589,7 @@ private[sql] object ParquetRelation extends Logging {
val metadata = footer.getParquetMetadata.getFileMetaData
val serializedSchema = metadata
.getKeyValueMetaData
- .toMap
+ .asScala.toMap
.get(CatalystReadSupport.SPARK_METADATA_KEY)
if (serializedSchema.isEmpty) {
// Falls back to Parquet schema if no Spark SQL schema found.
@@ -745,7 +746,7 @@ private[sql] object ParquetRelation extends Logging {
// Reads footers in multi-threaded manner within each task
val footers =
ParquetFileReader.readAllFootersInParallel(
- serializedConf.value, fakeFileStatuses, skipRowGroups)
+ serializedConf.value, fakeFileStatuses.asJava, skipRowGroups).asScala
// Converter used to convert Parquet `MessageType` to Spark SQL `StructType`
val converter =
@@ -772,7 +773,7 @@ private[sql] object ParquetRelation extends Logging {
val fileMetaData = footer.getParquetMetadata.getFileMetaData
fileMetaData
.getKeyValueMetaData
- .toMap
+ .asScala.toMap
.get(CatalystReadSupport.SPARK_METADATA_KEY)
.flatMap(deserializeSchemaString)
.getOrElse(converter.convert(fileMetaData.getSchema))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypesConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypesConverter.scala
index 42376ef7a9c1f57a920e132e84257b546b6151cb..142301fe87cb63601deb3b02cc0d1e45c497cf4f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypesConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTypesConverter.scala
@@ -18,8 +18,8 @@
package org.apache.spark.sql.execution.datasources.parquet
import java.io.IOException
+import java.util.{Collections, Arrays}
-import scala.collection.JavaConversions._
import scala.util.Try
import org.apache.hadoop.conf.Configuration
@@ -107,7 +107,7 @@ private[parquet] object ParquetTypesConverter extends Logging {
ParquetFileWriter.writeMetadataFile(
conf,
path,
- new Footer(path, new ParquetMetadata(metaData, Nil)) :: Nil)
+ Arrays.asList(new Footer(path, new ParquetMetadata(metaData, Collections.emptyList()))))
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashOuterJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashOuterJoin.scala
index ed282f98b7d716453b237d4c14bf6a5f6af0694a..d800c7456bdac3fb0fee3589e1d8afff288b0031 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashOuterJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/ShuffledHashOuterJoin.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.execution.joins
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.rdd.RDD
@@ -92,9 +92,9 @@ case class ShuffledHashOuterJoin(
case FullOuter =>
// TODO(davies): use UnsafeRow
val leftHashTable =
- buildHashTable(leftIter, numLeftRows, newProjection(leftKeys, left.output))
+ buildHashTable(leftIter, numLeftRows, newProjection(leftKeys, left.output)).asScala
val rightHashTable =
- buildHashTable(rightIter, numRightRows, newProjection(rightKeys, right.output))
+ buildHashTable(rightIter, numRightRows, newProjection(rightKeys, right.output)).asScala
(leftHashTable.keySet ++ rightHashTable.keySet).iterator.flatMap { key =>
fullOuterIterator(key,
leftHashTable.getOrElse(key, EMPTY_LIST),
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUDFs.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUDFs.scala
index 59f8b079ab3333d08efb99728a068b0d3b4c3e82..5a58d846ad80b7a5eb7deeacd8f958d41a3d2419 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUDFs.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/pythonUDFs.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution
import java.io.OutputStream
import java.util.{List => JList, Map => JMap}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import net.razorvine.pickle._
@@ -196,14 +196,15 @@ object EvaluatePython {
case (c, BinaryType) if c.getClass.isArray && c.getClass.getComponentType.getName == "byte" => c
case (c: java.util.List[_], ArrayType(elementType, _)) =>
- new GenericArrayData(c.map { e => fromJava(e, elementType)}.toArray)
+ new GenericArrayData(c.asScala.map { e => fromJava(e, elementType)}.toArray)
case (c, ArrayType(elementType, _)) if c.getClass.isArray =>
new GenericArrayData(c.asInstanceOf[Array[_]].map(e => fromJava(e, elementType)))
case (c: java.util.Map[_, _], MapType(keyType, valueType, _)) =>
- val keys = c.keysIterator.map(fromJava(_, keyType)).toArray
- val values = c.valuesIterator.map(fromJava(_, valueType)).toArray
+ val keyValues = c.asScala.toSeq
+ val keys = keyValues.map(kv => fromJava(kv._1, keyType)).toArray
+ val values = keyValues.map(kv => fromJava(kv._2, valueType)).toArray
ArrayBasedMapData(keys, values)
case (c, StructType(fields)) if c.getClass.isArray =>
@@ -367,7 +368,7 @@ case class BatchPythonEvaluation(udf: PythonUDF, output: Seq[Attribute], child:
val pickle = new Unpickler
iter.flatMap { pickedResult =>
val unpickledBatch = pickle.loads(pickedResult)
- unpickledBatch.asInstanceOf[java.util.ArrayList[Any]]
+ unpickledBatch.asInstanceOf[java.util.ArrayList[Any]].asScala
}
}.mapPartitions { iter =>
val row = new GenericMutableRow(1)
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
index 7abdd3db8034116030d384c70b0c0b2eff6b65c1..4867cebf5328c5bba951040b5ba289174d33e9e3 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDataFrameSuite.java
@@ -23,7 +23,7 @@ import java.util.Comparator;
import java.util.List;
import java.util.Map;
-import scala.collection.JavaConversions;
+import scala.collection.JavaConverters;
import scala.collection.Seq;
import com.google.common.collect.ImmutableMap;
@@ -96,7 +96,7 @@ public class JavaDataFrameSuite {
df.groupBy().agg(countDistinct("key", "value"));
df.groupBy().agg(countDistinct(col("key"), col("value")));
df.select(coalesce(col("key")));
-
+
// Varargs with mathfunctions
DataFrame df2 = context.table("testData2");
df2.select(exp("a"), exp("b"));
@@ -172,7 +172,7 @@ public class JavaDataFrameSuite {
Seq<Integer> outputBuffer = (Seq<Integer>) first.getJavaMap(2).get("hello");
Assert.assertArrayEquals(
bean.getC().get("hello"),
- Ints.toArray(JavaConversions.seqAsJavaList(outputBuffer)));
+ Ints.toArray(JavaConverters.seqAsJavaListConverter(outputBuffer).asJava()));
Seq<String> d = first.getAs(3);
Assert.assertEquals(bean.getD().size(), d.length());
for (int i = 0; i < d.length(); i++) {
@@ -206,7 +206,7 @@ public class JavaDataFrameSuite {
count++;
}
}
-
+
@Test
public void testFrequentItems() {
DataFrame df = context.table("testData2");
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
index cdaa14ac80785626d0fd853d2b8349bde3a6e581..329ffb66083b1a3826bafeed9303ad28cbf2dfb1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.sql.test.SharedSQLContext
@@ -153,11 +153,11 @@ class DataFrameNaFunctionsSuite extends QueryTest with SharedSQLContext {
// Test Java version
checkAnswer(
- df.na.fill(mapAsJavaMap(Map(
+ df.na.fill(Map(
"a" -> "test",
"c" -> 1,
"d" -> 2.2
- ))),
+ ).asJava),
Row("test", null, 1, 2.2))
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
index 4adcefb7dc4b19df4c64d1ef83a956be5ef4a812..3649c2a97b5ef8771e3a07165043ffc31d0784bb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql
import java.util.{Locale, TimeZone}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.sql.catalyst.plans._
import org.apache.spark.sql.catalyst.util._
@@ -145,7 +145,7 @@ object QueryTest {
}
def checkAnswer(df: DataFrame, expectedAnswer: java.util.List[Row]): String = {
- checkAnswer(df, expectedAnswer.toSeq) match {
+ checkAnswer(df, expectedAnswer.asScala) match {
case Some(errorMessage) => errorMessage
case None => null
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala
index 45db619567a22650c192fcac9ed6648f2c6c64d4..bd7cf8c10abefe0cc623964c4601aa68fb66a3d6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetAvroCompatibilitySuite.scala
@@ -20,8 +20,7 @@ package org.apache.spark.sql.execution.datasources.parquet
import java.nio.ByteBuffer
import java.util.{List => JList, Map => JMap}
-import scala.collection.JavaConverters.seqAsJavaListConverter
-import scala.collection.JavaConverters.mapAsJavaMapConverter
+import scala.collection.JavaConverters._
import org.apache.avro.Schema
import org.apache.avro.generic.IndexedRecord
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala
index d85c564e3e8d1f1fb65bf6390034be8fbdfc6f18..df68432faeeb380cd60226bb0f6334a0e906c39b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetCompatibilityTest.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.execution.datasources.parquet
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.hadoop.fs.{Path, PathFilter}
import org.apache.parquet.hadoop.ParquetFileReader
@@ -40,8 +40,9 @@ private[sql] abstract class ParquetCompatibilityTest extends QueryTest with Parq
override def accept(path: Path): Boolean = pathFilter(path)
}).toSeq
- val footers = ParquetFileReader.readAllFootersInParallel(configuration, parquetFiles, true)
- footers.head.getParquetMetadata.getFileMetaData.getSchema
+ val footers =
+ ParquetFileReader.readAllFootersInParallel(configuration, parquetFiles.asJava, true)
+ footers.iterator().next().getParquetMetadata.getFileMetaData.getSchema
}
protected def logParquetSchema(path: String): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index e6b0a2ea95e38480b16a52bfa727ecb9f7225ac1..08d2b9dee99b07735afd92dce23d8bc2247e6c43 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -17,7 +17,9 @@
package org.apache.spark.sql.execution.datasources.parquet
-import scala.collection.JavaConversions._
+import java.util.Collections
+
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import scala.reflect.runtime.universe.TypeTag
@@ -28,7 +30,7 @@ import org.apache.parquet.example.data.simple.SimpleGroup
import org.apache.parquet.example.data.{Group, GroupWriter}
import org.apache.parquet.hadoop.api.WriteSupport
import org.apache.parquet.hadoop.api.WriteSupport.WriteContext
-import org.apache.parquet.hadoop.metadata.{CompressionCodecName, FileMetaData, ParquetMetadata}
+import org.apache.parquet.hadoop.metadata.{BlockMetaData, CompressionCodecName, FileMetaData, ParquetMetadata}
import org.apache.parquet.hadoop.{Footer, ParquetFileWriter, ParquetOutputCommitter, ParquetWriter}
import org.apache.parquet.io.api.RecordConsumer
import org.apache.parquet.schema.{MessageType, MessageTypeParser}
@@ -205,9 +207,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext {
test("compression codec") {
def compressionCodecFor(path: String): String = {
val codecs = ParquetTypesConverter
- .readMetaData(new Path(path), Some(configuration))
- .getBlocks
- .flatMap(_.getColumns)
+ .readMetaData(new Path(path), Some(configuration)).getBlocks.asScala
+ .flatMap(_.getColumns.asScala)
.map(_.getCodec.name())
.distinct
@@ -348,14 +349,16 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext {
""".stripMargin)
withTempPath { location =>
- val extraMetadata = Map(CatalystReadSupport.SPARK_METADATA_KEY -> sparkSchema.toString)
+ val extraMetadata = Collections.singletonMap(
+ CatalystReadSupport.SPARK_METADATA_KEY, sparkSchema.toString)
val fileMetadata = new FileMetaData(parquetSchema, extraMetadata, "Spark")
val path = new Path(location.getCanonicalPath)
ParquetFileWriter.writeMetadataFile(
sqlContext.sparkContext.hadoopConfiguration,
path,
- new Footer(path, new ParquetMetadata(fileMetadata, Nil)) :: Nil)
+ Collections.singletonList(
+ new Footer(path, new ParquetMetadata(fileMetadata, Collections.emptyList()))))
assertResult(sqlContext.read.parquet(path.toString).schema) {
StructType(
@@ -386,7 +389,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext {
} finally {
// Hadoop 1 doesn't have `Configuration.unset`
configuration.clear()
- clonedConf.foreach(entry => configuration.set(entry.getKey, entry.getValue))
+ clonedConf.asScala.foreach(entry => configuration.set(entry.getKey, entry.getValue))
}
}
@@ -410,7 +413,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext {
} finally {
// Hadoop 1 doesn't have `Configuration.unset`
configuration.clear()
- clonedConf.foreach(entry => configuration.set(entry.getKey, entry.getValue))
+ clonedConf.asScala.foreach(entry => configuration.set(entry.getKey, entry.getValue))
}
}
@@ -434,7 +437,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext {
} finally {
// Hadoop 1 doesn't have `Configuration.unset`
configuration.clear()
- clonedConf.foreach(entry => configuration.set(entry.getKey, entry.getValue))
+ clonedConf.asScala.foreach(entry => configuration.set(entry.getKey, entry.getValue))
}
}
}
@@ -481,7 +484,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext {
} finally {
// Hadoop 1 doesn't have `Configuration.unset`
configuration.clear()
- clonedConf.foreach(entry => configuration.set(entry.getKey, entry.getValue))
+ clonedConf.asScala.foreach(entry => configuration.set(entry.getKey, entry.getValue))
}
}
}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index 02cc7e5efa5215725e2c0bcf9c8418dd7cdbbd50..306f98bcb5344991e5221242f7907105b7d3bed6 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -20,9 +20,9 @@ package org.apache.spark.sql.hive.thriftserver
import java.security.PrivilegedExceptionAction
import java.sql.{Date, Timestamp}
import java.util.concurrent.RejectedExecutionException
-import java.util.{Map => JMap, UUID}
+import java.util.{Arrays, Map => JMap, UUID}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, Map => SMap}
import scala.util.control.NonFatal
@@ -126,13 +126,13 @@ private[hive] class SparkExecuteStatementOperation(
def getResultSetSchema: TableSchema = {
if (result == null || result.queryExecution.analyzed.output.size == 0) {
- new TableSchema(new FieldSchema("Result", "string", "") :: Nil)
+ new TableSchema(Arrays.asList(new FieldSchema("Result", "string", "")))
} else {
logInfo(s"Result Schema: ${result.queryExecution.analyzed.output}")
val schema = result.queryExecution.analyzed.output.map { attr =>
new FieldSchema(attr.name, HiveMetastoreTypes.toMetastoreType(attr.dataType), "")
}
- new TableSchema(schema)
+ new TableSchema(schema.asJava)
}
}
@@ -298,7 +298,7 @@ private[hive] class SparkExecuteStatementOperation(
sqlOperationConf = new HiveConf(sqlOperationConf)
// apply overlay query specific settings, if any
- getConfOverlay().foreach { case (k, v) =>
+ getConfOverlay().asScala.foreach { case (k, v) =>
try {
sqlOperationConf.verifyAndSet(k, v)
} catch {
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index 7799704c819d9eaaf95a40ee456e1fccd6481f16..a29df567983b14cc9c2885c0eea4d89a635a037d 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -17,11 +17,11 @@
package org.apache.spark.sql.hive.thriftserver
-import scala.collection.JavaConversions._
-
import java.io._
import java.util.{ArrayList => JArrayList, Locale}
+import scala.collection.JavaConverters._
+
import jline.console.ConsoleReader
import jline.console.history.FileHistory
@@ -101,9 +101,9 @@ private[hive] object SparkSQLCLIDriver extends Logging {
// Set all properties specified via command line.
val conf: HiveConf = sessionState.getConf
- sessionState.cmdProperties.entrySet().foreach { item =>
- val key = item.getKey.asInstanceOf[String]
- val value = item.getValue.asInstanceOf[String]
+ sessionState.cmdProperties.entrySet().asScala.foreach { item =>
+ val key = item.getKey.toString
+ val value = item.getValue.toString
// We do not propagate metastore options to the execution copy of hive.
if (key != "javax.jdo.option.ConnectionURL") {
conf.set(key, value)
@@ -316,15 +316,15 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CLI_PRINT_HEADER)) {
// Print the column names.
- Option(driver.getSchema.getFieldSchemas).map { fields =>
- out.println(fields.map(_.getName).mkString("\t"))
+ Option(driver.getSchema.getFieldSchemas).foreach { fields =>
+ out.println(fields.asScala.map(_.getName).mkString("\t"))
}
}
var counter = 0
try {
while (!out.checkError() && driver.getResults(res)) {
- res.foreach{ l =>
+ res.asScala.foreach { l =>
counter += 1
out.println(l)
}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala
index 644165acf70a727fab61375a5168597dbff87e07..5ad8c54f296d5f9c236a6a017490d8a236baa5de 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIService.scala
@@ -21,6 +21,8 @@ import java.io.IOException
import java.util.{List => JList}
import javax.security.auth.login.LoginException
+import scala.collection.JavaConverters._
+
import org.apache.commons.logging.Log
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.shims.Utils
@@ -34,8 +36,6 @@ import org.apache.hive.service.{AbstractService, Service, ServiceException}
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.sql.hive.thriftserver.ReflectionUtils._
-import scala.collection.JavaConversions._
-
private[hive] class SparkSQLCLIService(hiveServer: HiveServer2, hiveContext: HiveContext)
extends CLIService(hiveServer)
with ReflectedCompositeService {
@@ -76,7 +76,7 @@ private[thriftserver] trait ReflectedCompositeService { this: AbstractService =>
def initCompositeService(hiveConf: HiveConf) {
// Emulating `CompositeService.init(hiveConf)`
val serviceList = getAncestorField[JList[Service]](this, 2, "serviceList")
- serviceList.foreach(_.init(hiveConf))
+ serviceList.asScala.foreach(_.init(hiveConf))
// Emulating `AbstractService.init(hiveConf)`
invoke(classOf[AbstractService], this, "ensureCurrentState", classOf[STATE] -> STATE.NOTINITED)
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
index 77272aecf28359037bd031943e2171c316aab53b..2619286afc148431111a2a389e6717e48d57a621 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
@@ -17,7 +17,9 @@
package org.apache.spark.sql.hive.thriftserver
-import java.util.{ArrayList => JArrayList, List => JList}
+import java.util.{Arrays, ArrayList => JArrayList, List => JList}
+
+import scala.collection.JavaConverters._
import org.apache.commons.lang3.exception.ExceptionUtils
import org.apache.hadoop.hive.metastore.api.{FieldSchema, Schema}
@@ -27,8 +29,6 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse
import org.apache.spark.Logging
import org.apache.spark.sql.hive.{HiveContext, HiveMetastoreTypes}
-import scala.collection.JavaConversions._
-
private[hive] class SparkSQLDriver(
val context: HiveContext = SparkSQLEnv.hiveContext)
extends Driver
@@ -43,14 +43,14 @@ private[hive] class SparkSQLDriver(
private def getResultSetSchema(query: context.QueryExecution): Schema = {
val analyzed = query.analyzed
logDebug(s"Result Schema: ${analyzed.output}")
- if (analyzed.output.size == 0) {
- new Schema(new FieldSchema("Response code", "string", "") :: Nil, null)
+ if (analyzed.output.isEmpty) {
+ new Schema(Arrays.asList(new FieldSchema("Response code", "string", "")), null)
} else {
val fieldSchemas = analyzed.output.map { attr =>
new FieldSchema(attr.name, HiveMetastoreTypes.toMetastoreType(attr.dataType), "")
}
- new Schema(fieldSchemas, null)
+ new Schema(fieldSchemas.asJava, null)
}
}
@@ -79,7 +79,7 @@ private[hive] class SparkSQLDriver(
if (hiveResponse == null) {
false
} else {
- res.asInstanceOf[JArrayList[String]].addAll(hiveResponse)
+ res.asInstanceOf[JArrayList[String]].addAll(hiveResponse.asJava)
hiveResponse = null
true
}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
index 1d41c461318282dc1fd8a78f98b9151b6ba3f1f0..bacf6cc458fd555f0e61b687bb1f9c8245fc1e7d 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.hive.thriftserver
import java.io.PrintStream
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.scheduler.StatsReportListener
import org.apache.spark.sql.hive.HiveContext
@@ -64,7 +64,7 @@ private[hive] object SparkSQLEnv extends Logging {
hiveContext.setConf("spark.sql.hive.version", HiveContext.hiveExecutionVersion)
if (log.isDebugEnabled) {
- hiveContext.hiveconf.getAllProperties.toSeq.sorted.foreach { case (k, v) =>
+ hiveContext.hiveconf.getAllProperties.asScala.toSeq.sorted.foreach { case (k, v) =>
logDebug(s"HiveConf var: $k=$v")
}
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
index 17cc83087fb1d3ca9947c35d7aa9321f9255ee07..c0a458fa9ab8d45c13476dfc90f3fef5cc7c68d2 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -22,7 +22,7 @@ import java.net.{URL, URLClassLoader}
import java.sql.Timestamp
import java.util.concurrent.TimeUnit
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.HashMap
import scala.language.implicitConversions
import scala.concurrent.duration._
@@ -194,7 +194,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) with Logging {
logInfo("defalt warehouse location is " + defaltWarehouseLocation)
// `configure` goes second to override other settings.
- val allConfig = metadataConf.iterator.map(e => e.getKey -> e.getValue).toMap ++ configure
+ val allConfig = metadataConf.asScala.map(e => e.getKey -> e.getValue).toMap ++ configure
val isolatedLoader = if (hiveMetastoreJars == "builtin") {
if (hiveExecutionVersion != hiveMetastoreVersion) {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 64fffdbf9b0207c529c63c94f3e6bb902ccae345..cfe2bb05ad89e1b677cca6cde48271d73986c7a1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.hive
+import scala.collection.JavaConverters._
+
import org.apache.hadoop.hive.common.`type`.{HiveDecimal, HiveVarchar}
import org.apache.hadoop.hive.serde2.objectinspector.primitive._
import org.apache.hadoop.hive.serde2.objectinspector.{StructField => HiveStructField, _}
@@ -31,9 +33,6 @@ import org.apache.spark.sql.types._
import org.apache.spark.sql.{AnalysisException, types}
import org.apache.spark.unsafe.types.UTF8String
-/* Implicit conversions */
-import scala.collection.JavaConversions._
-
/**
* 1. The Underlying data type in catalyst and in Hive
* In catalyst:
@@ -290,13 +289,13 @@ private[hive] trait HiveInspectors {
DateTimeUtils.fromJavaDate(poi.getWritableConstantValue.get())
case mi: StandardConstantMapObjectInspector =>
// take the value from the map inspector object, rather than the input data
- val map = mi.getWritableConstantValue
- val keys = map.keysIterator.map(unwrap(_, mi.getMapKeyObjectInspector)).toArray
- val values = map.valuesIterator.map(unwrap(_, mi.getMapValueObjectInspector)).toArray
+ val keyValues = mi.getWritableConstantValue.asScala.toSeq
+ val keys = keyValues.map(kv => unwrap(kv._1, mi.getMapKeyObjectInspector)).toArray
+ val values = keyValues.map(kv => unwrap(kv._2, mi.getMapValueObjectInspector)).toArray
ArrayBasedMapData(keys, values)
case li: StandardConstantListObjectInspector =>
// take the value from the list inspector object, rather than the input data
- val values = li.getWritableConstantValue
+ val values = li.getWritableConstantValue.asScala
.map(unwrap(_, li.getListElementObjectInspector))
.toArray
new GenericArrayData(values)
@@ -342,7 +341,7 @@ private[hive] trait HiveInspectors {
case li: ListObjectInspector =>
Option(li.getList(data))
.map { l =>
- val values = l.map(unwrap(_, li.getListElementObjectInspector)).toArray
+ val values = l.asScala.map(unwrap(_, li.getListElementObjectInspector)).toArray
new GenericArrayData(values)
}
.orNull
@@ -351,15 +350,16 @@ private[hive] trait HiveInspectors {
if (map == null) {
null
} else {
- val keys = map.keysIterator.map(unwrap(_, mi.getMapKeyObjectInspector)).toArray
- val values = map.valuesIterator.map(unwrap(_, mi.getMapValueObjectInspector)).toArray
+ val keyValues = map.asScala.toSeq
+ val keys = keyValues.map(kv => unwrap(kv._1, mi.getMapKeyObjectInspector)).toArray
+ val values = keyValues.map(kv => unwrap(kv._2, mi.getMapValueObjectInspector)).toArray
ArrayBasedMapData(keys, values)
}
// currently, hive doesn't provide the ConstantStructObjectInspector
case si: StructObjectInspector =>
val allRefs = si.getAllStructFieldRefs
- InternalRow.fromSeq(
- allRefs.map(r => unwrap(si.getStructFieldData(data, r), r.getFieldObjectInspector)))
+ InternalRow.fromSeq(allRefs.asScala.map(
+ r => unwrap(si.getStructFieldData(data, r), r.getFieldObjectInspector)))
}
@@ -403,14 +403,14 @@ private[hive] trait HiveInspectors {
case soi: StandardStructObjectInspector =>
val schema = dataType.asInstanceOf[StructType]
- val wrappers = soi.getAllStructFieldRefs.zip(schema.fields).map { case (ref, field) =>
- wrapperFor(ref.getFieldObjectInspector, field.dataType)
+ val wrappers = soi.getAllStructFieldRefs.asScala.zip(schema.fields).map {
+ case (ref, field) => wrapperFor(ref.getFieldObjectInspector, field.dataType)
}
(o: Any) => {
if (o != null) {
val struct = soi.create()
val row = o.asInstanceOf[InternalRow]
- soi.getAllStructFieldRefs.zip(wrappers).zipWithIndex.foreach {
+ soi.getAllStructFieldRefs.asScala.zip(wrappers).zipWithIndex.foreach {
case ((field, wrapper), i) =>
soi.setStructFieldData(struct, field, wrapper(row.get(i, schema(i).dataType)))
}
@@ -537,7 +537,7 @@ private[hive] trait HiveInspectors {
// 1. create the pojo (most likely) object
val result = x.create()
var i = 0
- while (i < fieldRefs.length) {
+ while (i < fieldRefs.size) {
// 2. set the property for the pojo
val tpe = structType(i).dataType
x.setStructFieldData(
@@ -552,9 +552,9 @@ private[hive] trait HiveInspectors {
val fieldRefs = x.getAllStructFieldRefs
val structType = dataType.asInstanceOf[StructType]
val row = a.asInstanceOf[InternalRow]
- val result = new java.util.ArrayList[AnyRef](fieldRefs.length)
+ val result = new java.util.ArrayList[AnyRef](fieldRefs.size)
var i = 0
- while (i < fieldRefs.length) {
+ while (i < fieldRefs.size) {
val tpe = structType(i).dataType
result.add(wrap(row.get(i, tpe), fieldRefs.get(i).getFieldObjectInspector, tpe))
i += 1
@@ -712,10 +712,10 @@ private[hive] trait HiveInspectors {
def inspectorToDataType(inspector: ObjectInspector): DataType = inspector match {
case s: StructObjectInspector =>
- StructType(s.getAllStructFieldRefs.map(f => {
+ StructType(s.getAllStructFieldRefs.asScala.map(f =>
types.StructField(
f.getFieldName, inspectorToDataType(f.getFieldObjectInspector), nullable = true)
- }))
+ ))
case l: ListObjectInspector => ArrayType(inspectorToDataType(l.getListElementObjectInspector))
case m: MapObjectInspector =>
MapType(
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
index 98d21aa76d64e75a229ff25ec3cf79e7e61e1245..b8da0840ae56909a00b72aeee7a52dd1e60e23cc 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.hive
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable
import com.google.common.base.Objects
@@ -483,7 +483,7 @@ private[hive] class HiveMetastoreCatalog(val client: ClientInterface, hive: Hive
// are empty.
val partitions = metastoreRelation.getHiveQlPartitions().map { p =>
val location = p.getLocation
- val values = InternalRow.fromSeq(p.getValues.zip(partitionColumnDataTypes).map {
+ val values = InternalRow.fromSeq(p.getValues.asScala.zip(partitionColumnDataTypes).map {
case (rawValue, dataType) => Cast(Literal(rawValue), dataType).eval(null)
})
ParquetPartition(values, location)
@@ -798,9 +798,9 @@ private[hive] case class MetastoreRelation
val sd = new org.apache.hadoop.hive.metastore.api.StorageDescriptor()
tTable.setSd(sd)
- sd.setCols(table.schema.map(c => new FieldSchema(c.name, c.hiveType, c.comment)))
+ sd.setCols(table.schema.map(c => new FieldSchema(c.name, c.hiveType, c.comment)).asJava)
tTable.setPartitionKeys(
- table.partitionColumns.map(c => new FieldSchema(c.name, c.hiveType, c.comment)))
+ table.partitionColumns.map(c => new FieldSchema(c.name, c.hiveType, c.comment)).asJava)
table.location.foreach(sd.setLocation)
table.inputFormat.foreach(sd.setInputFormat)
@@ -852,11 +852,11 @@ private[hive] case class MetastoreRelation
val tPartition = new org.apache.hadoop.hive.metastore.api.Partition
tPartition.setDbName(databaseName)
tPartition.setTableName(tableName)
- tPartition.setValues(p.values)
+ tPartition.setValues(p.values.asJava)
val sd = new org.apache.hadoop.hive.metastore.api.StorageDescriptor()
tPartition.setSd(sd)
- sd.setCols(table.schema.map(c => new FieldSchema(c.name, c.hiveType, c.comment)))
+ sd.setCols(table.schema.map(c => new FieldSchema(c.name, c.hiveType, c.comment)).asJava)
sd.setLocation(p.storage.location)
sd.setInputFormat(p.storage.inputFormat)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
index ad33dee555dd2f563f89e34a623a4dee3eb36bf1..d5cd7e98b5267045423ae07dbad4f52a0853b192 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
@@ -20,6 +20,9 @@ package org.apache.spark.sql.hive
import java.sql.Date
import java.util.Locale
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.conf.HiveConf.ConfVars
import org.apache.hadoop.hive.serde.serdeConstants
@@ -48,10 +51,6 @@ import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.CalendarInterval
import org.apache.spark.util.random.RandomSampler
-/* Implicit conversions */
-import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-
/**
* Used when we need to start parsing the AST before deciding that we are going to pass the command
* back for Hive to execute natively. Will be replaced with a native command that contains the
@@ -202,7 +201,7 @@ private[hive] object HiveQl extends Logging {
* Returns a scala.Seq equivalent to [s] or Nil if [s] is null.
*/
private def nilIfEmpty[A](s: java.util.List[A]): Seq[A] =
- Option(s).map(_.toSeq).getOrElse(Nil)
+ Option(s).map(_.asScala).getOrElse(Nil)
/**
* Returns this ASTNode with the text changed to `newText`.
@@ -217,7 +216,7 @@ private[hive] object HiveQl extends Logging {
*/
def withChildren(newChildren: Seq[ASTNode]): ASTNode = {
(1 to n.getChildCount).foreach(_ => n.deleteChild(0))
- n.addChildren(newChildren)
+ n.addChildren(newChildren.asJava)
n
}
@@ -323,11 +322,11 @@ private[hive] object HiveQl extends Logging {
assert(tree.asInstanceOf[ASTNode].getText == "TOK_CREATETABLE", "Only CREATE TABLE supported.")
val tableOps = tree.getChildren
val colList =
- tableOps
+ tableOps.asScala
.find(_.asInstanceOf[ASTNode].getText == "TOK_TABCOLLIST")
.getOrElse(sys.error("No columnList!")).getChildren
- colList.map(nodeToAttribute)
+ colList.asScala.map(nodeToAttribute)
}
/** Extractor for matching Hive's AST Tokens. */
@@ -337,7 +336,7 @@ private[hive] object HiveQl extends Logging {
case t: ASTNode =>
CurrentOrigin.setPosition(t.getLine, t.getCharPositionInLine)
Some((t.getText,
- Option(t.getChildren).map(_.toList).getOrElse(Nil).asInstanceOf[Seq[ASTNode]]))
+ Option(t.getChildren).map(_.asScala.toList).getOrElse(Nil).asInstanceOf[Seq[ASTNode]]))
case _ => None
}
}
@@ -424,7 +423,9 @@ private[hive] object HiveQl extends Logging {
protected def extractDbNameTableName(tableNameParts: Node): (Option[String], String) = {
val (db, tableName) =
- tableNameParts.getChildren.map { case Token(part, Nil) => cleanIdentifier(part) } match {
+ tableNameParts.getChildren.asScala.map {
+ case Token(part, Nil) => cleanIdentifier(part)
+ } match {
case Seq(tableOnly) => (None, tableOnly)
case Seq(databaseName, table) => (Some(databaseName), table)
}
@@ -433,7 +434,9 @@ private[hive] object HiveQl extends Logging {
}
protected def extractTableIdent(tableNameParts: Node): Seq[String] = {
- tableNameParts.getChildren.map { case Token(part, Nil) => cleanIdentifier(part) } match {
+ tableNameParts.getChildren.asScala.map {
+ case Token(part, Nil) => cleanIdentifier(part)
+ } match {
case Seq(tableOnly) => Seq(tableOnly)
case Seq(databaseName, table) => Seq(databaseName, table)
case other => sys.error("Hive only supports tables names like 'tableName' " +
@@ -624,7 +627,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
val cols = BaseSemanticAnalyzer.getColumns(list, true)
if (cols != null) {
tableDesc = tableDesc.copy(
- schema = cols.map { field =>
+ schema = cols.asScala.map { field =>
HiveColumn(field.getName, field.getType, field.getComment)
})
}
@@ -636,7 +639,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
val cols = BaseSemanticAnalyzer.getColumns(list(0), false)
if (cols != null) {
tableDesc = tableDesc.copy(
- partitionColumns = cols.map { field =>
+ partitionColumns = cols.asScala.map { field =>
HiveColumn(field.getName, field.getType, field.getComment)
})
}
@@ -672,7 +675,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
case _ => assert(false)
}
tableDesc = tableDesc.copy(
- serdeProperties = tableDesc.serdeProperties ++ serdeParams)
+ serdeProperties = tableDesc.serdeProperties ++ serdeParams.asScala)
case Token("TOK_TABLELOCATION", child :: Nil) =>
var location = BaseSemanticAnalyzer.unescapeSQLString(child.getText)
location = EximUtil.relativeToAbsolutePath(hiveConf, location)
@@ -684,7 +687,8 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
val serdeParams = new java.util.HashMap[String, String]()
BaseSemanticAnalyzer.readProps(
(child.getChild(1).getChild(0)).asInstanceOf[ASTNode], serdeParams)
- tableDesc = tableDesc.copy(serdeProperties = tableDesc.serdeProperties ++ serdeParams)
+ tableDesc = tableDesc.copy(
+ serdeProperties = tableDesc.serdeProperties ++ serdeParams.asScala)
}
case Token("TOK_FILEFORMAT_GENERIC", child :: Nil) =>
child.getText().toLowerCase(Locale.ENGLISH) match {
@@ -847,7 +851,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
}
val withWhere = whereClause.map { whereNode =>
- val Seq(whereExpr) = whereNode.getChildren.toSeq
+ val Seq(whereExpr) = whereNode.getChildren.asScala
Filter(nodeToExpr(whereExpr), relations)
}.getOrElse(relations)
@@ -856,7 +860,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
// Script transformations are expressed as a select clause with a single expression of type
// TOK_TRANSFORM
- val transformation = select.getChildren.head match {
+ val transformation = select.getChildren.iterator().next() match {
case Token("TOK_SELEXPR",
Token("TOK_TRANSFORM",
Token("TOK_EXPLIST", inputExprs) ::
@@ -925,10 +929,10 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
val withLateralView = lateralViewClause.map { lv =>
val Token("TOK_SELECT",
- Token("TOK_SELEXPR", clauses) :: Nil) = lv.getChildren.head
+ Token("TOK_SELEXPR", clauses) :: Nil) = lv.getChildren.iterator().next()
- val alias =
- getClause("TOK_TABALIAS", clauses).getChildren.head.asInstanceOf[ASTNode].getText
+ val alias = getClause("TOK_TABALIAS", clauses).getChildren.iterator().next()
+ .asInstanceOf[ASTNode].getText
val (generator, attributes) = nodesToGenerator(clauses)
Generate(
@@ -944,7 +948,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
// (if there is a group by) or a script transformation.
val withProject: LogicalPlan = transformation.getOrElse {
val selectExpressions =
- select.getChildren.flatMap(selExprNodeToExpr).map(UnresolvedAlias(_)).toSeq
+ select.getChildren.asScala.flatMap(selExprNodeToExpr).map(UnresolvedAlias)
Seq(
groupByClause.map(e => e match {
case Token("TOK_GROUPBY", children) =>
@@ -973,7 +977,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
// Handle HAVING clause.
val withHaving = havingClause.map { h =>
- val havingExpr = h.getChildren.toSeq match { case Seq(hexpr) => nodeToExpr(hexpr) }
+ val havingExpr = h.getChildren.asScala match { case Seq(hexpr) => nodeToExpr(hexpr) }
// Note that we added a cast to boolean. If the expression itself is already boolean,
// the optimizer will get rid of the unnecessary cast.
Filter(Cast(havingExpr, BooleanType), withProject)
@@ -983,32 +987,42 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
val withDistinct =
if (selectDistinctClause.isDefined) Distinct(withHaving) else withHaving
- // Handle ORDER BY, SORT BY, DISTRIBETU BY, and CLUSTER BY clause.
+ // Handle ORDER BY, SORT BY, DISTRIBUTE BY, and CLUSTER BY clause.
val withSort =
(orderByClause, sortByClause, distributeByClause, clusterByClause) match {
case (Some(totalOrdering), None, None, None) =>
- Sort(totalOrdering.getChildren.map(nodeToSortOrder), true, withDistinct)
+ Sort(totalOrdering.getChildren.asScala.map(nodeToSortOrder), true, withDistinct)
case (None, Some(perPartitionOrdering), None, None) =>
- Sort(perPartitionOrdering.getChildren.map(nodeToSortOrder), false, withDistinct)
+ Sort(
+ perPartitionOrdering.getChildren.asScala.map(nodeToSortOrder),
+ false, withDistinct)
case (None, None, Some(partitionExprs), None) =>
- RepartitionByExpression(partitionExprs.getChildren.map(nodeToExpr), withDistinct)
+ RepartitionByExpression(
+ partitionExprs.getChildren.asScala.map(nodeToExpr), withDistinct)
case (None, Some(perPartitionOrdering), Some(partitionExprs), None) =>
- Sort(perPartitionOrdering.getChildren.map(nodeToSortOrder), false,
- RepartitionByExpression(partitionExprs.getChildren.map(nodeToExpr), withDistinct))
+ Sort(
+ perPartitionOrdering.getChildren.asScala.map(nodeToSortOrder), false,
+ RepartitionByExpression(
+ partitionExprs.getChildren.asScala.map(nodeToExpr),
+ withDistinct))
case (None, None, None, Some(clusterExprs)) =>
- Sort(clusterExprs.getChildren.map(nodeToExpr).map(SortOrder(_, Ascending)), false,
- RepartitionByExpression(clusterExprs.getChildren.map(nodeToExpr), withDistinct))
+ Sort(
+ clusterExprs.getChildren.asScala.map(nodeToExpr).map(SortOrder(_, Ascending)),
+ false,
+ RepartitionByExpression(
+ clusterExprs.getChildren.asScala.map(nodeToExpr),
+ withDistinct))
case (None, None, None, None) => withDistinct
case _ => sys.error("Unsupported set of ordering / distribution clauses.")
}
val withLimit =
- limitClause.map(l => nodeToExpr(l.getChildren.head))
+ limitClause.map(l => nodeToExpr(l.getChildren.iterator().next()))
.map(Limit(_, withSort))
.getOrElse(withSort)
// Collect all window specifications defined in the WINDOW clause.
- val windowDefinitions = windowClause.map(_.getChildren.toSeq.collect {
+ val windowDefinitions = windowClause.map(_.getChildren.asScala.collect {
case Token("TOK_WINDOWDEF",
Token(windowName, Nil) :: Token("TOK_WINDOWSPEC", spec) :: Nil) =>
windowName -> nodesToWindowSpecification(spec)
@@ -1063,7 +1077,8 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
val Token("TOK_SELECT",
Token("TOK_SELEXPR", clauses) :: Nil) = selectClause
- val alias = getClause("TOK_TABALIAS", clauses).getChildren.head.asInstanceOf[ASTNode].getText
+ val alias = getClause("TOK_TABALIAS", clauses).getChildren.iterator().next()
+ .asInstanceOf[ASTNode].getText
val (generator, attributes) = nodesToGenerator(clauses)
Generate(
@@ -1092,7 +1107,9 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
}
val tableIdent =
- tableNameParts.getChildren.map{ case Token(part, Nil) => cleanIdentifier(part)} match {
+ tableNameParts.getChildren.asScala.map {
+ case Token(part, Nil) => cleanIdentifier(part)
+ } match {
case Seq(tableOnly) => Seq(tableOnly)
case Seq(databaseName, table) => Seq(databaseName, table)
case other => sys.error("Hive only supports tables names like 'tableName' " +
@@ -1139,7 +1156,8 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
val isPreserved = tableOrdinals.map(i => (i - 1 < 0) || joinArgs(i - 1).getText == "PRESERVE")
val tables = tableOrdinals.map(i => nodeToRelation(joinArgs(i)))
- val joinExpressions = tableOrdinals.map(i => joinArgs(i + 1).getChildren.map(nodeToExpr))
+ val joinExpressions =
+ tableOrdinals.map(i => joinArgs(i + 1).getChildren.asScala.map(nodeToExpr))
val joinConditions = joinExpressions.sliding(2).map {
case Seq(c1, c2) =>
@@ -1164,7 +1182,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
joinType = joinType.remove(joinType.length - 1))
}
- val groups = (0 until joinExpressions.head.size).map(i => Coalesce(joinExpressions.map(_(i))))
+ val groups = joinExpressions.head.indices.map(i => Coalesce(joinExpressions.map(_(i))))
// Unique join is not really the same as an outer join so we must group together results where
// the joinExpressions are the same, taking the First of each value is only okay because the
@@ -1229,7 +1247,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
val tableIdent = extractTableIdent(tableNameParts)
- val partitionKeys = partitionClause.map(_.getChildren.map {
+ val partitionKeys = partitionClause.map(_.getChildren.asScala.map {
// Parse partitions. We also make keys case insensitive.
case Token("TOK_PARTVAL", Token(key, Nil) :: Token(value, Nil) :: Nil) =>
cleanIdentifier(key.toLowerCase) -> Some(PlanUtils.stripQuotes(value))
@@ -1249,7 +1267,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
val tableIdent = extractTableIdent(tableNameParts)
- val partitionKeys = partitionClause.map(_.getChildren.map {
+ val partitionKeys = partitionClause.map(_.getChildren.asScala.map {
// Parse partitions. We also make keys case insensitive.
case Token("TOK_PARTVAL", Token(key, Nil) :: Token(value, Nil) :: Nil) =>
cleanIdentifier(key.toLowerCase) -> Some(PlanUtils.stripQuotes(value))
@@ -1590,18 +1608,18 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
val (partitionByClause :: orderByClause :: sortByClause :: clusterByClause :: Nil) =
getClauses(
Seq("TOK_DISTRIBUTEBY", "TOK_ORDERBY", "TOK_SORTBY", "TOK_CLUSTERBY"),
- partitionAndOrdering.getChildren.toSeq.asInstanceOf[Seq[ASTNode]])
+ partitionAndOrdering.getChildren.asScala.asInstanceOf[Seq[ASTNode]])
(partitionByClause, orderByClause.orElse(sortByClause), clusterByClause) match {
case (Some(partitionByExpr), Some(orderByExpr), None) =>
- (partitionByExpr.getChildren.map(nodeToExpr),
- orderByExpr.getChildren.map(nodeToSortOrder))
+ (partitionByExpr.getChildren.asScala.map(nodeToExpr),
+ orderByExpr.getChildren.asScala.map(nodeToSortOrder))
case (Some(partitionByExpr), None, None) =>
- (partitionByExpr.getChildren.map(nodeToExpr), Nil)
+ (partitionByExpr.getChildren.asScala.map(nodeToExpr), Nil)
case (None, Some(orderByExpr), None) =>
- (Nil, orderByExpr.getChildren.map(nodeToSortOrder))
+ (Nil, orderByExpr.getChildren.asScala.map(nodeToSortOrder))
case (None, None, Some(clusterByExpr)) =>
- val expressions = clusterByExpr.getChildren.map(nodeToExpr)
+ val expressions = clusterByExpr.getChildren.asScala.map(nodeToExpr)
(expressions, expressions.map(SortOrder(_, Ascending)))
case _ =>
throw new NotImplementedError(
@@ -1639,7 +1657,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
}
rowFrame.orElse(rangeFrame).map { frame =>
- frame.getChildren.toList match {
+ frame.getChildren.asScala.toList match {
case precedingNode :: followingNode :: Nil =>
SpecifiedWindowFrame(
frameType,
@@ -1701,7 +1719,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
case other => sys.error(s"Non ASTNode encountered: $other")
}
- Option(node.getChildren).map(_.toList).getOrElse(Nil).foreach(dumpTree(_, builder, indent + 1))
+ Option(node.getChildren).map(_.asScala).getOrElse(Nil).foreach(dumpTree(_, builder, indent + 1))
builder
}
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
index 267074f3ad1022a822cd8246e0d42066732596a8..004805f3aed0b74beaf7160589af1384ce35fa0f 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
@@ -22,8 +22,7 @@ import java.rmi.server.UID
import org.apache.avro.Schema
-/* Implicit conversions */
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.language.implicitConversions
import scala.reflect.ClassTag
@@ -73,7 +72,7 @@ private[hive] object HiveShim {
*/
def appendReadColumns(conf: Configuration, ids: Seq[Integer], names: Seq[String]) {
if (ids != null && ids.nonEmpty) {
- ColumnProjectionUtils.appendReadColumns(conf, ids)
+ ColumnProjectionUtils.appendReadColumns(conf, ids.asJava)
}
if (names != null && names.nonEmpty) {
appendReadColumnNames(conf, names)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/ClientWrapper.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/ClientWrapper.scala
index f49c97de8ff4e8286ba8bcf95b99600451b2b88a..4d1e3ed9198e684653a8fe01510d56e5f9e9899e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/ClientWrapper.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/ClientWrapper.scala
@@ -21,7 +21,7 @@ import java.io.{File, PrintStream}
import java.util.{Map => JMap}
import javax.annotation.concurrent.GuardedBy
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.language.reflectiveCalls
import org.apache.hadoop.fs.Path
@@ -305,10 +305,11 @@ private[hive] class ClientWrapper(
HiveTable(
name = h.getTableName,
specifiedDatabase = Option(h.getDbName),
- schema = h.getCols.map(f => HiveColumn(f.getName, f.getType, f.getComment)),
- partitionColumns = h.getPartCols.map(f => HiveColumn(f.getName, f.getType, f.getComment)),
- properties = h.getParameters.toMap,
- serdeProperties = h.getTTable.getSd.getSerdeInfo.getParameters.toMap,
+ schema = h.getCols.asScala.map(f => HiveColumn(f.getName, f.getType, f.getComment)),
+ partitionColumns = h.getPartCols.asScala.map(f =>
+ HiveColumn(f.getName, f.getType, f.getComment)),
+ properties = h.getParameters.asScala.toMap,
+ serdeProperties = h.getTTable.getSd.getSerdeInfo.getParameters.asScala.toMap,
tableType = h.getTableType match {
case HTableType.MANAGED_TABLE => ManagedTable
case HTableType.EXTERNAL_TABLE => ExternalTable
@@ -334,9 +335,9 @@ private[hive] class ClientWrapper(
private def toQlTable(table: HiveTable): metadata.Table = {
val qlTable = new metadata.Table(table.database, table.name)
- qlTable.setFields(table.schema.map(c => new FieldSchema(c.name, c.hiveType, c.comment)))
+ qlTable.setFields(table.schema.map(c => new FieldSchema(c.name, c.hiveType, c.comment)).asJava)
qlTable.setPartCols(
- table.partitionColumns.map(c => new FieldSchema(c.name, c.hiveType, c.comment)))
+ table.partitionColumns.map(c => new FieldSchema(c.name, c.hiveType, c.comment)).asJava)
table.properties.foreach { case (k, v) => qlTable.setProperty(k, v) }
table.serdeProperties.foreach { case (k, v) => qlTable.setSerdeParam(k, v) }
@@ -366,13 +367,13 @@ private[hive] class ClientWrapper(
private def toHivePartition(partition: metadata.Partition): HivePartition = {
val apiPartition = partition.getTPartition
HivePartition(
- values = Option(apiPartition.getValues).map(_.toSeq).getOrElse(Seq.empty),
+ values = Option(apiPartition.getValues).map(_.asScala).getOrElse(Seq.empty),
storage = HiveStorageDescriptor(
location = apiPartition.getSd.getLocation,
inputFormat = apiPartition.getSd.getInputFormat,
outputFormat = apiPartition.getSd.getOutputFormat,
serde = apiPartition.getSd.getSerdeInfo.getSerializationLib,
- serdeProperties = apiPartition.getSd.getSerdeInfo.getParameters.toMap))
+ serdeProperties = apiPartition.getSd.getSerdeInfo.getParameters.asScala.toMap))
}
override def getPartitionOption(
@@ -397,7 +398,7 @@ private[hive] class ClientWrapper(
}
override def listTables(dbName: String): Seq[String] = withHiveState {
- client.getAllTables(dbName)
+ client.getAllTables(dbName).asScala
}
/**
@@ -514,17 +515,17 @@ private[hive] class ClientWrapper(
}
def reset(): Unit = withHiveState {
- client.getAllTables("default").foreach { t =>
+ client.getAllTables("default").asScala.foreach { t =>
logDebug(s"Deleting table $t")
val table = client.getTable("default", t)
- client.getIndexes("default", t, 255).foreach { index =>
+ client.getIndexes("default", t, 255).asScala.foreach { index =>
shim.dropIndex(client, "default", t, index.getIndexName)
}
if (!table.isIndexTable) {
client.dropTable("default", t)
}
}
- client.getAllDatabases.filterNot(_ == "default").foreach { db =>
+ client.getAllDatabases.asScala.filterNot(_ == "default").foreach { db =>
logDebug(s"Dropping Database: $db")
client.dropDatabase(db, true, false, true)
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 8fc8935b1dc3c91ff2da3cbd6f214235c7ffde1d..48bbb21e6c1de3a9bb74e96f2aca1a3415e31638 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -23,7 +23,7 @@ import java.net.URI
import java.util.{ArrayList => JArrayList, List => JList, Map => JMap, Set => JSet}
import java.util.concurrent.TimeUnit
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.hive.conf.HiveConf
@@ -201,7 +201,7 @@ private[client] class Shim_v0_12 extends Shim with Logging {
setDataLocationMethod.invoke(table, new URI(loc))
override def getAllPartitions(hive: Hive, table: Table): Seq[Partition] =
- getAllPartitionsMethod.invoke(hive, table).asInstanceOf[JSet[Partition]].toSeq
+ getAllPartitionsMethod.invoke(hive, table).asInstanceOf[JSet[Partition]].asScala.toSeq
override def getPartitionsByFilter(
hive: Hive,
@@ -220,7 +220,7 @@ private[client] class Shim_v0_12 extends Shim with Logging {
override def getDriverResults(driver: Driver): Seq[String] = {
val res = new JArrayList[String]()
getDriverResultsMethod.invoke(driver, res)
- res.toSeq
+ res.asScala
}
override def getMetastoreClientConnectRetryDelayMillis(conf: HiveConf): Long = {
@@ -310,7 +310,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
setDataLocationMethod.invoke(table, new Path(loc))
override def getAllPartitions(hive: Hive, table: Table): Seq[Partition] =
- getAllPartitionsMethod.invoke(hive, table).asInstanceOf[JSet[Partition]].toSeq
+ getAllPartitionsMethod.invoke(hive, table).asInstanceOf[JSet[Partition]].asScala.toSeq
/**
* Converts catalyst expression to the format that Hive's getPartitionsByFilter() expects, i.e.
@@ -320,7 +320,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
*/
def convertFilters(table: Table, filters: Seq[Expression]): String = {
// hive varchar is treated as catalyst string, but hive varchar can't be pushed down.
- val varcharKeys = table.getPartitionKeys
+ val varcharKeys = table.getPartitionKeys.asScala
.filter(col => col.getType.startsWith(serdeConstants.VARCHAR_TYPE_NAME))
.map(col => col.getName).toSet
@@ -354,7 +354,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
getPartitionsByFilterMethod.invoke(hive, table, filter).asInstanceOf[JArrayList[Partition]]
}
- partitions.toSeq
+ partitions.asScala.toSeq
}
override def getCommandProcessor(token: String, conf: HiveConf): CommandProcessor =
@@ -363,7 +363,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
override def getDriverResults(driver: Driver): Seq[String] = {
val res = new JArrayList[Object]()
getDriverResultsMethod.invoke(driver, res)
- res.map { r =>
+ res.asScala.map { r =>
r match {
case s: String => s
case a: Array[Object] => a(0).asInstanceOf[String]
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
index 5f0ed5393d191276530e7906d78be3be5bbd08d5..441b6b6033e1f15730622f6cfe61dcb07dce70dc 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/DescribeHiveTableCommand.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.hive.execution
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.hadoop.hive.metastore.api.FieldSchema
@@ -39,8 +39,8 @@ case class DescribeHiveTableCommand(
// Trying to mimic the format of Hive's output. But not exactly the same.
var results: Seq[(String, String, String)] = Nil
- val columns: Seq[FieldSchema] = table.hiveQlTable.getCols
- val partitionColumns: Seq[FieldSchema] = table.hiveQlTable.getPartCols
+ val columns: Seq[FieldSchema] = table.hiveQlTable.getCols.asScala
+ val partitionColumns: Seq[FieldSchema] = table.hiveQlTable.getPartCols.asScala
results ++= columns.map(field => (field.getName, field.getType, field.getComment))
if (partitionColumns.nonEmpty) {
val partColumnInfo =
@@ -48,7 +48,7 @@ case class DescribeHiveTableCommand(
results ++=
partColumnInfo ++
Seq(("# Partition Information", "", "")) ++
- Seq((s"# ${output.get(0).name}", output.get(1).name, output.get(2).name)) ++
+ Seq((s"# ${output(0).name}", output(1).name, output(2).name)) ++
partColumnInfo
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScan.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScan.scala
index ba7eb15a1c0c65dfd8ee56c3f2773cfe2d7af491..806d2b9b0b7d405c2b7cd6befea0b63dde79ab79 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScan.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScan.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.hive.execution
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.ql.metadata.{Partition => HivePartition}
@@ -98,7 +98,7 @@ case class HiveTableScan(
.asInstanceOf[StructObjectInspector]
val columnTypeNames = structOI
- .getAllStructFieldRefs
+ .getAllStructFieldRefs.asScala
.map(_.getFieldObjectInspector)
.map(TypeInfoUtils.getTypeInfoFromObjectInspector(_).getTypeName)
.mkString(",")
@@ -118,9 +118,8 @@ case class HiveTableScan(
case None => partitions
case Some(shouldKeep) => partitions.filter { part =>
val dataTypes = relation.partitionKeys.map(_.dataType)
- val castedValues = for ((value, dataType) <- part.getValues.zip(dataTypes)) yield {
- castFromString(value, dataType)
- }
+ val castedValues = part.getValues.asScala.zip(dataTypes)
+ .map { case (value, dataType) => castFromString(value, dataType) }
// Only partitioned values are needed here, since the predicate has already been bound to
// partition key attribute references.
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
index 62efda613a176aae866fc5564a7ffabba852656d..58f7fa640e8a977f4217da3712140041c2b86aa9 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
@@ -19,9 +19,10 @@ package org.apache.spark.sql.hive.execution
import java.util
+import scala.collection.JavaConverters._
+
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.apache.hadoop.hive.metastore.MetaStoreUtils
import org.apache.hadoop.hive.ql.plan.TableDesc
import org.apache.hadoop.hive.ql.{Context, ErrorMsg}
import org.apache.hadoop.hive.serde2.Serializer
@@ -38,8 +39,6 @@ import org.apache.spark.sql.hive.HiveShim.{ShimFileSinkDesc => FileSinkDesc}
import org.apache.spark.sql.hive._
import org.apache.spark.sql.types.DataType
import org.apache.spark.{SparkException, TaskContext}
-
-import scala.collection.JavaConversions._
import org.apache.spark.util.SerializableJobConf
private[hive]
@@ -94,7 +93,8 @@ case class InsertIntoHiveTable(
ObjectInspectorCopyOption.JAVA)
.asInstanceOf[StructObjectInspector]
- val fieldOIs = standardOI.getAllStructFieldRefs.map(_.getFieldObjectInspector).toArray
+ val fieldOIs = standardOI.getAllStructFieldRefs.asScala
+ .map(_.getFieldObjectInspector).toArray
val dataTypes: Array[DataType] = child.output.map(_.dataType).toArray
val wrappers = fieldOIs.zip(dataTypes).map { case (f, dt) => wrapperFor(f, dt)}
val outputData = new Array[Any](fieldOIs.length)
@@ -198,7 +198,7 @@ case class InsertIntoHiveTable(
// loadPartition call orders directories created on the iteration order of the this map
val orderedPartitionSpec = new util.LinkedHashMap[String, String]()
- table.hiveQlTable.getPartCols().foreach { entry =>
+ table.hiveQlTable.getPartCols.asScala.foreach { entry =>
orderedPartitionSpec.put(entry.getName, partitionSpec.get(entry.getName).getOrElse(""))
}
@@ -226,7 +226,7 @@ case class InsertIntoHiveTable(
val oldPart =
catalog.client.getPartitionOption(
catalog.client.getTable(table.databaseName, table.tableName),
- partitionSpec)
+ partitionSpec.asJava)
if (oldPart.isEmpty || !ifNotExists) {
catalog.client.loadPartition(
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala
index ade27454b9d29bb18b97f141b47992a2dbe93b1d..c7651daffe36e5de20a396a101fc17aac556c754 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformation.scala
@@ -21,7 +21,7 @@ import java.io._
import java.util.Properties
import javax.annotation.Nullable
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.util.control.NonFatal
import org.apache.hadoop.hive.serde.serdeConstants
@@ -61,7 +61,7 @@ case class ScriptTransformation(
protected override def doExecute(): RDD[InternalRow] = {
def processIterator(inputIterator: Iterator[InternalRow]): Iterator[InternalRow] = {
val cmd = List("/bin/bash", "-c", script)
- val builder = new ProcessBuilder(cmd)
+ val builder = new ProcessBuilder(cmd.asJava)
val proc = builder.start()
val inputStream = proc.getInputStream
@@ -172,10 +172,10 @@ case class ScriptTransformation(
val fieldList = outputSoi.getAllStructFieldRefs()
var i = 0
while (i < dataList.size()) {
- if (dataList(i) == null) {
+ if (dataList.get(i) == null) {
mutableRow.setNullAt(i)
} else {
- mutableRow(i) = unwrap(dataList(i), fieldList(i).getFieldObjectInspector)
+ mutableRow(i) = unwrap(dataList.get(i), fieldList.get(i).getFieldObjectInspector)
}
i += 1
}
@@ -307,7 +307,7 @@ case class HiveScriptIOSchema (
val serde = initSerDe(serdeClass, columns, columnTypes, inputSerdeProps)
val fieldObjectInspectors = columnTypes.map(toInspector)
val objectInspector = ObjectInspectorFactory
- .getStandardStructObjectInspector(columns, fieldObjectInspectors)
+ .getStandardStructObjectInspector(columns.asJava, fieldObjectInspectors.asJava)
.asInstanceOf[ObjectInspector]
(serde, objectInspector)
}
@@ -342,7 +342,7 @@ case class HiveScriptIOSchema (
propsMap = propsMap + (serdeConstants.LIST_COLUMN_TYPES -> columnTypesNames)
val properties = new Properties()
- properties.putAll(propsMap)
+ properties.putAll(propsMap.asJava)
serde.initialize(null, properties)
serde
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index 7182246e466a473d2072e04e4cb020dc6cbd6804..cad02373e5ba140f29b4aa597c73b269eea43c51 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.hive
import scala.collection.mutable.ArrayBuffer
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.util.Try
import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, ConstantObjectInspector}
@@ -81,8 +81,7 @@ private[hive] class HiveFunctionRegistry(underlying: analysis.FunctionRegistry)
/* List all of the registered function names. */
override def listFunction(): Seq[String] = {
- val a = FunctionRegistry.getFunctionNames ++ underlying.listFunction()
- a.toList.sorted
+ (FunctionRegistry.getFunctionNames.asScala ++ underlying.listFunction()).toList.sorted
}
/* Get the class of the registered function by specified name. */
@@ -116,7 +115,7 @@ private[hive] case class HiveSimpleUDF(funcWrapper: HiveFunctionWrapper, childre
@transient
private lazy val method =
- function.getResolver.getEvalMethod(children.map(_.dataType.toTypeInfo))
+ function.getResolver.getEvalMethod(children.map(_.dataType.toTypeInfo).asJava)
@transient
private lazy val arguments = children.map(toInspector).toArray
@@ -541,7 +540,7 @@ private[hive] case class HiveGenericUDTF(
@transient
protected lazy val collector = new UDTFCollector
- lazy val elementTypes = outputInspector.getAllStructFieldRefs.map {
+ lazy val elementTypes = outputInspector.getAllStructFieldRefs.asScala.map {
field => (inspectorToDataType(field.getFieldObjectInspector), true)
}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala
index 9f4f8b5789afeb9368bc0f1c178ff03c48e676b9..1cff5cf9c3543b8606da11d164effca7246efb1f 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcRelation.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.hive.orc
import java.util.Properties
+import scala.collection.JavaConverters._
+
import com.google.common.base.Objects
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileStatus, Path}
@@ -43,9 +45,6 @@ import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.{Row, SQLContext}
import org.apache.spark.util.SerializableConfiguration
-/* Implicit conversions */
-import scala.collection.JavaConversions._
-
private[sql] class DefaultSource extends HadoopFsRelationProvider with DataSourceRegister {
override def shortName(): String = "orc"
@@ -97,7 +96,8 @@ private[orc] class OrcOutputWriter(
private val reusableOutputBuffer = new Array[Any](dataSchema.length)
// Used to convert Catalyst values into Hadoop `Writable`s.
- private val wrappers = structOI.getAllStructFieldRefs.zip(dataSchema.fields.map(_.dataType))
+ private val wrappers = structOI.getAllStructFieldRefs.asScala
+ .zip(dataSchema.fields.map(_.dataType))
.map { case (ref, dt) =>
wrapperFor(ref.getFieldObjectInspector, dt)
}.toArray
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
index 4da86636ac100d073cb3b8753f29faf164165910..572eaebe81ac202f10b3c99fdff711e276510670 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.hive.test
import java.io.File
import java.util.{Set => JavaSet}
+import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.language.implicitConversions
@@ -37,9 +38,6 @@ import org.apache.spark.sql.hive.execution.HiveNativeCommand
import org.apache.spark.util.{ShutdownHookManager, Utils}
import org.apache.spark.{SparkConf, SparkContext}
-/* Implicit conversions */
-import scala.collection.JavaConversions._
-
// SPARK-3729: Test key required to check for initialization errors with config.
object TestHive
extends TestHiveContext(
@@ -405,7 +403,7 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) {
def reset() {
try {
// HACK: Hive is too noisy by default.
- org.apache.log4j.LogManager.getCurrentLoggers.foreach { log =>
+ org.apache.log4j.LogManager.getCurrentLoggers.asScala.foreach { log =>
log.asInstanceOf[org.apache.log4j.Logger].setLevel(org.apache.log4j.Level.WARN)
}
@@ -415,9 +413,8 @@ class TestHiveContext(sc: SparkContext) extends HiveContext(sc) {
catalog.client.reset()
catalog.unregisterAllTables()
- FunctionRegistry.getFunctionNames.filterNot(originalUDFs.contains(_)).foreach { udfName =>
- FunctionRegistry.unregisterTemporaryUDF(udfName)
- }
+ FunctionRegistry.getFunctionNames.asScala.filterNot(originalUDFs.contains(_)).
+ foreach { udfName => FunctionRegistry.unregisterTemporaryUDF(udfName) }
// Some tests corrupt this value on purpose, which breaks the RESET call below.
hiveconf.set("fs.default.name", new File(".").toURI.toString)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala
index 0efcf80bd4ea722e147ccb6a2363bc7cd8e53e0a..5e7b93d45710604c3504385289fe39bdd06bf19d 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/FiltersSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.hive.client
-import scala.collection.JavaConversions._
+import java.util.Collections
import org.apache.hadoop.hive.metastore.api.FieldSchema
import org.apache.hadoop.hive.serde.serdeConstants
@@ -38,7 +38,7 @@ class FiltersSuite extends SparkFunSuite with Logging {
private val varCharCol = new FieldSchema()
varCharCol.setName("varchar")
varCharCol.setType(serdeConstants.VARCHAR_TYPE_NAME)
- testTable.setPartCols(varCharCol :: Nil)
+ testTable.setPartCols(Collections.singletonList(varCharCol))
filterTest("string filter",
(a("stringcol", StringType) > Literal("test")) :: Nil,
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index b03a35132325d35bc2769faf1998da58fbd45f85..9c10ffe1113dc9dc09ef4ce550e425ec30ffcca1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -18,8 +18,7 @@
package org.apache.spark.sql.hive.execution
import java.io.{DataInput, DataOutput}
-import java.util
-import java.util.Properties
+import java.util.{ArrayList, Arrays, Properties}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hive.ql.udf.generic.{GenericUDAFAverage, GenericUDF}
@@ -33,8 +32,6 @@ import org.apache.spark.sql.hive.test.TestHive
import org.apache.spark.util.Utils
-import scala.collection.JavaConversions._
-
case class Fields(f1: Int, f2: Int, f3: Int, f4: Int, f5: Int)
// Case classes for the custom UDF's.
@@ -326,11 +323,11 @@ class PairSerDe extends AbstractSerDe {
override def getObjectInspector: ObjectInspector = {
ObjectInspectorFactory
.getStandardStructObjectInspector(
- Seq("pair"),
- Seq(ObjectInspectorFactory.getStandardStructObjectInspector(
- Seq("id", "value"),
- Seq(PrimitiveObjectInspectorFactory.javaIntObjectInspector,
- PrimitiveObjectInspectorFactory.javaIntObjectInspector))
+ Arrays.asList("pair"),
+ Arrays.asList(ObjectInspectorFactory.getStandardStructObjectInspector(
+ Arrays.asList("id", "value"),
+ Arrays.asList(PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector))
))
}
@@ -343,10 +340,10 @@ class PairSerDe extends AbstractSerDe {
override def deserialize(value: Writable): AnyRef = {
val pair = value.asInstanceOf[TestPair]
- val row = new util.ArrayList[util.ArrayList[AnyRef]]
- row.add(new util.ArrayList[AnyRef](2))
- row(0).add(Integer.valueOf(pair.entry._1))
- row(0).add(Integer.valueOf(pair.entry._2))
+ val row = new ArrayList[ArrayList[AnyRef]]
+ row.add(new ArrayList[AnyRef](2))
+ row.get(0).add(Integer.valueOf(pair.entry._1))
+ row.get(0).add(Integer.valueOf(pair.entry._2))
row
}
@@ -355,9 +352,9 @@ class PairSerDe extends AbstractSerDe {
class PairUDF extends GenericUDF {
override def initialize(p1: Array[ObjectInspector]): ObjectInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(
- Seq("id", "value"),
- Seq(PrimitiveObjectInspectorFactory.javaIntObjectInspector,
- PrimitiveObjectInspectorFactory.javaIntObjectInspector)
+ Arrays.asList("id", "value"),
+ Arrays.asList(PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector)
)
override def evaluate(args: Array[DeferredObject]): AnyRef = {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruningSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruningSuite.scala
index 3bf8f3ac20480e5bb98a10d5610ceacd3bc32048..210d5667454150de930bcb37a93ea38ed78d6609 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruningSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruningSuite.scala
@@ -17,13 +17,12 @@
package org.apache.spark.sql.hive.execution
+import scala.collection.JavaConverters._
+
import org.scalatest.BeforeAndAfter
import org.apache.spark.sql.hive.test.TestHive
-/* Implicit conversions */
-import scala.collection.JavaConversions._
-
/**
* A set of test cases that validate partition and column pruning.
*/
@@ -161,7 +160,7 @@ class PruningSuite extends HiveComparisonTest with BeforeAndAfter {
assert(actualOutputColumns === expectedOutputColumns, "Output columns mismatch")
assert(actualScannedColumns === expectedScannedColumns, "Scanned columns mismatch")
- val actualPartitions = actualPartValues.map(_.toSeq.mkString(",")).sorted
+ val actualPartitions = actualPartValues.map(_.asScala.mkString(",")).sorted
val expectedPartitions = expectedPartValues.map(_.mkString(",")).sorted
assert(actualPartitions === expectedPartitions, "Partitions selected do not match")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 55ecbd5b5f21d12b8c20aaa0d0652b55dee04570..1ff1d9a2934cc5a0110e19bd8483569950781fc3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.hive.execution
import java.sql.{Date, Timestamp}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.DefaultParserDialect
@@ -164,7 +164,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils {
test("show functions") {
val allFunctions =
(FunctionRegistry.builtin.listFunction().toSet[String] ++
- org.apache.hadoop.hive.ql.exec.FunctionRegistry.getFunctionNames).toList.sorted
+ org.apache.hadoop.hive.ql.exec.FunctionRegistry.getFunctionNames.asScala).toList.sorted
checkAnswer(sql("SHOW functions"), allFunctions.map(Row(_)))
checkAnswer(sql("SHOW functions abs"), Row("abs"))
checkAnswer(sql("SHOW functions 'abs'"), Row("abs"))
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/hadoopFsRelationSuites.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/hadoopFsRelationSuites.scala
index 5bbca14bad320ac2d03500a1728743ce7bea0deb..7966b43596e750225fc57c4bab03f2cd239f5a84 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/hadoopFsRelationSuites.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/hadoopFsRelationSuites.scala
@@ -17,9 +17,7 @@
package org.apache.spark.sql.sources
-import java.sql.Date
-
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
@@ -552,7 +550,7 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils {
} finally {
// Hadoop 1 doesn't have `Configuration.unset`
configuration.clear()
- clonedConf.foreach(entry => configuration.set(entry.getKey, entry.getValue))
+ clonedConf.asScala.foreach(entry => configuration.set(entry.getKey, entry.getValue))
}
}
@@ -600,7 +598,7 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils {
} finally {
// Hadoop 1 doesn't have `Configuration.unset`
configuration.clear()
- clonedConf.foreach(entry => configuration.set(entry.getKey, entry.getValue))
+ clonedConf.asScala.foreach(entry => configuration.set(entry.getKey, entry.getValue))
sqlContext.sparkContext.conf.set("spark.speculation", speculationEnabled.toString)
}
}
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaDStreamLike.scala b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaDStreamLike.scala
index 214cd80108b9b489686b6827ca5ca5643cde0ef4..edfa474677f159542c059f0d430e692da54aed56 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaDStreamLike.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaDStreamLike.scala
@@ -17,11 +17,10 @@
package org.apache.spark.streaming.api.java
-import java.util
import java.lang.{Long => JLong}
import java.util.{List => JList}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.language.implicitConversions
import scala.reflect.ClassTag
@@ -145,8 +144,7 @@ trait JavaDStreamLike[T, This <: JavaDStreamLike[T, This, R], R <: JavaRDDLike[T
* an array.
*/
def glom(): JavaDStream[JList[T]] =
- new JavaDStream(dstream.glom().map(x => new java.util.ArrayList[T](x.toSeq)))
-
+ new JavaDStream(dstream.glom().map(_.toSeq.asJava))
/** Return the [[org.apache.spark.streaming.StreamingContext]] associated with this DStream */
@@ -191,7 +189,7 @@ trait JavaDStreamLike[T, This <: JavaDStreamLike[T, This, R], R <: JavaRDDLike[T
*/
def mapPartitions[U](f: FlatMapFunction[java.util.Iterator[T], U]): JavaDStream[U] = {
def fn: (Iterator[T]) => Iterator[U] = {
- (x: Iterator[T]) => asScalaIterator(f.call(asJavaIterator(x)).iterator())
+ (x: Iterator[T]) => f.call(x.asJava).iterator().asScala
}
new JavaDStream(dstream.mapPartitions(fn)(fakeClassTag[U]))(fakeClassTag[U])
}
@@ -204,7 +202,7 @@ trait JavaDStreamLike[T, This <: JavaDStreamLike[T, This, R], R <: JavaRDDLike[T
def mapPartitionsToPair[K2, V2](f: PairFlatMapFunction[java.util.Iterator[T], K2, V2])
: JavaPairDStream[K2, V2] = {
def fn: (Iterator[T]) => Iterator[(K2, V2)] = {
- (x: Iterator[T]) => asScalaIterator(f.call(asJavaIterator(x)).iterator())
+ (x: Iterator[T]) => f.call(x.asJava).iterator().asScala
}
new JavaPairDStream(dstream.mapPartitions(fn))(fakeClassTag[K2], fakeClassTag[V2])
}
@@ -282,7 +280,7 @@ trait JavaDStreamLike[T, This <: JavaDStreamLike[T, This, R], R <: JavaRDDLike[T
* Return all the RDDs between 'fromDuration' to 'toDuration' (both included)
*/
def slice(fromTime: Time, toTime: Time): JList[R] = {
- new util.ArrayList(dstream.slice(fromTime, toTime).map(wrapRDD(_)).toSeq)
+ dstream.slice(fromTime, toTime).map(wrapRDD).asJava
}
/**
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala
index 26383e420101e8e3c41c491351f09cffebd820f5..e2aec6c2f63e705941b64330394c32e546b5b610 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaPairDStream.scala
@@ -20,7 +20,7 @@ package org.apache.spark.streaming.api.java
import java.lang.{Long => JLong, Iterable => JIterable}
import java.util.{List => JList}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.language.implicitConversions
import scala.reflect.ClassTag
@@ -116,14 +116,14 @@ class JavaPairDStream[K, V](val dstream: DStream[(K, V)])(
* generate the RDDs with Spark's default number of partitions.
*/
def groupByKey(): JavaPairDStream[K, JIterable[V]] =
- dstream.groupByKey().mapValues(asJavaIterable _)
+ dstream.groupByKey().mapValues(_.asJava)
/**
* Return a new DStream by applying `groupByKey` to each RDD. Hash partitioning is used to
* generate the RDDs with `numPartitions` partitions.
*/
def groupByKey(numPartitions: Int): JavaPairDStream[K, JIterable[V]] =
- dstream.groupByKey(numPartitions).mapValues(asJavaIterable _)
+ dstream.groupByKey(numPartitions).mapValues(_.asJava)
/**
* Return a new DStream by applying `groupByKey` on each RDD of `this` DStream.
@@ -132,7 +132,7 @@ class JavaPairDStream[K, V](val dstream: DStream[(K, V)])(
* is used to control the partitioning of each RDD.
*/
def groupByKey(partitioner: Partitioner): JavaPairDStream[K, JIterable[V]] =
- dstream.groupByKey(partitioner).mapValues(asJavaIterable _)
+ dstream.groupByKey(partitioner).mapValues(_.asJava)
/**
* Return a new DStream by applying `reduceByKey` to each RDD. The values for each key are
@@ -197,7 +197,7 @@ class JavaPairDStream[K, V](val dstream: DStream[(K, V)])(
* batching interval
*/
def groupByKeyAndWindow(windowDuration: Duration): JavaPairDStream[K, JIterable[V]] = {
- dstream.groupByKeyAndWindow(windowDuration).mapValues(asJavaIterable _)
+ dstream.groupByKeyAndWindow(windowDuration).mapValues(_.asJava)
}
/**
@@ -212,7 +212,7 @@ class JavaPairDStream[K, V](val dstream: DStream[(K, V)])(
*/
def groupByKeyAndWindow(windowDuration: Duration, slideDuration: Duration)
: JavaPairDStream[K, JIterable[V]] = {
- dstream.groupByKeyAndWindow(windowDuration, slideDuration).mapValues(asJavaIterable _)
+ dstream.groupByKeyAndWindow(windowDuration, slideDuration).mapValues(_.asJava)
}
/**
@@ -228,8 +228,7 @@ class JavaPairDStream[K, V](val dstream: DStream[(K, V)])(
*/
def groupByKeyAndWindow(windowDuration: Duration, slideDuration: Duration, numPartitions: Int)
: JavaPairDStream[K, JIterable[V]] = {
- dstream.groupByKeyAndWindow(windowDuration, slideDuration, numPartitions)
- .mapValues(asJavaIterable _)
+ dstream.groupByKeyAndWindow(windowDuration, slideDuration, numPartitions).mapValues(_.asJava)
}
/**
@@ -248,8 +247,7 @@ class JavaPairDStream[K, V](val dstream: DStream[(K, V)])(
slideDuration: Duration,
partitioner: Partitioner
): JavaPairDStream[K, JIterable[V]] = {
- dstream.groupByKeyAndWindow(windowDuration, slideDuration, partitioner)
- .mapValues(asJavaIterable _)
+ dstream.groupByKeyAndWindow(windowDuration, slideDuration, partitioner).mapValues(_.asJava)
}
/**
@@ -431,7 +429,7 @@ class JavaPairDStream[K, V](val dstream: DStream[(K, V)])(
private def convertUpdateStateFunction[S](in: JFunction2[JList[V], Optional[S], Optional[S]]):
(Seq[V], Option[S]) => Option[S] = {
val scalaFunc: (Seq[V], Option[S]) => Option[S] = (values, state) => {
- val list: JList[V] = values
+ val list: JList[V] = values.asJava
val scalaState: Optional[S] = JavaUtils.optionToOptional(state)
val result: Optional[S] = in.apply(list, scalaState)
result.isPresent match {
@@ -539,7 +537,7 @@ class JavaPairDStream[K, V](val dstream: DStream[(K, V)])(
*/
def cogroup[W](other: JavaPairDStream[K, W]): JavaPairDStream[K, (JIterable[V], JIterable[W])] = {
implicit val cm: ClassTag[W] = fakeClassTag
- dstream.cogroup(other.dstream).mapValues(t => (asJavaIterable(t._1), asJavaIterable((t._2))))
+ dstream.cogroup(other.dstream).mapValues(t => (t._1.asJava, t._2.asJava))
}
/**
@@ -551,8 +549,7 @@ class JavaPairDStream[K, V](val dstream: DStream[(K, V)])(
numPartitions: Int
): JavaPairDStream[K, (JIterable[V], JIterable[W])] = {
implicit val cm: ClassTag[W] = fakeClassTag
- dstream.cogroup(other.dstream, numPartitions)
- .mapValues(t => (asJavaIterable(t._1), asJavaIterable((t._2))))
+ dstream.cogroup(other.dstream, numPartitions).mapValues(t => (t._1.asJava, t._2.asJava))
}
/**
@@ -564,8 +561,7 @@ class JavaPairDStream[K, V](val dstream: DStream[(K, V)])(
partitioner: Partitioner
): JavaPairDStream[K, (JIterable[V], JIterable[W])] = {
implicit val cm: ClassTag[W] = fakeClassTag
- dstream.cogroup(other.dstream, partitioner)
- .mapValues(t => (asJavaIterable(t._1), asJavaIterable((t._2))))
+ dstream.cogroup(other.dstream, partitioner).mapValues(t => (t._1.asJava, t._2.asJava))
}
/**
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
index 35cc3ce5cf468f698e903911a90d542705a70f39..13f371f29603a8ae358d3a89c78f3f9395c8c7f8 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaStreamingContext.scala
@@ -21,7 +21,7 @@ import java.lang.{Boolean => JBoolean}
import java.io.{Closeable, InputStream}
import java.util.{List => JList, Map => JMap}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import akka.actor.{Props, SupervisorStrategy}
@@ -115,7 +115,13 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
sparkHome: String,
jars: Array[String],
environment: JMap[String, String]) =
- this(new StreamingContext(master, appName, batchDuration, sparkHome, jars, environment))
+ this(new StreamingContext(
+ master,
+ appName,
+ batchDuration,
+ sparkHome,
+ jars,
+ environment.asScala))
/**
* Create a JavaStreamingContext using an existing JavaSparkContext.
@@ -197,7 +203,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
converter: JFunction[InputStream, java.lang.Iterable[T]],
storageLevel: StorageLevel)
: JavaReceiverInputDStream[T] = {
- def fn: (InputStream) => Iterator[T] = (x: InputStream) => converter.call(x).toIterator
+ def fn: (InputStream) => Iterator[T] = (x: InputStream) => converter.call(x).iterator().asScala
implicit val cmt: ClassTag[T] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
ssc.socketStream(hostname, port, fn, storageLevel)
@@ -432,7 +438,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
implicit val cm: ClassTag[T] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
val sQueue = new scala.collection.mutable.Queue[RDD[T]]
- sQueue.enqueue(queue.map(_.rdd).toSeq: _*)
+ sQueue.enqueue(queue.asScala.map(_.rdd).toSeq: _*)
ssc.queueStream(sQueue)
}
@@ -456,7 +462,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
implicit val cm: ClassTag[T] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
val sQueue = new scala.collection.mutable.Queue[RDD[T]]
- sQueue.enqueue(queue.map(_.rdd).toSeq: _*)
+ sQueue.enqueue(queue.asScala.map(_.rdd).toSeq: _*)
ssc.queueStream(sQueue, oneAtATime)
}
@@ -481,7 +487,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
implicit val cm: ClassTag[T] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
val sQueue = new scala.collection.mutable.Queue[RDD[T]]
- sQueue.enqueue(queue.map(_.rdd).toSeq: _*)
+ sQueue.enqueue(queue.asScala.map(_.rdd).toSeq: _*)
ssc.queueStream(sQueue, oneAtATime, defaultRDD.rdd)
}
@@ -500,7 +506,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
* Create a unified DStream from multiple DStreams of the same type and same slide duration.
*/
def union[T](first: JavaDStream[T], rest: JList[JavaDStream[T]]): JavaDStream[T] = {
- val dstreams: Seq[DStream[T]] = (Seq(first) ++ asScalaBuffer(rest)).map(_.dstream)
+ val dstreams: Seq[DStream[T]] = (Seq(first) ++ rest.asScala).map(_.dstream)
implicit val cm: ClassTag[T] = first.classTag
ssc.union(dstreams)(cm)
}
@@ -512,7 +518,7 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
first: JavaPairDStream[K, V],
rest: JList[JavaPairDStream[K, V]]
): JavaPairDStream[K, V] = {
- val dstreams: Seq[DStream[(K, V)]] = (Seq(first) ++ asScalaBuffer(rest)).map(_.dstream)
+ val dstreams: Seq[DStream[(K, V)]] = (Seq(first) ++ rest.asScala).map(_.dstream)
implicit val cm: ClassTag[(K, V)] = first.classTag
implicit val kcm: ClassTag[K] = first.kManifest
implicit val vcm: ClassTag[V] = first.vManifest
@@ -534,12 +540,11 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
): JavaDStream[T] = {
implicit val cmt: ClassTag[T] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
- val scalaDStreams = dstreams.map(_.dstream).toSeq
val scalaTransformFunc = (rdds: Seq[RDD[_]], time: Time) => {
- val jrdds = rdds.map(rdd => JavaRDD.fromRDD[AnyRef](rdd.asInstanceOf[RDD[AnyRef]])).toList
+ val jrdds = rdds.map(JavaRDD.fromRDD(_)).asJava
transformFunc.call(jrdds, time).rdd
}
- ssc.transform(scalaDStreams, scalaTransformFunc)
+ ssc.transform(dstreams.asScala.map(_.dstream).toSeq, scalaTransformFunc)
}
/**
@@ -559,12 +564,11 @@ class JavaStreamingContext(val ssc: StreamingContext) extends Closeable {
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[K]]
implicit val cmv: ClassTag[V] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[V]]
- val scalaDStreams = dstreams.map(_.dstream).toSeq
val scalaTransformFunc = (rdds: Seq[RDD[_]], time: Time) => {
- val jrdds = rdds.map(rdd => JavaRDD.fromRDD[AnyRef](rdd.asInstanceOf[RDD[AnyRef]])).toList
+ val jrdds = rdds.map(JavaRDD.fromRDD(_)).asJava
transformFunc.call(jrdds, time).rdd
}
- ssc.transform(scalaDStreams, scalaTransformFunc)
+ ssc.transform(dstreams.asScala.map(_.dstream).toSeq, scalaTransformFunc)
}
/**
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/api/python/PythonDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/api/python/PythonDStream.scala
index d06401245ff17f4a6ae40669d97dad52c39e9bf9..2c373640d2fd96423eeff260823ab319bb4ec83f 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/api/python/PythonDStream.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/api/python/PythonDStream.scala
@@ -20,14 +20,13 @@ package org.apache.spark.streaming.api.python
import java.io.{ObjectInputStream, ObjectOutputStream}
import java.lang.reflect.Proxy
import java.util.{ArrayList => JArrayList, List => JList}
-import scala.collection.JavaConversions._
+
import scala.collection.JavaConverters._
import scala.language.existentials
import py4j.GatewayServer
import org.apache.spark.api.java._
-import org.apache.spark.api.python._
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{Interval, Duration, Time}
@@ -161,7 +160,7 @@ private[python] object PythonDStream {
*/
def toRDDQueue(rdds: JArrayList[JavaRDD[Array[Byte]]]): java.util.Queue[JavaRDD[Array[Byte]]] = {
val queue = new java.util.LinkedList[JavaRDD[Array[Byte]]]
- rdds.forall(queue.add(_))
+ rdds.asScala.foreach(queue.add)
queue
}
}
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/receiver/Receiver.scala b/streaming/src/main/scala/org/apache/spark/streaming/receiver/Receiver.scala
index 554aae0117b24f7c3713ec0448705e706498eafe..2252e28f22af8365b25f5b941018a9757e7a9462 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/receiver/Receiver.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/receiver/Receiver.scala
@@ -20,7 +20,7 @@ package org.apache.spark.streaming.receiver
import java.nio.ByteBuffer
import scala.collection.mutable.ArrayBuffer
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import org.apache.spark.storage.StorageLevel
import org.apache.spark.annotation.DeveloperApi
@@ -144,12 +144,12 @@ abstract class Receiver[T](val storageLevel: StorageLevel) extends Serializable
* for being used in the corresponding InputDStream.
*/
def store(dataIterator: java.util.Iterator[T], metadata: Any) {
- supervisor.pushIterator(dataIterator, Some(metadata), None)
+ supervisor.pushIterator(dataIterator.asScala, Some(metadata), None)
}
/** Store an iterator of received data as a data block into Spark's memory. */
def store(dataIterator: java.util.Iterator[T]) {
- supervisor.pushIterator(dataIterator, None, None)
+ supervisor.pushIterator(dataIterator.asScala, None, None)
}
/**
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala
index 6d4cdc4aa6b105c61be0e8f125421717ae2fb8a0..0cd39594ee92365682775bab1239de115376b33e 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala
@@ -19,7 +19,7 @@ package org.apache.spark.streaming.scheduler
import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.util.{Failure, Success}
import org.apache.spark.Logging
@@ -128,7 +128,7 @@ class JobScheduler(val ssc: StreamingContext) extends Logging {
}
def getPendingTimes(): Seq[Time] = {
- jobSets.keySet.toSeq
+ jobSets.asScala.keys.toSeq
}
def reportError(msg: String, e: Throwable) {
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceivedBlockTracker.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceivedBlockTracker.scala
index 53b96d51c918094e7f9361d166e0f7f8b71a35a9..f2711d1355e60ef44a143524d2c0fdcb09cc3552 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceivedBlockTracker.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/ReceivedBlockTracker.scala
@@ -19,6 +19,7 @@ package org.apache.spark.streaming.scheduler
import java.nio.ByteBuffer
+import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.language.implicitConversions
@@ -196,8 +197,7 @@ private[streaming] class ReceivedBlockTracker(
writeAheadLogOption.foreach { writeAheadLog =>
logInfo(s"Recovering from write ahead logs in ${checkpointDirOption.get}")
- import scala.collection.JavaConversions._
- writeAheadLog.readAll().foreach { byteBuffer =>
+ writeAheadLog.readAll().asScala.foreach { byteBuffer =>
logTrace("Recovering record " + byteBuffer)
Utils.deserialize[ReceivedBlockTrackerLogEvent](
byteBuffer.array, Thread.currentThread().getContextClassLoader) match {
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala
index fe6328b1ce7275f3eda45cb9d69d72fad8960b24..9f4a4d6806ab5fab8e8c6e1ca9ea9f6ac29629c4 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala
@@ -19,6 +19,7 @@ package org.apache.spark.streaming.util
import java.nio.ByteBuffer
import java.util.{Iterator => JIterator}
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import scala.concurrent.{Await, ExecutionContext, Future}
import scala.language.postfixOps
@@ -118,7 +119,6 @@ private[streaming] class FileBasedWriteAheadLog(
* hence the implementation is kept simple.
*/
def readAll(): JIterator[ByteBuffer] = synchronized {
- import scala.collection.JavaConversions._
val logFilesToRead = pastLogs.map{ _.path} ++ currentLogPath
logInfo("Reading from the logs: " + logFilesToRead.mkString("\n"))
@@ -126,7 +126,7 @@ private[streaming] class FileBasedWriteAheadLog(
logDebug(s"Creating log reader with $file")
val reader = new FileBasedWriteAheadLogReader(file, hadoopConf)
CompletionIterator[ByteBuffer, Iterator[ByteBuffer]](reader, reader.close _)
- } flatMap { x => x }
+ }.flatten.asJava
}
/**
diff --git a/streaming/src/test/java/org/apache/spark/streaming/JavaTestUtils.scala b/streaming/src/test/java/org/apache/spark/streaming/JavaTestUtils.scala
index bb80bff6dc2e64f581b992dde1f3b3d199e3ae5a..57b50bdfd6520563f93d2c570d74305ab51f228a 100644
--- a/streaming/src/test/java/org/apache/spark/streaming/JavaTestUtils.scala
+++ b/streaming/src/test/java/org/apache/spark/streaming/JavaTestUtils.scala
@@ -17,16 +17,13 @@
package org.apache.spark.streaming
-import scala.collection.mutable.{SynchronizedBuffer, ArrayBuffer}
+import java.util.{List => JList}
+
+import scala.collection.JavaConverters._
import scala.reflect.ClassTag
-import java.util.{List => JList}
-import org.apache.spark.streaming.api.java.{JavaPairDStream, JavaDStreamLike, JavaDStream, JavaStreamingContext}
-import org.apache.spark.streaming._
-import java.util.ArrayList
-import collection.JavaConversions._
import org.apache.spark.api.java.JavaRDDLike
-import org.apache.spark.streaming.dstream.DStream
+import org.apache.spark.streaming.api.java.{JavaDStreamLike, JavaDStream, JavaStreamingContext}
/** Exposes streaming test functionality in a Java-friendly way. */
trait JavaTestBase extends TestSuiteBase {
@@ -39,7 +36,7 @@ trait JavaTestBase extends TestSuiteBase {
ssc: JavaStreamingContext,
data: JList[JList[T]],
numPartitions: Int) = {
- val seqData = data.map(Seq(_:_*))
+ val seqData = data.asScala.map(_.asScala)
implicit val cm: ClassTag[T] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]
@@ -72,9 +69,7 @@ trait JavaTestBase extends TestSuiteBase {
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[V]]
ssc.getState()
val res = runStreams[V](ssc.ssc, numBatches, numExpectedOutput)
- val out = new ArrayList[JList[V]]()
- res.map(entry => out.append(new ArrayList[V](entry)))
- out
+ res.map(_.asJava).asJava
}
/**
@@ -90,12 +85,7 @@ trait JavaTestBase extends TestSuiteBase {
implicit val cm: ClassTag[V] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[V]]
val res = runStreamsWithPartitions[V](ssc.ssc, numBatches, numExpectedOutput)
- val out = new ArrayList[JList[JList[V]]]()
- res.map{entry =>
- val lists = entry.map(new ArrayList[V](_))
- out.append(new ArrayList[JList[V]](lists))
- }
- out
+ res.map(entry => entry.map(_.asJava).asJava).asJava
}
}
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala
index 325ff7c74c39d84a4dcda9fb8a6011c595000d58..5e49fd00769ad3f116bea6fd2a5b4b36e527be7b 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/util/WriteAheadLogSuite.scala
@@ -20,6 +20,7 @@ import java.io._
import java.nio.ByteBuffer
import java.util
+import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import scala.concurrent.duration._
import scala.language.{implicitConversions, postfixOps}
@@ -417,9 +418,8 @@ object WriteAheadLogSuite {
/** Read all the data in the log file in a directory using the WriteAheadLog class. */
def readDataUsingWriteAheadLog(logDirectory: String): Seq[String] = {
- import scala.collection.JavaConversions._
val wal = new FileBasedWriteAheadLog(new SparkConf(), logDirectory, hadoopConf, 1, 1)
- val data = wal.readAll().map(byteBufferToString).toSeq
+ val data = wal.readAll().asScala.map(byteBufferToString).toSeq
wal.close()
data
}
diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
index 9418beb6b3e3a717a62a06196fff47d255efe089..a0524cabff2d4a5a4194a6d76abebc8c9549d9b4 100644
--- a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
+++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
@@ -22,7 +22,7 @@ import java.io.File
import java.util.jar.JarFile
import scala.collection.mutable
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.reflect.runtime.universe.runtimeMirror
import scala.reflect.runtime.{universe => unv}
import scala.util.Try
@@ -161,7 +161,7 @@ object GenerateMIMAIgnore {
val path = packageName.replace('.', '/')
val resources = classLoader.getResources(path)
- val jars = resources.filter(x => x.getProtocol == "jar")
+ val jars = resources.asScala.filter(_.getProtocol == "jar")
.map(_.getFile.split(":")(1).split("!")(0)).toSeq
jars.flatMap(getClassesFromJar(_, path))
@@ -175,7 +175,7 @@ object GenerateMIMAIgnore {
private def getClassesFromJar(jarPath: String, packageName: String) = {
import scala.collection.mutable
val jar = new JarFile(new File(jarPath))
- val enums = jar.entries().map(_.getName).filter(_.startsWith(packageName))
+ val enums = jar.entries().asScala.map(_.getName).filter(_.startsWith(packageName))
val classes = mutable.HashSet[Class[_]]()
for (entry <- enums if entry.endsWith(".class")) {
try {
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index bff585b46cbbebbdfe632b76e5ffa9c7a796ec26..e9a02baafd28e7f48cbe3b39391f4d2cff3d74fd 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -25,7 +25,7 @@ import java.security.PrivilegedExceptionAction
import java.util.{Properties, UUID}
import java.util.zip.{ZipEntry, ZipOutputStream}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, ListBuffer, Map}
import scala.reflect.runtime.universe
import scala.util.{Try, Success, Failure}
@@ -511,7 +511,7 @@ private[spark] class Client(
val nns = YarnSparkHadoopUtil.get.getNameNodesToAccess(sparkConf) + stagingDirPath
YarnSparkHadoopUtil.get.obtainTokensForNamenodes(
nns, hadoopConf, creds, Some(sparkConf.get("spark.yarn.principal")))
- val t = creds.getAllTokens
+ val t = creds.getAllTokens.asScala
.filter(_.getKind == DelegationTokenIdentifier.HDFS_DELEGATION_KIND)
.head
val newExpiration = t.renew(hadoopConf)
@@ -650,8 +650,8 @@ private[spark] class Client(
distCacheMgr.setDistArchivesEnv(launchEnv)
val amContainer = Records.newRecord(classOf[ContainerLaunchContext])
- amContainer.setLocalResources(localResources)
- amContainer.setEnvironment(launchEnv)
+ amContainer.setLocalResources(localResources.asJava)
+ amContainer.setEnvironment(launchEnv.asJava)
val javaOpts = ListBuffer[String]()
@@ -782,7 +782,7 @@ private[spark] class Client(
// TODO: it would be nicer to just make sure there are no null commands here
val printableCommands = commands.map(s => if (s == null) "null" else s).toList
- amContainer.setCommands(printableCommands)
+ amContainer.setCommands(printableCommands.asJava)
logDebug("===============================================================================")
logDebug("YARN AM launch context:")
@@ -797,7 +797,8 @@ private[spark] class Client(
// send the acl settings into YARN to control who has access via YARN interfaces
val securityManager = new SecurityManager(sparkConf)
- amContainer.setApplicationACLs(YarnSparkHadoopUtil.getApplicationAclsForYarn(securityManager))
+ amContainer.setApplicationACLs(
+ YarnSparkHadoopUtil.getApplicationAclsForYarn(securityManager).asJava)
setupSecurityToken(amContainer)
UserGroupInformation.getCurrentUser().addCredentials(credentials)
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index 4cc50483a17ff99c740f6d28695934d8c60a3e1d..9abd09b3cc7a52bd76ff58a347a93aeeab6a4cff 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -20,14 +20,13 @@ package org.apache.spark.deploy.yarn
import java.io.File
import java.net.URI
import java.nio.ByteBuffer
+import java.util.Collections
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
-import org.apache.spark.util.Utils
-
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.{HashMap, ListBuffer}
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.DataOutputBuffer
import org.apache.hadoop.security.UserGroupInformation
@@ -40,6 +39,7 @@ import org.apache.hadoop.yarn.util.{ConverterUtils, Records}
import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkException}
import org.apache.spark.network.util.JavaUtils
+import org.apache.spark.util.Utils
class ExecutorRunnable(
container: Container,
@@ -74,9 +74,9 @@ class ExecutorRunnable(
.asInstanceOf[ContainerLaunchContext]
val localResources = prepareLocalResources
- ctx.setLocalResources(localResources)
+ ctx.setLocalResources(localResources.asJava)
- ctx.setEnvironment(env)
+ ctx.setEnvironment(env.asJava)
val credentials = UserGroupInformation.getCurrentUser().getCredentials()
val dob = new DataOutputBuffer()
@@ -96,8 +96,9 @@ class ExecutorRunnable(
|===============================================================================
""".stripMargin)
- ctx.setCommands(commands)
- ctx.setApplicationACLs(YarnSparkHadoopUtil.getApplicationAclsForYarn(securityMgr))
+ ctx.setCommands(commands.asJava)
+ ctx.setApplicationACLs(
+ YarnSparkHadoopUtil.getApplicationAclsForYarn(securityMgr).asJava)
// If external shuffle service is enabled, register with the Yarn shuffle service already
// started on the NodeManager and, if authentication is enabled, provide it with our secret
@@ -112,7 +113,7 @@ class ExecutorRunnable(
// Authentication is not enabled, so just provide dummy metadata
ByteBuffer.allocate(0)
}
- ctx.setServiceData(Map[String, ByteBuffer]("spark_shuffle" -> secretBytes))
+ ctx.setServiceData(Collections.singletonMap("spark_shuffle", secretBytes))
}
// Send the start request to the ContainerManager
@@ -314,7 +315,8 @@ class ExecutorRunnable(
env("SPARK_LOG_URL_STDOUT") = s"$baseUrl/stdout?start=-4096"
}
- System.getenv().filterKeys(_.startsWith("SPARK")).foreach { case (k, v) => env(k) = v }
+ System.getenv().asScala.filterKeys(_.startsWith("SPARK"))
+ .foreach { case (k, v) => env(k) = v }
env
}
}
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
index ccf753e69f4b655c1d07e69ed8a31c77d75e3871..5f897cbcb4e9f74bdbfa511ac32176f6a8c7f521 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -21,9 +21,7 @@ import java.util.Collections
import java.util.concurrent._
import java.util.regex.Pattern
-import org.apache.spark.util.Utils
-
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
import com.google.common.util.concurrent.ThreadFactoryBuilder
@@ -39,8 +37,8 @@ import org.apache.log4j.{Level, Logger}
import org.apache.spark.{Logging, SecurityManager, SparkConf}
import org.apache.spark.deploy.yarn.YarnSparkHadoopUtil._
import org.apache.spark.rpc.RpcEndpointRef
-import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._
+import org.apache.spark.util.Utils
/**
* YarnAllocator is charged with requesting containers from the YARN ResourceManager and deciding
@@ -164,7 +162,7 @@ private[yarn] class YarnAllocator(
* Number of container requests at the given location that have not yet been fulfilled.
*/
private def getNumPendingAtLocation(location: String): Int =
- amClient.getMatchingRequests(RM_REQUEST_PRIORITY, location, resource).map(_.size).sum
+ amClient.getMatchingRequests(RM_REQUEST_PRIORITY, location, resource).asScala.map(_.size).sum
/**
* Request as many executors from the ResourceManager as needed to reach the desired total. If
@@ -231,14 +229,14 @@ private[yarn] class YarnAllocator(
numExecutorsRunning,
allocateResponse.getAvailableResources))
- handleAllocatedContainers(allocatedContainers)
+ handleAllocatedContainers(allocatedContainers.asScala)
}
val completedContainers = allocateResponse.getCompletedContainersStatuses()
if (completedContainers.size > 0) {
logDebug("Completed %d containers".format(completedContainers.size))
- processCompletedContainers(completedContainers)
+ processCompletedContainers(completedContainers.asScala)
logDebug("Finished processing %d completed containers. Current running executor count: %d."
.format(completedContainers.size, numExecutorsRunning))
@@ -271,7 +269,7 @@ private[yarn] class YarnAllocator(
val request = createContainerRequest(resource, locality.nodes, locality.racks)
amClient.addContainerRequest(request)
val nodes = request.getNodes
- val hostStr = if (nodes == null || nodes.isEmpty) "Any" else nodes.last
+ val hostStr = if (nodes == null || nodes.isEmpty) "Any" else nodes.asScala.last
logInfo(s"Container request (host: $hostStr, capability: $resource)")
}
} else if (missing < 0) {
@@ -280,7 +278,8 @@ private[yarn] class YarnAllocator(
val matchingRequests = amClient.getMatchingRequests(RM_REQUEST_PRIORITY, ANY_HOST, resource)
if (!matchingRequests.isEmpty) {
- matchingRequests.head.take(numToCancel).foreach(amClient.removeContainerRequest)
+ matchingRequests.iterator().next().asScala
+ .take(numToCancel).foreach(amClient.removeContainerRequest)
} else {
logWarning("Expected to find pending requests, but found none.")
}
@@ -459,7 +458,7 @@ private[yarn] class YarnAllocator(
}
}
- if (allocatedContainerToHostMap.containsKey(containerId)) {
+ if (allocatedContainerToHostMap.contains(containerId)) {
val host = allocatedContainerToHostMap.get(containerId).get
val containerSet = allocatedHostToContainersMap.get(host).get
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala
index 4999f9c06210a6e4b1fd6680d2fdb1e1804297ac..df042bf291de7ccc531f61b682647ce113270a06 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClient.scala
@@ -19,17 +19,15 @@ package org.apache.spark.deploy.yarn
import java.util.{List => JList}
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import scala.collection.{Map, Set}
import scala.util.Try
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.yarn.api.ApplicationConstants
import org.apache.hadoop.yarn.api.records._
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest
import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.yarn.util.ConverterUtils
import org.apache.hadoop.yarn.webapp.util.WebAppUtils
import org.apache.spark.{Logging, SecurityManager, SparkConf}
@@ -108,8 +106,8 @@ private[spark] class YarnRMClient(args: ApplicationMasterArguments) extends Logg
val method = classOf[WebAppUtils].getMethod("getProxyHostsAndPortsForAmFilter",
classOf[Configuration])
val proxies = method.invoke(null, conf).asInstanceOf[JList[String]]
- val hosts = proxies.map { proxy => proxy.split(":")(0) }
- val uriBases = proxies.map { proxy => prefix + proxy + proxyBase }
+ val hosts = proxies.asScala.map { proxy => proxy.split(":")(0) }
+ val uriBases = proxies.asScala.map { proxy => prefix + proxy + proxyBase }
Map("PROXY_HOSTS" -> hosts.mkString(","), "PROXY_URI_BASES" -> uriBases.mkString(","))
} catch {
case e: NoSuchMethodException =>
diff --git a/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala b/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala
index 128e996b71fe5a692c26e679ec351c7034ed8e6b..b4f8049bff57736c29ed713d539f9f4a9d407ab4 100644
--- a/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala
+++ b/yarn/src/test/scala/org/apache/spark/deploy/yarn/BaseYarnClusterSuite.scala
@@ -21,7 +21,7 @@ import java.io.{File, FileOutputStream, OutputStreamWriter}
import java.util.Properties
import java.util.concurrent.TimeUnit
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
import com.google.common.base.Charsets.UTF_8
import com.google.common.io.Files
@@ -132,7 +132,7 @@ abstract class BaseYarnClusterSuite
props.setProperty("spark.driver.extraJavaOptions", "-Dfoo=\"one two three\"")
props.setProperty("spark.executor.extraJavaOptions", "-Dfoo=\"one two three\"")
- yarnCluster.getConfig().foreach { e =>
+ yarnCluster.getConfig.asScala.foreach { e =>
props.setProperty("spark.hadoop." + e.getKey(), e.getValue())
}
@@ -149,7 +149,7 @@ abstract class BaseYarnClusterSuite
props.store(writer, "Spark properties.")
writer.close()
- val extraJarArgs = if (!extraJars.isEmpty()) Seq("--jars", extraJars.mkString(",")) else Nil
+ val extraJarArgs = if (extraJars.nonEmpty) Seq("--jars", extraJars.mkString(",")) else Nil
val mainArgs =
if (klass.endsWith(".py")) {
Seq(klass)
diff --git a/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
index 0a5402c89e764521841af48919e4e0357c1103af..e7f2501e7899fcfe3044ae0ef1caf401f3b14c3b 100644
--- a/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
+++ b/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
@@ -20,8 +20,8 @@ package org.apache.spark.deploy.yarn
import java.io.File
import java.net.URI
-import scala.collection.JavaConversions._
-import scala.collection.mutable.{ HashMap => MutableHashMap }
+import scala.collection.JavaConverters._
+import scala.collection.mutable.{HashMap => MutableHashMap}
import scala.reflect.ClassTag
import scala.util.Try
@@ -38,7 +38,7 @@ import org.mockito.Matchers._
import org.mockito.Mockito._
import org.scalatest.{BeforeAndAfterAll, Matchers}
-import org.apache.spark.{SparkConf, SparkException, SparkFunSuite}
+import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.util.Utils
class ClientSuite extends SparkFunSuite with Matchers with BeforeAndAfterAll {
@@ -201,7 +201,7 @@ class ClientSuite extends SparkFunSuite with Matchers with BeforeAndAfterAll {
appContext.getClass.getMethods.filter(_.getName.equals("getApplicationTags")).foreach{ method =>
val tags = method.invoke(appContext).asInstanceOf[java.util.Set[String]]
tags should contain allOf ("tag1", "dup", "tag2", "multi word")
- tags.filter(!_.isEmpty).size should be (4)
+ tags.asScala.filter(_.nonEmpty).size should be (4)
}
appContext.getMaxAppAttempts should be (42)
}
diff --git a/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala b/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala
index 128350b648992f595e18f8f55e6f8c1aabbfe584..5a4ea2ea2f4ff636830fbecd0d15ced2970819a3 100644
--- a/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala
+++ b/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala
@@ -21,7 +21,6 @@ import java.io.File
import java.net.URL
import scala.collection.mutable
-import scala.collection.JavaConversions._
import com.google.common.base.Charsets.UTF_8
import com.google.common.io.{ByteStreams, Files}
@@ -216,8 +215,8 @@ private object YarnClusterDriver extends Logging with Matchers {
assert(listener.driverLogs.nonEmpty)
val driverLogs = listener.driverLogs.get
assert(driverLogs.size === 2)
- assert(driverLogs.containsKey("stderr"))
- assert(driverLogs.containsKey("stdout"))
+ assert(driverLogs.contains("stderr"))
+ assert(driverLogs.contains("stdout"))
val urlStr = driverLogs("stderr")
// Ensure that this is a valid URL, else this will throw an exception
new URL(urlStr)