From 29ac7ea52fbb0c6531e14305e2fb1ccba9678f7e Mon Sep 17 00:00:00 2001
From: Punya Biswal <pbiswal@palantir.com>
Date: Sat, 22 Feb 2014 17:53:48 -0800
Subject: [PATCH] Migrate Java code to Scala or move it to src/main/java
These classes can't be migrated:
StorageLevels: impossible to create static fields in Scala
JavaSparkContextVarargsWorkaround: incompatible varargs
JavaAPISuite: should test Java APIs in pure Java (for sanity)
Author: Punya Biswal <pbiswal@palantir.com>
Closes #605 from punya/move-java-sources and squashes the following commits:
25b00b2 [Punya Biswal] Remove redundant type param; reformat
853da46 [Punya Biswal] Use factory method rather than constructor
e5d53d9 [Punya Biswal] Migrate Java code to Scala or move it to src/main/java
---
.../JavaSparkContextVarargsWorkaround.java | 0
.../apache/spark/api/java/StorageLevels.java | 22 +++++++++---------
.../{SparkFiles.java => SparkFiles.scala} | 19 +++++++--------
...ction.java => DoubleFlatMapFunction.scala} | 10 ++++----
...ubleFunction.java => DoubleFunction.scala} | 7 +++---
.../{Function.java => Function.scala} | 14 ++++-------
.../{Function2.java => Function2.scala} | 16 ++++---------
.../{Function3.java => Function3.scala} | 17 ++++----------
...unction.java => PairFlatMapFunction.scala} | 23 +++++++------------
.../{PairFunction.java => PairFunction.scala} | 20 +++++-----------
.../org/apache/spark/JavaAPISuite.java | 0
11 files changed, 56 insertions(+), 92 deletions(-)
rename core/src/main/{scala => java}/org/apache/spark/api/java/JavaSparkContextVarargsWorkaround.java (100%)
rename core/src/main/{scala => java}/org/apache/spark/api/java/StorageLevels.java (58%)
rename core/src/main/scala/org/apache/spark/{SparkFiles.java => SparkFiles.scala} (77%)
rename core/src/main/scala/org/apache/spark/api/java/function/{DoubleFlatMapFunction.java => DoubleFlatMapFunction.scala} (80%)
rename core/src/main/scala/org/apache/spark/api/java/function/{DoubleFunction.java => DoubleFunction.scala} (85%)
rename core/src/main/scala/org/apache/spark/api/java/function/{Function.java => Function.scala} (76%)
rename core/src/main/scala/org/apache/spark/api/java/function/{Function2.java => Function2.scala} (72%)
rename core/src/main/scala/org/apache/spark/api/java/function/{Function3.java => Function3.scala} (70%)
rename core/src/main/scala/org/apache/spark/api/java/function/{PairFlatMapFunction.java => PairFlatMapFunction.scala} (69%)
rename core/src/main/scala/org/apache/spark/api/java/function/{PairFunction.java => PairFunction.scala} (68%)
rename core/src/test/{scala => java}/org/apache/spark/JavaAPISuite.java (100%)
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContextVarargsWorkaround.java b/core/src/main/java/org/apache/spark/api/java/JavaSparkContextVarargsWorkaround.java
similarity index 100%
rename from core/src/main/scala/org/apache/spark/api/java/JavaSparkContextVarargsWorkaround.java
rename to core/src/main/java/org/apache/spark/api/java/JavaSparkContextVarargsWorkaround.java
diff --git a/core/src/main/scala/org/apache/spark/api/java/StorageLevels.java b/core/src/main/java/org/apache/spark/api/java/StorageLevels.java
similarity index 58%
rename from core/src/main/scala/org/apache/spark/api/java/StorageLevels.java
rename to core/src/main/java/org/apache/spark/api/java/StorageLevels.java
index 0744269773..9f13b39909 100644
--- a/core/src/main/scala/org/apache/spark/api/java/StorageLevels.java
+++ b/core/src/main/java/org/apache/spark/api/java/StorageLevels.java
@@ -23,17 +23,17 @@ import org.apache.spark.storage.StorageLevel;
* Expose some commonly useful storage level constants.
*/
public class StorageLevels {
- public static final StorageLevel NONE = new StorageLevel(false, false, false, 1);
- public static final StorageLevel DISK_ONLY = new StorageLevel(true, false, false, 1);
- public static final StorageLevel DISK_ONLY_2 = new StorageLevel(true, false, false, 2);
- public static final StorageLevel MEMORY_ONLY = new StorageLevel(false, true, true, 1);
- public static final StorageLevel MEMORY_ONLY_2 = new StorageLevel(false, true, true, 2);
- public static final StorageLevel MEMORY_ONLY_SER = new StorageLevel(false, true, false, 1);
- public static final StorageLevel MEMORY_ONLY_SER_2 = new StorageLevel(false, true, false, 2);
- public static final StorageLevel MEMORY_AND_DISK = new StorageLevel(true, true, true, 1);
- public static final StorageLevel MEMORY_AND_DISK_2 = new StorageLevel(true, true, true, 2);
- public static final StorageLevel MEMORY_AND_DISK_SER = new StorageLevel(true, true, false, 1);
- public static final StorageLevel MEMORY_AND_DISK_SER_2 = new StorageLevel(true, true, false, 2);
+ public static final StorageLevel NONE = create(false, false, false, 1);
+ public static final StorageLevel DISK_ONLY = create(true, false, false, 1);
+ public static final StorageLevel DISK_ONLY_2 = create(true, false, false, 2);
+ public static final StorageLevel MEMORY_ONLY = create(false, true, true, 1);
+ public static final StorageLevel MEMORY_ONLY_2 = create(false, true, true, 2);
+ public static final StorageLevel MEMORY_ONLY_SER = create(false, true, false, 1);
+ public static final StorageLevel MEMORY_ONLY_SER_2 = create(false, true, false, 2);
+ public static final StorageLevel MEMORY_AND_DISK = create(true, true, true, 1);
+ public static final StorageLevel MEMORY_AND_DISK_2 = create(true, true, true, 2);
+ public static final StorageLevel MEMORY_AND_DISK_SER = create(true, true, false, 1);
+ public static final StorageLevel MEMORY_AND_DISK_SER_2 = create(true, true, false, 2);
/**
* Create a new StorageLevel object.
diff --git a/core/src/main/scala/org/apache/spark/SparkFiles.java b/core/src/main/scala/org/apache/spark/SparkFiles.scala
similarity index 77%
rename from core/src/main/scala/org/apache/spark/SparkFiles.java
rename to core/src/main/scala/org/apache/spark/SparkFiles.scala
index af9cf85e37..e85b89fd01 100644
--- a/core/src/main/scala/org/apache/spark/SparkFiles.java
+++ b/core/src/main/scala/org/apache/spark/SparkFiles.scala
@@ -15,28 +15,25 @@
* limitations under the License.
*/
-package org.apache.spark;
+package org.apache.spark
-import java.io.File;
+import java.io.File
/**
* Resolves paths to files added through `SparkContext.addFile()`.
*/
-public class SparkFiles {
-
- private SparkFiles() {}
+object SparkFiles {
/**
* Get the absolute path of a file added through `SparkContext.addFile()`.
*/
- public static String get(String filename) {
- return new File(getRootDirectory(), filename).getAbsolutePath();
- }
+ def get(filename: String): String =
+ new File(getRootDirectory(), filename).getAbsolutePath()
/**
* Get the root directory that contains files added through `SparkContext.addFile()`.
*/
- public static String getRootDirectory() {
- return SparkEnv.get().sparkFilesDir();
- }
+ def getRootDirectory(): String =
+ SparkEnv.get.sparkFilesDir
+
}
diff --git a/core/src/main/scala/org/apache/spark/api/java/function/DoubleFlatMapFunction.java b/core/src/main/scala/org/apache/spark/api/java/function/DoubleFlatMapFunction.scala
similarity index 80%
rename from core/src/main/scala/org/apache/spark/api/java/function/DoubleFlatMapFunction.java
rename to core/src/main/scala/org/apache/spark/api/java/function/DoubleFlatMapFunction.scala
index 30e6a52474..7500a89436 100644
--- a/core/src/main/scala/org/apache/spark/api/java/function/DoubleFlatMapFunction.java
+++ b/core/src/main/scala/org/apache/spark/api/java/function/DoubleFlatMapFunction.scala
@@ -15,16 +15,16 @@
* limitations under the License.
*/
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
-import java.io.Serializable;
+import java.lang.{Double => JDouble, Iterable => JIterable}
/**
* A function that returns zero or more records of type Double from each input record.
*/
// DoubleFlatMapFunction does not extend FlatMapFunction because flatMap is
// overloaded for both FlatMapFunction and DoubleFlatMapFunction.
-public abstract class DoubleFlatMapFunction<T> extends WrappedFunction1<T, Iterable<Double>>
- implements Serializable {
- // Intentionally left blank
+abstract class DoubleFlatMapFunction[T] extends WrappedFunction1[T, JIterable[JDouble]]
+ with Serializable {
+ // Intentionally left blank
}
diff --git a/core/src/main/scala/org/apache/spark/api/java/function/DoubleFunction.java b/core/src/main/scala/org/apache/spark/api/java/function/DoubleFunction.scala
similarity index 85%
rename from core/src/main/scala/org/apache/spark/api/java/function/DoubleFunction.java
rename to core/src/main/scala/org/apache/spark/api/java/function/DoubleFunction.scala
index 490da255bc..2cdf2e92c3 100644
--- a/core/src/main/scala/org/apache/spark/api/java/function/DoubleFunction.java
+++ b/core/src/main/scala/org/apache/spark/api/java/function/DoubleFunction.scala
@@ -15,16 +15,15 @@
* limitations under the License.
*/
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
-import java.io.Serializable;
+import java.lang.{Double => JDouble}
/**
* A function that returns Doubles, and can be used to construct DoubleRDDs.
*/
// DoubleFunction does not extend Function because some UDF functions, like map,
// are overloaded for both Function and DoubleFunction.
-public abstract class DoubleFunction<T> extends WrappedFunction1<T, Double>
- implements Serializable {
+abstract class DoubleFunction[T] extends WrappedFunction1[T, JDouble] with Serializable {
// Intentionally left blank
}
diff --git a/core/src/main/scala/org/apache/spark/api/java/function/Function.java b/core/src/main/scala/org/apache/spark/api/java/function/Function.scala
similarity index 76%
rename from core/src/main/scala/org/apache/spark/api/java/function/Function.java
rename to core/src/main/scala/org/apache/spark/api/java/function/Function.scala
index e0fcd460c8..a5e1701f77 100644
--- a/core/src/main/scala/org/apache/spark/api/java/function/Function.java
+++ b/core/src/main/scala/org/apache/spark/api/java/function/Function.scala
@@ -15,21 +15,17 @@
* limitations under the License.
*/
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
-import java.io.Serializable;
-
-import scala.reflect.ClassTag;
-import scala.reflect.ClassTag$;
+import scala.reflect.ClassTag
+import org.apache.spark.api.java.JavaSparkContext
/**
* Base class for functions whose return types do not create special RDDs. PairFunction and
* DoubleFunction are handled separately, to allow PairRDDs and DoubleRDDs to be constructed
* when mapping RDDs of other types.
*/
-public abstract class Function<T, R> extends WrappedFunction1<T, R> implements Serializable {
- public ClassTag<R> returnType() {
- return ClassTag$.MODULE$.apply(Object.class);
- }
+abstract class Function[T, R] extends WrappedFunction1[T, R] with Serializable {
+ def returnType(): ClassTag[R] = JavaSparkContext.fakeClassTag
}
diff --git a/core/src/main/scala/org/apache/spark/api/java/function/Function2.java b/core/src/main/scala/org/apache/spark/api/java/function/Function2.scala
similarity index 72%
rename from core/src/main/scala/org/apache/spark/api/java/function/Function2.java
rename to core/src/main/scala/org/apache/spark/api/java/function/Function2.scala
index 16d7379462..fa3616cbcb 100644
--- a/core/src/main/scala/org/apache/spark/api/java/function/Function2.java
+++ b/core/src/main/scala/org/apache/spark/api/java/function/Function2.scala
@@ -15,21 +15,15 @@
* limitations under the License.
*/
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
-import java.io.Serializable;
-
-import scala.reflect.ClassTag;
-import scala.reflect.ClassTag$;
+import scala.reflect.ClassTag
+import org.apache.spark.api.java.JavaSparkContext
/**
* A two-argument function that takes arguments of type T1 and T2 and returns an R.
*/
-public abstract class Function2<T1, T2, R> extends WrappedFunction2<T1, T2, R>
- implements Serializable {
-
- public ClassTag<R> returnType() {
- return (ClassTag<R>) ClassTag$.MODULE$.apply(Object.class);
- }
+abstract class Function2[T1, T2, R] extends WrappedFunction2[T1, T2, R] with Serializable {
+ def returnType(): ClassTag[R] = JavaSparkContext.fakeClassTag
}
diff --git a/core/src/main/scala/org/apache/spark/api/java/function/Function3.java b/core/src/main/scala/org/apache/spark/api/java/function/Function3.scala
similarity index 70%
rename from core/src/main/scala/org/apache/spark/api/java/function/Function3.java
rename to core/src/main/scala/org/apache/spark/api/java/function/Function3.scala
index 096eb71f95..45152891e9 100644
--- a/core/src/main/scala/org/apache/spark/api/java/function/Function3.java
+++ b/core/src/main/scala/org/apache/spark/api/java/function/Function3.scala
@@ -15,21 +15,14 @@
* limitations under the License.
*/
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
-import java.io.Serializable;
-
-import scala.reflect.ClassTag;
-import scala.reflect.ClassTag$;
+import org.apache.spark.api.java.JavaSparkContext
+import scala.reflect.ClassTag
/**
* A three-argument function that takes arguments of type T1, T2 and T3 and returns an R.
*/
-public abstract class Function3<T1, T2, T3, R> extends WrappedFunction3<T1, T2, T3, R>
- implements Serializable {
-
- public ClassTag<R> returnType() {
- return (ClassTag<R>) ClassTag$.MODULE$.apply(Object.class);
- }
+abstract class Function3[T1, T2, T3, R] extends WrappedFunction3[T1, T2, T3, R] with Serializable {
+ def returnType(): ClassTag[R] = JavaSparkContext.fakeClassTag
}
-
diff --git a/core/src/main/scala/org/apache/spark/api/java/function/PairFlatMapFunction.java b/core/src/main/scala/org/apache/spark/api/java/function/PairFlatMapFunction.scala
similarity index 69%
rename from core/src/main/scala/org/apache/spark/api/java/function/PairFlatMapFunction.java
rename to core/src/main/scala/org/apache/spark/api/java/function/PairFlatMapFunction.scala
index c72b98c28a..8467bbb892 100644
--- a/core/src/main/scala/org/apache/spark/api/java/function/PairFlatMapFunction.java
+++ b/core/src/main/scala/org/apache/spark/api/java/function/PairFlatMapFunction.scala
@@ -15,13 +15,11 @@
* limitations under the License.
*/
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
-import java.io.Serializable;
-
-import scala.Tuple2;
-import scala.reflect.ClassTag;
-import scala.reflect.ClassTag$;
+import java.lang.{Iterable => JIterable}
+import org.apache.spark.api.java.JavaSparkContext
+import scala.reflect.ClassTag
/**
* A function that returns zero or more key-value pair records from each input record. The
@@ -29,15 +27,10 @@ import scala.reflect.ClassTag$;
*/
// PairFlatMapFunction does not extend FlatMapFunction because flatMap is
// overloaded for both FlatMapFunction and PairFlatMapFunction.
-public abstract class PairFlatMapFunction<T, K, V>
- extends WrappedFunction1<T, Iterable<Tuple2<K, V>>>
- implements Serializable {
+abstract class PairFlatMapFunction[T, K, V] extends WrappedFunction1[T, JIterable[(K, V)]]
+ with Serializable {
- public ClassTag<K> keyType() {
- return (ClassTag<K>) ClassTag$.MODULE$.apply(Object.class);
- }
+ def keyType(): ClassTag[K] = JavaSparkContext.fakeClassTag
- public ClassTag<V> valueType() {
- return (ClassTag<V>) ClassTag$.MODULE$.apply(Object.class);
- }
+ def valueType(): ClassTag[V] = JavaSparkContext.fakeClassTag
}
diff --git a/core/src/main/scala/org/apache/spark/api/java/function/PairFunction.java b/core/src/main/scala/org/apache/spark/api/java/function/PairFunction.scala
similarity index 68%
rename from core/src/main/scala/org/apache/spark/api/java/function/PairFunction.java
rename to core/src/main/scala/org/apache/spark/api/java/function/PairFunction.scala
index 84b9136d98..d0ba0b6307 100644
--- a/core/src/main/scala/org/apache/spark/api/java/function/PairFunction.java
+++ b/core/src/main/scala/org/apache/spark/api/java/function/PairFunction.scala
@@ -15,27 +15,19 @@
* limitations under the License.
*/
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
-import java.io.Serializable;
-
-import scala.Tuple2;
-import scala.reflect.ClassTag;
-import scala.reflect.ClassTag$;
+import scala.reflect.ClassTag
+import org.apache.spark.api.java.JavaSparkContext
/**
* A function that returns key-value pairs (Tuple2<K, V>), and can be used to construct PairRDDs.
*/
// PairFunction does not extend Function because some UDF functions, like map,
// are overloaded for both Function and PairFunction.
-public abstract class PairFunction<T, K, V> extends WrappedFunction1<T, Tuple2<K, V>>
- implements Serializable {
+abstract class PairFunction[T, K, V] extends WrappedFunction1[T, (K, V)] with Serializable {
- public ClassTag<K> keyType() {
- return (ClassTag<K>) ClassTag$.MODULE$.apply(Object.class);
- }
+ def keyType(): ClassTag[K] = JavaSparkContext.fakeClassTag
- public ClassTag<V> valueType() {
- return (ClassTag<V>) ClassTag$.MODULE$.apply(Object.class);
- }
+ def valueType(): ClassTag[V] = JavaSparkContext.fakeClassTag
}
diff --git a/core/src/test/scala/org/apache/spark/JavaAPISuite.java b/core/src/test/java/org/apache/spark/JavaAPISuite.java
similarity index 100%
rename from core/src/test/scala/org/apache/spark/JavaAPISuite.java
rename to core/src/test/java/org/apache/spark/JavaAPISuite.java
--
GitLab