Add zeroLengthPartitions() test to make sure, e.g., StatCounter.scala can...

Add zeroLengthPartitions() test to make sure, e.g., StatCounter.scala can handle empty partitions without incorrectly returning NaN

core/src/test/scala/spark/JavaAPISuite.java

@@ -314,6 +314,28 @@ public class JavaAPISuite implements Serializable {
     List<Double> take = rdd.take(5);
   }
 
+  @Test
+  public void zeroLengthPartitions() {
+    // Create RDD with some consecutive empty partitions (including the "first" one)
+    JavaDoubleRDD rdd = sc
+        .parallelizeDoubles(Arrays.asList(-1.0, -1.0, -1.0, -1.0, 2.0, 4.0, -1.0, -1.0), 8)
+        .filter(new Function<Double, Boolean>() {
+          @Override
+          public Boolean call(Double x) {
+            return x > 0.0;
+          }
+        });
+    
+    // Run the partitions, including the consecutive empty ones, through StatCounter
+    StatCounter stats = rdd.stats();
+    Assert.assertEquals(6.0, stats.sum(), 0.01);
+    Assert.assertEquals(6.0/2, rdd.mean(), 0.01);
+    Assert.assertEquals(1.0, rdd.variance(), 0.01);
+    Assert.assertEquals(1.0, rdd.stdev(), 0.01);
+    
+    // Add other tests here for classes that should be able to handle empty partitions correctly
+  }
+
   @Test
   public void map() {
     JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5));

project/plugins.sbt

@@ -16,3 +16,5 @@ addSbtPlugin("io.spray" %% "sbt-twirl" % "0.6.1")
 //resolvers += Resolver.url("sbt-plugin-releases", new URL("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases/"))(Resolver.ivyStylePatterns)
 
 //addSbtPlugin("com.jsuereth" % "xsbt-gpg-plugin" % "0.6")
+
+libraryDependencies += "com.novocode" % "junit-interface" % "0.10-M4" % "test"