From 9f97f2f9d86f542e818e2991588be958c1ffede1 Mon Sep 17 00:00:00 2001
From: Imran Rashid <imran@quantifind.com>
Date: Sun, 10 Mar 2013 22:30:18 -0700
Subject: [PATCH] add a small wait to one task to make sure some task runtime
really is non-zero
---
.../scala/spark/scheduler/SparkListenerSuite.scala | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/core/src/test/scala/spark/scheduler/SparkListenerSuite.scala b/core/src/test/scala/spark/scheduler/SparkListenerSuite.scala
index dd9f2d7e91..a45a5efbb4 100644
--- a/core/src/test/scala/spark/scheduler/SparkListenerSuite.scala
+++ b/core/src/test/scala/spark/scheduler/SparkListenerSuite.scala
@@ -17,16 +17,22 @@ class SparkListenerSuite extends FunSuite with LocalSparkContext with ShouldMatc
val listener = new SaveStageInfo
sc.addSparkListener(listener)
sc.addSparkListener(new StatsReportListener)
+ //just to make sure some of the tasks take a noticeable amount of time
+ val w = {i:Int =>
+ if (i == 0)
+ Thread.sleep(25)
+ i
+ }
- val d = sc.parallelize(1 to 1e4.toInt, 64)
+ val d = sc.parallelize(1 to 1e4.toInt, 64).map{i => w(i)}
d.count
listener.stageInfos.size should be (1)
- val d2 = d.map{i => i -> i * 2}.setName("shuffle input 1")
+ val d2 = d.map{i => w(i) -> i * 2}.setName("shuffle input 1")
- val d3 = d.map{i => i -> (0 to (i % 5))}.setName("shuffle input 2")
+ val d3 = d.map{i => w(i) -> (0 to (i % 5))}.setName("shuffle input 2")
- val d4 = d2.cogroup(d3, 64).map{case(k,(v1,v2)) => k -> (v1.size, v2.size)}
+ val d4 = d2.cogroup(d3, 64).map{case(k,(v1,v2)) => w(k) -> (v1.size, v2.size)}
d4.setName("A Cogroup")
d4.collectAsMap
--
GitLab