From e5c307c50a660f706799f1f7f6890bcec888d96b Mon Sep 17 00:00:00 2001
From: Weiqing Yang <yangweiqing001@gmail.com>
Date: Tue, 3 Jan 2017 09:56:42 +0000
Subject: [PATCH] [MINOR] Add missing sc.stop() to end of examples
## What changes were proposed in this pull request?
Add `finally` clause for `sc.stop()` in the `test("register and deregister Spark listener from SparkContext")`.
## How was this patch tested?
Pass the build and unit tests.
Author: Weiqing Yang <yangweiqing001@gmail.com>
Closes #16426 from weiqingy/testIssue.
---
.../org/apache/spark/SparkContextSuite.scala | 20 +++++++++----------
...avaBinaryClassificationMetricsExample.java | 2 ++
.../examples/mllib/JavaLBFGSExample.java | 2 ++
...ulticlassClassificationMetricsExample.java | 2 ++
.../spark/examples/mllib/JavaPCAExample.java | 1 +
.../mllib/AssociationRulesExample.scala | 2 ++
.../BinaryClassificationMetricsExample.scala | 1 +
.../DecisionTreeClassificationExample.scala | 2 ++
.../mllib/DecisionTreeRegressionExample.scala | 2 ++
...radientBoostingClassificationExample.scala | 2 ++
.../GradientBoostingRegressionExample.scala | 2 ++
.../mllib/IsotonicRegressionExample.scala | 2 ++
.../spark/examples/mllib/LBFGSExample.scala | 2 ++
.../mllib/MultiLabelMetricsExample.scala | 2 ++
.../mllib/MulticlassMetricsExample.scala | 2 ++
.../examples/mllib/NaiveBayesExample.scala | 2 ++
.../mllib/PCAOnRowMatrixExample.scala | 2 ++
.../mllib/PCAOnSourceVectorExample.scala | 2 ++
.../examples/mllib/PrefixSpanExample.scala | 2 ++
.../RandomForestClassificationExample.scala | 2 ++
.../mllib/RandomForestRegressionExample.scala | 2 ++
.../mllib/RecommendationExample.scala | 2 ++
.../spark/examples/mllib/SVDExample.scala | 2 ++
.../spark/examples/mllib/SimpleFPGrowth.scala | 2 ++
.../mllib/util/KMeansDataGenerator.scala | 1 +
.../cluster/YarnSchedulerBackendSuite.scala | 1 +
26 files changed, 55 insertions(+), 11 deletions(-)
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index 8fba82de54..8ae5d2fe96 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -455,16 +455,14 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext {
test("register and deregister Spark listener from SparkContext") {
sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
- try {
- val sparkListener1 = new SparkListener { }
- val sparkListener2 = new SparkListener { }
- sc.addSparkListener(sparkListener1)
- sc.addSparkListener(sparkListener2)
- assert(sc.listenerBus.listeners.contains(sparkListener1))
- assert(sc.listenerBus.listeners.contains(sparkListener2))
- sc.removeSparkListener(sparkListener1)
- assert(!sc.listenerBus.listeners.contains(sparkListener1))
- assert(sc.listenerBus.listeners.contains(sparkListener2))
- }
+ val sparkListener1 = new SparkListener { }
+ val sparkListener2 = new SparkListener { }
+ sc.addSparkListener(sparkListener1)
+ sc.addSparkListener(sparkListener2)
+ assert(sc.listenerBus.listeners.contains(sparkListener1))
+ assert(sc.listenerBus.listeners.contains(sparkListener2))
+ sc.removeSparkListener(sparkListener1)
+ assert(!sc.listenerBus.listeners.contains(sparkListener1))
+ assert(sc.listenerBus.listeners.contains(sparkListener2))
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassificationMetricsExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassificationMetricsExample.java
index 7561a1f653..12aa14f710 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassificationMetricsExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassificationMetricsExample.java
@@ -111,5 +111,7 @@ public class JavaBinaryClassificationMetricsExample {
model.save(sc, "target/tmp/LogisticRegressionModel");
LogisticRegressionModel.load(sc, "target/tmp/LogisticRegressionModel");
// $example off$
+
+ sc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaLBFGSExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaLBFGSExample.java
index 355883f61b..f6f91f486f 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaLBFGSExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaLBFGSExample.java
@@ -103,6 +103,8 @@ public class JavaLBFGSExample {
System.out.println(l);
System.out.println("Area under ROC = " + auROC);
// $example off$
+
+ sc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaMulticlassClassificationMetricsExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaMulticlassClassificationMetricsExample.java
index e84a3a712d..2d12bdd2a6 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaMulticlassClassificationMetricsExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaMulticlassClassificationMetricsExample.java
@@ -91,5 +91,7 @@ public class JavaMulticlassClassificationMetricsExample {
LogisticRegressionModel sameModel = LogisticRegressionModel.load(sc,
"target/tmp/LogisticRegressionModel");
// $example off$
+
+ sc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaPCAExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaPCAExample.java
index a42c29f52f..3077f557ef 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaPCAExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaPCAExample.java
@@ -61,5 +61,6 @@ public class JavaPCAExample {
for (Vector vector : collectPartitions) {
System.out.println("\t" + vector);
}
+ sc.stop();
}
}
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/AssociationRulesExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/AssociationRulesExample.scala
index 11e18c9f04..ff44de5683 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/AssociationRulesExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/AssociationRulesExample.scala
@@ -47,6 +47,8 @@ object AssociationRulesExample {
+ rule.consequent.mkString(",") + "]," + rule.confidence)
}
// $example off$
+
+ sc.stop()
}
}
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
index ade33fc509..b9263ac6fc 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
@@ -98,6 +98,7 @@ object BinaryClassificationMetricsExample {
val auROC = metrics.areaUnderROC
println("Area under ROC = " + auROC)
// $example off$
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeClassificationExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeClassificationExample.scala
index c6c7c6f5e2..b50b459277 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeClassificationExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeClassificationExample.scala
@@ -62,6 +62,8 @@ object DecisionTreeClassificationExample {
model.save(sc, "target/tmp/myDecisionTreeClassificationModel")
val sameModel = DecisionTreeModel.load(sc, "target/tmp/myDecisionTreeClassificationModel")
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRegressionExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRegressionExample.scala
index 9c8baed3b8..2af45afae3 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRegressionExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRegressionExample.scala
@@ -61,6 +61,8 @@ object DecisionTreeRegressionExample {
model.save(sc, "target/tmp/myDecisionTreeRegressionModel")
val sameModel = DecisionTreeModel.load(sc, "target/tmp/myDecisionTreeRegressionModel")
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingClassificationExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingClassificationExample.scala
index 0ec2e11214..00bb3348d2 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingClassificationExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingClassificationExample.scala
@@ -62,6 +62,8 @@ object GradientBoostingClassificationExample {
val sameModel = GradientBoostedTreesModel.load(sc,
"target/tmp/myGradientBoostingClassificationModel")
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingRegressionExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingRegressionExample.scala
index b87ba0defe..d8c2634608 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingRegressionExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingRegressionExample.scala
@@ -61,6 +61,8 @@ object GradientBoostingRegressionExample {
val sameModel = GradientBoostedTreesModel.load(sc,
"target/tmp/myGradientBoostingRegressionModel")
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/IsotonicRegressionExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/IsotonicRegressionExample.scala
index e5dea129c1..4aee951f5b 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/IsotonicRegressionExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/IsotonicRegressionExample.scala
@@ -62,6 +62,8 @@ object IsotonicRegressionExample {
model.save(sc, "target/tmp/myIsotonicRegressionModel")
val sameModel = IsotonicRegressionModel.load(sc, "target/tmp/myIsotonicRegressionModel")
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/LBFGSExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/LBFGSExample.scala
index 75a0419da5..fedcefa098 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/LBFGSExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/LBFGSExample.scala
@@ -84,6 +84,8 @@ object LBFGSExample {
loss.foreach(println)
println("Area under ROC = " + auROC)
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/MultiLabelMetricsExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/MultiLabelMetricsExample.scala
index c0d447bf69..ebab81b334 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/MultiLabelMetricsExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/MultiLabelMetricsExample.scala
@@ -64,6 +64,8 @@ object MultiLabelMetricsExample {
// Subset accuracy
println(s"Subset accuracy = ${metrics.subsetAccuracy}")
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/MulticlassMetricsExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/MulticlassMetricsExample.scala
index 12394c867e..e0b98eeb44 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/MulticlassMetricsExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/MulticlassMetricsExample.scala
@@ -90,6 +90,8 @@ object MulticlassMetricsExample {
println(s"Weighted F1 score: ${metrics.weightedFMeasure}")
println(s"Weighted false positive rate: ${metrics.weightedFalsePositiveRate}")
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/NaiveBayesExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/NaiveBayesExample.scala
index b321d8e127..24c8e3445e 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/NaiveBayesExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/NaiveBayesExample.scala
@@ -45,6 +45,8 @@ object NaiveBayesExample {
model.save(sc, "target/tmp/myNaiveBayesModel")
val sameModel = NaiveBayesModel.load(sc, "target/tmp/myNaiveBayesModel")
// $example off$
+
+ sc.stop()
}
}
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnRowMatrixExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnRowMatrixExample.scala
index 234de230eb..a137ba2a2f 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnRowMatrixExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnRowMatrixExample.scala
@@ -53,6 +53,8 @@ object PCAOnRowMatrixExample {
val collect = projected.rows.collect()
println("Projected Row Matrix of principal component:")
collect.foreach { vector => println(vector) }
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnSourceVectorExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnSourceVectorExample.scala
index f7694879df..cef5402581 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnSourceVectorExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnSourceVectorExample.scala
@@ -52,6 +52,8 @@ object PCAOnSourceVectorExample {
val collect = projected.collect()
println("Projected vector of principal component:")
collect.foreach { vector => println(vector) }
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/PrefixSpanExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/PrefixSpanExample.scala
index ef86eab9e4..69c72c4336 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/PrefixSpanExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/PrefixSpanExample.scala
@@ -46,6 +46,8 @@ object PrefixSpanExample {
", " + freqSequence.freq)
}
// $example off$
+
+ sc.stop()
}
}
// scalastyle:off println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestClassificationExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestClassificationExample.scala
index 7805153ba7..f1ebdf1a73 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestClassificationExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestClassificationExample.scala
@@ -62,6 +62,8 @@ object RandomForestClassificationExample {
model.save(sc, "target/tmp/myRandomForestClassificationModel")
val sameModel = RandomForestModel.load(sc, "target/tmp/myRandomForestClassificationModel")
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestRegressionExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestRegressionExample.scala
index 655a277e28..11d612e651 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestRegressionExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestRegressionExample.scala
@@ -62,6 +62,8 @@ object RandomForestRegressionExample {
model.save(sc, "target/tmp/myRandomForestRegressionModel")
val sameModel = RandomForestModel.load(sc, "target/tmp/myRandomForestRegressionModel")
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/RecommendationExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/RecommendationExample.scala
index bc946951ae..6df742d737 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/RecommendationExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/RecommendationExample.scala
@@ -62,6 +62,8 @@ object RecommendationExample {
model.save(sc, "target/tmp/myCollaborativeFilter")
val sameModel = MatrixFactorizationModel.load(sc, "target/tmp/myCollaborativeFilter")
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/SVDExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/SVDExample.scala
index c26580d4c1..b286a3f7b9 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/SVDExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/SVDExample.scala
@@ -56,6 +56,8 @@ object SVDExample {
collect.foreach { vector => println(vector) }
println(s"Singular values are: $s")
println(s"V factor is:\n$V")
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/SimpleFPGrowth.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/SimpleFPGrowth.scala
index ab15ac2c54..b5c3033bcb 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/SimpleFPGrowth.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/SimpleFPGrowth.scala
@@ -53,6 +53,8 @@ object SimpleFPGrowth {
+ ", " + rule.confidence)
}
// $example off$
+
+ sc.stop()
}
}
// scalastyle:on println
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
index 00fd1606a3..7f84be9f37 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
@@ -86,6 +86,7 @@ object KMeansDataGenerator {
val data = generateKMeansRDD(sc, numPoints, k, d, r, parts)
data.map(_.mkString(" ")).saveAsTextFile(outputPath)
+ sc.stop()
System.exit(0)
}
}
diff --git a/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala b/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala
index ffa0b58ee7..4079d9e40f 100644
--- a/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala
+++ b/yarn/src/test/scala/org/apache/spark/scheduler/cluster/YarnSchedulerBackendSuite.scala
@@ -52,6 +52,7 @@ class YarnSchedulerBackendSuite extends SparkFunSuite with MockitoSugar with Loc
// Serialize to make sure serialization doesn't throw an error
ser.serialize(req)
}
+ sc.stop()
}
}
--
GitLab