From 1502a0f6c5d2f85a331b29d3bf50002911ea393e Mon Sep 17 00:00:00 2001
From: jerryshao <sshao@hortonworks.com>
Date: Fri, 28 Aug 2015 09:32:52 -0500
Subject: [PATCH] [YARN] [MINOR] Avoid hard code port number in
YarnShuffleService test
Current port number is fixed as default (7337) in test, this will introduce port contention exception, better to change to a random number in unit test.
squito , seems you're author of this unit test, mind taking a look at this fix? Thanks a lot.
```
[info] - executor state kept across NM restart *** FAILED *** (597 milliseconds)
[info] org.apache.hadoop.service.ServiceStateException: java.net.BindException: Address already in use
[info] at org.apache.hadoop.service.ServiceStateException.convert(ServiceStateException.java:59)
[info] at org.apache.hadoop.service.AbstractService.init(AbstractService.java:172)
[info] at org.apache.spark.network.yarn.YarnShuffleServiceSuite$$anonfun$1.apply$mcV$sp(YarnShuffleServiceSuite.scala:72)
[info] at org.apache.spark.network.yarn.YarnShuffleServiceSuite$$anonfun$1.apply(YarnShuffleServiceSuite.scala:70)
[info] at org.apache.spark.network.yarn.YarnShuffleServiceSuite$$anonfun$1.apply(YarnShuffleServiceSuite.scala:70)
[info] at org.scalatest.Transformer$$anonfun$apply$1.apply$mcV$sp(Transformer.scala:22)
[info] at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85)
[info] at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
[info] at org.scalatest.Transformer.apply(Transformer.scala:22)
[info] at org.scalatest.Transformer.apply(Transformer.scala:20)
[info] at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:166)
[info] at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:42)
...
```
Author: jerryshao <sshao@hortonworks.com>
Closes #8502 from jerryshao/avoid-hardcode-port.
---
.../org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala | 1 +
1 file changed, 1 insertion(+)
diff --git a/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala b/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
index 2f22cbdbea..6aa8c814cd 100644
--- a/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
+++ b/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
@@ -37,6 +37,7 @@ class YarnShuffleServiceSuite extends SparkFunSuite with Matchers with BeforeAnd
yarnConfig.set(YarnConfiguration.NM_AUX_SERVICES, "spark_shuffle")
yarnConfig.set(YarnConfiguration.NM_AUX_SERVICE_FMT.format("spark_shuffle"),
classOf[YarnShuffleService].getCanonicalName)
+ yarnConfig.setInt("spark.shuffle.service.port", 0)
yarnConfig.get("yarn.nodemanager.local-dirs").split(",").foreach { dir =>
val d = new File(dir)
--
GitLab