From 247c529b35d73c5a690bf6512b52999daba1a938 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.co.jp>
Date: Tue, 28 Oct 2014 12:37:09 -0700
Subject: [PATCH] [SPARK-3657] yarn alpha YarnRMClientImpl throws NPE
appMasterRequest.setTrackingUrl starting spark-shell
tgravescs reported this issue.
Following is quoted from tgravescs' report.
YarnRMClientImpl.registerApplicationMaster can throw null pointer exception when setting the trackingurl if its empty:
appMasterRequest.setTrackingUrl(new URI(uiAddress).getAuthority())
I hit this just start spark-shell without the tracking url set.
14/09/23 16:18:34 INFO yarn.YarnRMClientImpl: Connecting to ResourceManager at kryptonitered-jt1.red.ygrid.yahoo.com/98.139.154.99:8030
Exception in thread "main" java.lang.NullPointerException
at org.apache.hadoop.yarn.proto.YarnServiceProtos$RegisterApplicationMasterRequestProto$Builder.setTrackingUrl(YarnServiceProtos.java:710)
at org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterRequestPBImpl.setTrackingUrl(RegisterApplicationMasterRequestPBImpl.java:132)
at org.apache.spark.deploy.yarn.YarnRMClientImpl.registerApplicationMaster(YarnRMClientImpl.scala:102)
at org.apache.spark.deploy.yarn.YarnRMClientImpl.register(YarnRMClientImpl.scala:55)
at org.apache.spark.deploy.yarn.YarnRMClientImpl.register(YarnRMClientImpl.scala:38)
at org.apache.spark.deploy.yarn.ApplicationMaster.registerAM(ApplicationMaster.scala:168)
at org.apache.spark.deploy.yarn.ApplicationMaster.runExecutorLauncher(ApplicationMaster.scala:206)
at org.apache.spark.deploy.yarn.ApplicationMaster.run(ApplicationMaster.scala:120)
Author: Kousuke Saruta <sarutak@oss.nttdata.co.jp>
Closes #2981 from sarutak/SPARK-3657-2 and squashes the following commits:
e2fd6bc [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into SPARK-3657
70b8882 [Kousuke Saruta] Fixed NPE thrown
---
.../org/apache/spark/deploy/yarn/YarnRMClientImpl.scala | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala
index 7faf55bc63..e342cc82f4 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnRMClientImpl.scala
@@ -18,7 +18,7 @@
package org.apache.spark.deploy.yarn
import scala.collection.{Map, Set}
-import java.net.URI;
+import java.net.URI
import org.apache.hadoop.net.NetUtils
import org.apache.hadoop.yarn.api._
@@ -109,7 +109,9 @@ private class YarnRMClientImpl(args: ApplicationMasterArguments) extends YarnRMC
appMasterRequest.setHost(Utils.localHostName())
appMasterRequest.setRpcPort(0)
// remove the scheme from the url if it exists since Hadoop does not expect scheme
- appMasterRequest.setTrackingUrl(new URI(uiAddress).getAuthority())
+ val uri = new URI(uiAddress)
+ val authority = if (uri.getScheme == null) uiAddress else uri.getAuthority
+ appMasterRequest.setTrackingUrl(authority)
resourceManager.registerApplicationMaster(appMasterRequest)
}
--
GitLab