From 446bb3417a2855a194d49acc0ac316a021eced9d Mon Sep 17 00:00:00 2001
From: Thomas Graves <tgraves@apache.org>
Date: Fri, 11 Apr 2014 13:17:48 +0530
Subject: [PATCH] SPARK-1417: Spark on Yarn - spark UI link from
resourcemanager is broken
Author: Thomas Graves <tgraves@apache.org>
Closes #344 from tgravescs/SPARK-1417 and squashes the following commits:
c450b5f [Thomas Graves] fix test
e1c1d7e [Thomas Graves] add missing $ to appUIAddress
e982ddb [Thomas Graves] use appUIHostPort in appUIAddress
0803ec2 [Thomas Graves] Review comment updates - remove extra newline, simplify assert in test
658a8ec [Thomas Graves] Add a appUIHostPort routine
0614208 [Thomas Graves] Fix test
2a6b1b7 [Thomas Graves] SPARK-1417: Spark on Yarn - spark UI link from resourcemanager is broken
---
.../scala/org/apache/spark/ui/SparkUI.scala | 7 +++-
.../scala/org/apache/spark/SparkUISuite.scala | 35 +++++++++++++++++++
.../spark/deploy/yarn/ApplicationMaster.scala | 2 +-
.../spark/deploy/yarn/ApplicationMaster.scala | 2 +-
4 files changed, 43 insertions(+), 3 deletions(-)
create mode 100644 core/src/test/scala/org/apache/spark/SparkUISuite.scala
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index 4c891d73af..7fa4fd3149 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -113,7 +113,12 @@ private[spark] class SparkUI(
logInfo("Stopped Spark Web UI at %s".format(appUIAddress))
}
- private[spark] def appUIAddress = "http://" + publicHost + ":" + boundPort
+ /**
+ * Return the application UI host:port. This does not include the scheme (http://).
+ */
+ private[spark] def appUIHostPort = publicHost + ":" + boundPort
+
+ private[spark] def appUIAddress = s"http://$appUIHostPort"
}
diff --git a/core/src/test/scala/org/apache/spark/SparkUISuite.scala b/core/src/test/scala/org/apache/spark/SparkUISuite.scala
new file mode 100644
index 0000000000..d0d119c150
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/SparkUISuite.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import java.net.URI
+
+import org.scalatest.FunSuite
+
+class SparkUISuite extends FunSuite with SharedSparkContext {
+
+ test("verify appUIAddress contains the scheme") {
+ val uiAddress = sc.ui.appUIAddress
+ assert(uiAddress.equals("http://" + sc.ui.appUIHostPort))
+ }
+
+ test("verify appUIAddress contains the port") {
+ val splitUIAddress = sc.ui.appUIAddress.split(':')
+ assert(splitUIAddress(2).toInt == sc.ui.boundPort)
+ }
+}
diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 910484ed54..67ec95c8fc 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -234,7 +234,7 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration,
assert(sparkContext != null || count >= numTries)
if (null != sparkContext) {
- uiAddress = sparkContext.ui.appUIAddress
+ uiAddress = sparkContext.ui.appUIHostPort
this.yarnAllocator = YarnAllocationHandler.newAllocator(
yarnConf,
resourceManager,
diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index c8a4d2e647..61af0f9ac5 100644
--- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -220,7 +220,7 @@ class ApplicationMaster(args: ApplicationMasterArguments, conf: Configuration,
assert(sparkContext != null || numTries >= maxNumTries)
if (sparkContext != null) {
- uiAddress = sparkContext.ui.appUIAddress
+ uiAddress = sparkContext.ui.appUIHostPort
this.yarnAllocator = YarnAllocationHandler.newAllocator(
yarnConf,
amClient,
--
GitLab