From 950645d597dbc5a8c5010bcb1a9b51c6abad86ea Mon Sep 17 00:00:00 2001
From: Dean Chen <deanchen5@gmail.com>
Date: Mon, 13 Apr 2015 12:08:55 +0100
Subject: [PATCH] [SPARK-6868][YARN] Fix broken container log link on executor
page when HTTPS_ONLY.
Correct http schema in YARN container log link in Spark UI when container logs when YARN is configured to be HTTPS_ONLY.
Uses the same logic as the YARN jobtracker webapp. Entry point is [JobBlock](https://github.com/apache/hadoop/blob/e1109fb65608a668cd53dc324dadc6f63a74eeb9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java#L108) and logic is in [MRWebAppUtil](https://github.com/apache/hadoop/blob/e1109fb65608a668cd53dc324dadc6f63a74eeb9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java#L75).
I chose to migrate the logic over instead of importing MRWebAppUtil(but can update the PR to do so) since the class is designated as private and the logic was straightforward.
Author: Dean Chen <deanchen5@gmail.com>
Closes #5477 from deanchen/container-url and squashes the following commits:
91d3090 [Dean Chen] Correct http schema in YARN container log link in Spark UI when container logs when YARN is configured to be HTTPS_ONLY.
---
.../apache/spark/deploy/yarn/ExecutorRunnable.scala | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index 1ce10d906a..b06069c07f 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -290,10 +290,19 @@ class ExecutorRunnable(
YarnSparkHadoopUtil.setEnvFromInputString(env, userEnvs)
}
+ // lookup appropriate http scheme for container log urls
+ val yarnHttpPolicy = yarnConf.get(
+ YarnConfiguration.YARN_HTTP_POLICY_KEY,
+ YarnConfiguration.YARN_HTTP_POLICY_DEFAULT
+ )
+ val httpScheme = if (yarnHttpPolicy == "HTTPS_ONLY") "https://" else "http://"
+
// Add log urls
sys.env.get("SPARK_USER").foreach { user =>
- val baseUrl = "http://%s/node/containerlogs/%s/%s"
- .format(container.getNodeHttpAddress, ConverterUtils.toString(container.getId), user)
+ val containerId = ConverterUtils.toString(container.getId)
+ val address = container.getNodeHttpAddress
+ val baseUrl = s"$httpScheme$address/node/containerlogs/$containerId/$user"
+
env("SPARK_LOG_URL_STDERR") = s"$baseUrl/stderr?start=0"
env("SPARK_LOG_URL_STDOUT") = s"$baseUrl/stdout?start=0"
}
--
GitLab