From 9ff203346ca4decf2999e33bfb8c400ec75313e6 Mon Sep 17 00:00:00 2001
From: Wisely Chen <wiselychen@appier.com>
Date: Mon, 6 Jul 2015 16:04:01 -0700
Subject: [PATCH] [SPARK-8656] [WEBUI] Fix the webUI and JSON API number is not
synced
Spark standalone master web UI show "Alive Workers" total core, total used cores and "Alive workers" total memory, memory used.
But the JSON API page "http://MASTERURL:8088/json" shows "ALL workers" core, memory number.
This webUI data is not sync with the JSON API.
The proper way is to sync the number with webUI and JSON API.
Author: Wisely Chen <wiselychen@appier.com>
Closes #7038 from thegiive/SPARK-8656 and squashes the following commits:
9e54bf0 [Wisely Chen] Change variable name to camel case
2c8ea89 [Wisely Chen] Change some styling and add local variable
431d2b0 [Wisely Chen] Worker List should contain DEAD node also
8b3b8e8 [Wisely Chen] [SPARK-8656] Fix the webUI and JSON API number is not synced
---
.../scala/org/apache/spark/deploy/JsonProtocol.scala | 9 +++++----
.../org/apache/spark/deploy/master/WorkerInfo.scala | 2 ++
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
index 2954f932b4..ccffb36652 100644
--- a/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/JsonProtocol.scala
@@ -76,12 +76,13 @@ private[deploy] object JsonProtocol {
}
def writeMasterState(obj: MasterStateResponse): JObject = {
+ val aliveWorkers = obj.workers.filter(_.isAlive())
("url" -> obj.uri) ~
("workers" -> obj.workers.toList.map(writeWorkerInfo)) ~
- ("cores" -> obj.workers.map(_.cores).sum) ~
- ("coresused" -> obj.workers.map(_.coresUsed).sum) ~
- ("memory" -> obj.workers.map(_.memory).sum) ~
- ("memoryused" -> obj.workers.map(_.memoryUsed).sum) ~
+ ("cores" -> aliveWorkers.map(_.cores).sum) ~
+ ("coresused" -> aliveWorkers.map(_.coresUsed).sum) ~
+ ("memory" -> aliveWorkers.map(_.memory).sum) ~
+ ("memoryused" -> aliveWorkers.map(_.memoryUsed).sum) ~
("activeapps" -> obj.activeApps.toList.map(writeApplicationInfo)) ~
("completedapps" -> obj.completedApps.toList.map(writeApplicationInfo)) ~
("activedrivers" -> obj.activeDrivers.toList.map(writeDriverInfo)) ~
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala b/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala
index 471811037e..f751966605 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/WorkerInfo.scala
@@ -105,4 +105,6 @@ private[spark] class WorkerInfo(
def setState(state: WorkerState.Value): Unit = {
this.state = state
}
+
+ def isAlive(): Boolean = this.state == WorkerState.ALIVE
}
--
GitLab