From 28f1108e2bf6902db03488c9f605c0a1bf338090 Mon Sep 17 00:00:00 2001
From: Skeirik <skeirik2@sp18-cs525-g07-01.cs.illinois.edu>
Date: Sun, 6 May 2018 20:03:40 -0500
Subject: [PATCH] added more logging to getOrCompute in RDD.scala
---
core/src/main/scala/org/apache/spark/rdd/RDD.scala | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index 9daa777074..76908fa608 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -369,17 +369,20 @@ abstract class RDD[T: ClassTag](
blockExInfos.put(key, new BlockExInfo(key))
}
+ logInfo("Phoenix: about to check parents of node in current stage")
blockManager.stageExInfos.get(blockManager.currentStage) match {
case Some(curStageExInfo) =>
var parExist = true
+ // get parents of current node
for (par <- curStageExInfo.depMap(id)) {
val parBlockId = new RDDBlockId(par, partition.index)
if (blockExInfos.containsKey(parBlockId) &&
blockExInfos.get(parBlockId).isExist == 1) {
// par is exist
-
+ logInfo("Phoenix: A parent for this node exists")
} else {
// par not exist now, add this key to it's par's watching set
+ logInfo("Phoenix: A parent for this node was not found")
parExist = false
if (!blockExInfos.containsKey(parBlockId)) {
blockExInfos.put(parBlockId, new BlockExInfo(parBlockId))
@@ -389,12 +392,11 @@ abstract class RDD[T: ClassTag](
}
if (parExist) {
// par are all exist so we update this rdd's start time
- // logEarne("par all exist, store start time of " + key)
- blockExInfos.get(key).creatStartTime =
- System.currentTimeMillis()
+ logInfo("Phoenix: Nodes parents all exist, store start time of " + key)
+ blockExInfos.get(key).creatStartTime = System.currentTimeMillis()
}
case None =>
- logInfo("An Error With CS525 Project")
+ logInfo("Phoenix: stage info not found")
}
// SS }
// This method is called on executors, so we need call SparkEnv.get instead of sc.env.
--
GitLab