Skip to content
Snippets Groups Projects
Commit 28f1108e authored by Skeirik's avatar Skeirik
Browse files

added more logging to getOrCompute in RDD.scala

parent 82d4d569
No related branches found
No related tags found
No related merge requests found
...@@ -369,17 +369,20 @@ abstract class RDD[T: ClassTag]( ...@@ -369,17 +369,20 @@ abstract class RDD[T: ClassTag](
blockExInfos.put(key, new BlockExInfo(key)) blockExInfos.put(key, new BlockExInfo(key))
} }
logInfo("Phoenix: about to check parents of node in current stage")
blockManager.stageExInfos.get(blockManager.currentStage) match { blockManager.stageExInfos.get(blockManager.currentStage) match {
case Some(curStageExInfo) => case Some(curStageExInfo) =>
var parExist = true var parExist = true
// get parents of current node
for (par <- curStageExInfo.depMap(id)) { for (par <- curStageExInfo.depMap(id)) {
val parBlockId = new RDDBlockId(par, partition.index) val parBlockId = new RDDBlockId(par, partition.index)
if (blockExInfos.containsKey(parBlockId) && if (blockExInfos.containsKey(parBlockId) &&
blockExInfos.get(parBlockId).isExist == 1) { blockExInfos.get(parBlockId).isExist == 1) {
// par is exist // par is exist
logInfo("Phoenix: A parent for this node exists")
} else { } else {
// par not exist now, add this key to it's par's watching set // par not exist now, add this key to it's par's watching set
logInfo("Phoenix: A parent for this node was not found")
parExist = false parExist = false
if (!blockExInfos.containsKey(parBlockId)) { if (!blockExInfos.containsKey(parBlockId)) {
blockExInfos.put(parBlockId, new BlockExInfo(parBlockId)) blockExInfos.put(parBlockId, new BlockExInfo(parBlockId))
...@@ -389,12 +392,11 @@ abstract class RDD[T: ClassTag]( ...@@ -389,12 +392,11 @@ abstract class RDD[T: ClassTag](
} }
if (parExist) { if (parExist) {
// par are all exist so we update this rdd's start time // par are all exist so we update this rdd's start time
// logEarne("par all exist, store start time of " + key) logInfo("Phoenix: Nodes parents all exist, store start time of " + key)
blockExInfos.get(key).creatStartTime = blockExInfos.get(key).creatStartTime = System.currentTimeMillis()
System.currentTimeMillis()
} }
case None => case None =>
logInfo("An Error With CS525 Project") logInfo("Phoenix: stage info not found")
} }
// SS } // SS }
// This method is called on executors, so we need call SparkEnv.get instead of sc.env. // This method is called on executors, so we need call SparkEnv.get instead of sc.env.
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment