Skip to content
Snippets Groups Projects
Commit a3086643 authored by Alexander Pivovarov's avatar Alexander Pivovarov
Browse files

SHARK-94 Log the files computed by HadoopRDD and NewHadoopRDD

parent 9d7dfd2d
No related branches found
No related tags found
No related merge requests found
...@@ -88,6 +88,7 @@ class HadoopRDD[K, V]( ...@@ -88,6 +88,7 @@ class HadoopRDD[K, V](
override def compute(theSplit: Partition, context: TaskContext) = new NextIterator[(K, V)] { override def compute(theSplit: Partition, context: TaskContext) = new NextIterator[(K, V)] {
val split = theSplit.asInstanceOf[HadoopPartition] val split = theSplit.asInstanceOf[HadoopPartition]
logInfo("Input split: " + split.inputSplit)
var reader: RecordReader[K, V] = null var reader: RecordReader[K, V] = null
val conf = confBroadcast.value.value val conf = confBroadcast.value.value
......
...@@ -73,6 +73,7 @@ class NewHadoopRDD[K, V]( ...@@ -73,6 +73,7 @@ class NewHadoopRDD[K, V](
override def compute(theSplit: Partition, context: TaskContext) = new Iterator[(K, V)] { override def compute(theSplit: Partition, context: TaskContext) = new Iterator[(K, V)] {
val split = theSplit.asInstanceOf[NewHadoopPartition] val split = theSplit.asInstanceOf[NewHadoopPartition]
logInfo("Input split: " + split.serializableHadoopSplit)
val conf = confBroadcast.value.value val conf = confBroadcast.value.value
val attemptId = newTaskAttemptID(jobtrackerId, id, true, split.index, 0) val attemptId = newTaskAttemptID(jobtrackerId, id, true, split.index, 0)
val hadoopAttemptContext = newTaskAttemptContext(conf, attemptId) val hadoopAttemptContext = newTaskAttemptContext(conf, attemptId)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment