Skip to content
Snippets Groups Projects
Commit d6d3e507 authored by Dongjoon Hyun's avatar Dongjoon Hyun Committed by Andrew Or
Browse files

[MINOR][CORE] Fix a HadoopRDD log message and remove unused imports in rdd files.

## What changes were proposed in this pull request?

This PR fixes the following typos in log message and comments of `HadoopRDD.scala`. Also, this removes unused imports.
```scala
-      logWarning("Caching NewHadoopRDDs as deserialized objects usually leads to undesired" +
+      logWarning("Caching HadoopRDDs as deserialized objects usually leads to undesired" +
...
-      // since its not removed yet
+      // since it's not removed yet
```

## How was this patch tested?

Manual.

Author: Dongjoon Hyun <dongjoon@apache.org>

Closes #13294 from dongjoon-hyun/minor_rdd_fix_log_message.
parent 8239fdcb
No related branches found
No related tags found
No related merge requests found
...@@ -17,7 +17,7 @@ ...@@ -17,7 +17,7 @@
package org.apache.spark.rdd package org.apache.spark.rdd
import org.apache.hadoop.conf.{ Configurable, Configuration } import org.apache.hadoop.conf.{Configurable, Configuration}
import org.apache.hadoop.io.Writable import org.apache.hadoop.io.Writable
import org.apache.hadoop.mapreduce._ import org.apache.hadoop.mapreduce._
import org.apache.hadoop.mapreduce.task.JobContextImpl import org.apache.hadoop.mapreduce.task.JobContextImpl
......
...@@ -43,7 +43,6 @@ import org.apache.spark._ ...@@ -43,7 +43,6 @@ import org.apache.spark._
import org.apache.spark.annotation.DeveloperApi import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.broadcast.Broadcast import org.apache.spark.broadcast.Broadcast
import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.executor.DataReadMethod
import org.apache.spark.internal.Logging import org.apache.spark.internal.Logging
import org.apache.spark.rdd.HadoopRDD.HadoopMapPartitionsWithSplitRDD import org.apache.spark.rdd.HadoopRDD.HadoopMapPartitionsWithSplitRDD
import org.apache.spark.scheduler.{HDFSCacheTaskLocation, HostTaskLocation} import org.apache.spark.scheduler.{HDFSCacheTaskLocation, HostTaskLocation}
...@@ -70,7 +69,7 @@ private[spark] class HadoopPartition(rddId: Int, override val index: Int, s: Inp ...@@ -70,7 +69,7 @@ private[spark] class HadoopPartition(rddId: Int, override val index: Int, s: Inp
val envVars: Map[String, String] = if (inputSplit.value.isInstanceOf[FileSplit]) { val envVars: Map[String, String] = if (inputSplit.value.isInstanceOf[FileSplit]) {
val is: FileSplit = inputSplit.value.asInstanceOf[FileSplit] val is: FileSplit = inputSplit.value.asInstanceOf[FileSplit]
// map_input_file is deprecated in favor of mapreduce_map_input_file but set both // map_input_file is deprecated in favor of mapreduce_map_input_file but set both
// since its not removed yet // since it's not removed yet
Map("map_input_file" -> is.getPath().toString(), Map("map_input_file" -> is.getPath().toString(),
"mapreduce_map_input_file" -> is.getPath().toString()) "mapreduce_map_input_file" -> is.getPath().toString())
} else { } else {
...@@ -335,7 +334,7 @@ class HadoopRDD[K, V]( ...@@ -335,7 +334,7 @@ class HadoopRDD[K, V](
override def persist(storageLevel: StorageLevel): this.type = { override def persist(storageLevel: StorageLevel): this.type = {
if (storageLevel.deserialized) { if (storageLevel.deserialized) {
logWarning("Caching NewHadoopRDDs as deserialized objects usually leads to undesired" + logWarning("Caching HadoopRDDs as deserialized objects usually leads to undesired" +
" behavior because Hadoop's RecordReader reuses the same Writable object for all records." + " behavior because Hadoop's RecordReader reuses the same Writable object for all records." +
" Use a map transformation to make copies of the records.") " Use a map transformation to make copies of the records.")
} }
......
...@@ -32,7 +32,6 @@ import org.apache.hadoop.mapreduce.task.{JobContextImpl, TaskAttemptContextImpl} ...@@ -32,7 +32,6 @@ import org.apache.hadoop.mapreduce.task.{JobContextImpl, TaskAttemptContextImpl}
import org.apache.spark._ import org.apache.spark._
import org.apache.spark.annotation.DeveloperApi import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.executor.DataReadMethod
import org.apache.spark.internal.Logging import org.apache.spark.internal.Logging
import org.apache.spark.rdd.NewHadoopRDD.NewHadoopMapPartitionsWithSplitRDD import org.apache.spark.rdd.NewHadoopRDD.NewHadoopMapPartitionsWithSplitRDD
import org.apache.spark.storage.StorageLevel import org.apache.spark.storage.StorageLevel
......
...@@ -40,7 +40,7 @@ import org.apache.spark._ ...@@ -40,7 +40,7 @@ import org.apache.spark._
import org.apache.spark.Partitioner.defaultPartitioner import org.apache.spark.Partitioner.defaultPartitioner
import org.apache.spark.annotation.Experimental import org.apache.spark.annotation.Experimental
import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.executor.{DataWriteMethod, OutputMetrics} import org.apache.spark.executor.OutputMetrics
import org.apache.spark.internal.Logging import org.apache.spark.internal.Logging
import org.apache.spark.partial.{BoundedDouble, PartialResult} import org.apache.spark.partial.{BoundedDouble, PartialResult}
import org.apache.spark.serializer.Serializer import org.apache.spark.serializer.Serializer
......
...@@ -31,7 +31,6 @@ import scala.collection.Map ...@@ -31,7 +31,6 @@ import scala.collection.Map
import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.ArrayBuffer
import scala.io.Source import scala.io.Source
import scala.reflect.ClassTag import scala.reflect.ClassTag
import scala.util.control.NonFatal
import org.apache.spark.{Partition, SparkEnv, TaskContext} import org.apache.spark.{Partition, SparkEnv, TaskContext}
import org.apache.spark.util.Utils import org.apache.spark.util.Utils
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment