Skip to content
Snippets Groups Projects
Commit 24b818b9 authored by shitis's avatar shitis Committed by Andrew Or
Browse files

[SPARK-3944][Core] Using Option[String] where value of String can be null

Author: shitis <ssaxena.ece@gmail.com>

Closes #2795 from Shiti/master and squashes the following commits:

46897d7 [shitis] Using Option Wrapper to convert String with value null to None
parent 7ced88b0
No related branches found
No related tags found
No related merge requests found
...@@ -340,8 +340,8 @@ private[spark] object Utils extends Logging { ...@@ -340,8 +340,8 @@ private[spark] object Utils extends Logging {
val targetFile = new File(targetDir, filename) val targetFile = new File(targetDir, filename)
val uri = new URI(url) val uri = new URI(url)
val fileOverwrite = conf.getBoolean("spark.files.overwrite", defaultValue = false) val fileOverwrite = conf.getBoolean("spark.files.overwrite", defaultValue = false)
uri.getScheme match { Option(uri.getScheme) match {
case "http" | "https" | "ftp" => case Some("http") | Some("https") | Some("ftp") =>
logInfo("Fetching " + url + " to " + tempFile) logInfo("Fetching " + url + " to " + tempFile)
var uc: URLConnection = null var uc: URLConnection = null
...@@ -374,7 +374,7 @@ private[spark] object Utils extends Logging { ...@@ -374,7 +374,7 @@ private[spark] object Utils extends Logging {
} }
} }
Files.move(tempFile, targetFile) Files.move(tempFile, targetFile)
case "file" | null => case Some("file") | None =>
// In the case of a local file, copy the local file to the target directory. // In the case of a local file, copy the local file to the target directory.
// Note the difference between uri vs url. // Note the difference between uri vs url.
val sourceFile = if (uri.isAbsolute) new File(uri) else new File(url) val sourceFile = if (uri.isAbsolute) new File(uri) else new File(url)
...@@ -403,7 +403,7 @@ private[spark] object Utils extends Logging { ...@@ -403,7 +403,7 @@ private[spark] object Utils extends Logging {
logInfo("Copying " + sourceFile.getAbsolutePath + " to " + targetFile.getAbsolutePath) logInfo("Copying " + sourceFile.getAbsolutePath + " to " + targetFile.getAbsolutePath)
Files.copy(sourceFile, targetFile) Files.copy(sourceFile, targetFile)
} }
case _ => case Some(other) =>
// Use the Hadoop filesystem library, which supports file://, hdfs://, s3://, and others // Use the Hadoop filesystem library, which supports file://, hdfs://, s3://, and others
val fs = getHadoopFileSystem(uri, hadoopConf) val fs = getHadoopFileSystem(uri, hadoopConf)
val in = fs.open(new Path(uri)) val in = fs.open(new Path(uri))
...@@ -1368,16 +1368,17 @@ private[spark] object Utils extends Logging { ...@@ -1368,16 +1368,17 @@ private[spark] object Utils extends Logging {
if (uri.getPath == null) { if (uri.getPath == null) {
throw new IllegalArgumentException(s"Given path is malformed: $uri") throw new IllegalArgumentException(s"Given path is malformed: $uri")
} }
uri.getScheme match {
case windowsDrive(d) if windows => Option(uri.getScheme) match {
case Some(windowsDrive(d)) if windows =>
new URI("file:/" + uri.toString.stripPrefix("/")) new URI("file:/" + uri.toString.stripPrefix("/"))
case null => case None =>
// Preserve fragments for HDFS file name substitution (denoted by "#") // Preserve fragments for HDFS file name substitution (denoted by "#")
// For instance, in "abc.py#xyz.py", "xyz.py" is the name observed by the application // For instance, in "abc.py#xyz.py", "xyz.py" is the name observed by the application
val fragment = uri.getFragment val fragment = uri.getFragment
val part = new File(uri.getPath).toURI val part = new File(uri.getPath).toURI
new URI(part.getScheme, part.getPath, fragment) new URI(part.getScheme, part.getPath, fragment)
case _ => case Some(other) =>
uri uri
} }
} }
...@@ -1399,10 +1400,11 @@ private[spark] object Utils extends Logging { ...@@ -1399,10 +1400,11 @@ private[spark] object Utils extends Logging {
} else { } else {
paths.split(",").filter { p => paths.split(",").filter { p =>
val formattedPath = if (windows) formatWindowsPath(p) else p val formattedPath = if (windows) formatWindowsPath(p) else p
new URI(formattedPath).getScheme match { val uri = new URI(formattedPath)
case windowsDrive(d) if windows => false Option(uri.getScheme) match {
case "local" | "file" | null => false case Some(windowsDrive(d)) if windows => false
case _ => true case Some("local") | Some("file") | None => false
case Some(other) => true
} }
} }
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment