From 58da1a2455258156fe8ba57241611eac1a7928ef Mon Sep 17 00:00:00 2001
From: Devaraj K <devaraj@apache.org>
Date: Tue, 1 Aug 2017 13:38:55 -0700
Subject: [PATCH] [SPARK-21339][CORE] spark-shell --packages option does not
add jars to classpath on windows
The --packages option jars are getting added to the classpath with the scheme as "file:///", in Unix it doesn't have problem with this since the scheme contains the Unix Path separator which separates the jar name with location in the classpath. In Windows, the jar file is not getting resolved from the classpath because of the scheme.
Windows : file:///C:/Users/<user>/.ivy2/jars/<jar-name>.jar
Unix : file:///home/<user>/.ivy2/jars/<jar-name>.jar
With this PR, we are avoiding the 'file://' scheme to get added to the packages jar files.
I have verified manually in Windows and Unix environments, with the change it adds the jar to classpath like below,
Windows : C:\Users\<user>\.ivy2\jars\<jar-name>.jar
Unix : /home/<user>/.ivy2/jars/<jar-name>.jar
Author: Devaraj K <devaraj@apache.org>
Closes #18708 from devaraj-kavali/SPARK-21339.
---
.../src/main/scala/org/apache/spark/repl/Main.scala | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala
index b8b38e828b..9702a1e653 100644
--- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala
+++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala
@@ -18,6 +18,7 @@
package org.apache.spark.repl
import java.io.File
+import java.net.URI
import java.util.Locale
import scala.tools.nsc.GenericRunnerSettings
@@ -56,7 +57,10 @@ object Main extends Logging {
// Visible for testing
private[repl] def doMain(args: Array[String], _interp: SparkILoop): Unit = {
interp = _interp
- val jars = Utils.getUserJars(conf, isShell = true).mkString(File.pathSeparator)
+ val jars = Utils.getUserJars(conf, isShell = true)
+ // Remove file:///, file:// or file:/ scheme if exists for each jar
+ .map { x => if (x.startsWith("file:")) new File(new URI(x)).getPath else x }
+ .mkString(File.pathSeparator)
val interpArguments = List(
"-Yrepl-class-based",
"-Yrepl-outdir", s"${outputDir.getAbsolutePath}",
--
GitLab