Skip to content
Snippets Groups Projects
Commit d7f796da authored by Burak Yavuz's avatar Burak Yavuz Committed by Andrew Or
Browse files

[SPARK-8410] [SPARK-8475] remove previous ivy resolution when using spark-submit

This PR also includes re-ordering the order that repositories are used when resolving packages. User provided repositories will be prioritized.

cc andrewor14

Author: Burak Yavuz <brkyvz@gmail.com>

Closes #7089 from brkyvz/delete-prev-ivy-resolution and squashes the following commits:

a21f95a [Burak Yavuz] remove previous ivy resolution when using spark-submit
parent 5d30eae5
No related branches found
No related tags found
No related merge requests found
...@@ -756,6 +756,20 @@ private[spark] object SparkSubmitUtils { ...@@ -756,6 +756,20 @@ private[spark] object SparkSubmitUtils {
val cr = new ChainResolver val cr = new ChainResolver
cr.setName("list") cr.setName("list")
val repositoryList = remoteRepos.getOrElse("")
// add any other remote repositories other than maven central
if (repositoryList.trim.nonEmpty) {
repositoryList.split(",").zipWithIndex.foreach { case (repo, i) =>
val brr: IBiblioResolver = new IBiblioResolver
brr.setM2compatible(true)
brr.setUsepoms(true)
brr.setRoot(repo)
brr.setName(s"repo-${i + 1}")
cr.add(brr)
printStream.println(s"$repo added as a remote repository with the name: ${brr.getName}")
}
}
val localM2 = new IBiblioResolver val localM2 = new IBiblioResolver
localM2.setM2compatible(true) localM2.setM2compatible(true)
localM2.setRoot(m2Path.toURI.toString) localM2.setRoot(m2Path.toURI.toString)
...@@ -786,20 +800,6 @@ private[spark] object SparkSubmitUtils { ...@@ -786,20 +800,6 @@ private[spark] object SparkSubmitUtils {
sp.setRoot("http://dl.bintray.com/spark-packages/maven") sp.setRoot("http://dl.bintray.com/spark-packages/maven")
sp.setName("spark-packages") sp.setName("spark-packages")
cr.add(sp) cr.add(sp)
val repositoryList = remoteRepos.getOrElse("")
// add any other remote repositories other than maven central
if (repositoryList.trim.nonEmpty) {
repositoryList.split(",").zipWithIndex.foreach { case (repo, i) =>
val brr: IBiblioResolver = new IBiblioResolver
brr.setM2compatible(true)
brr.setUsepoms(true)
brr.setRoot(repo)
brr.setName(s"repo-${i + 1}")
cr.add(brr)
printStream.println(s"$repo added as a remote repository with the name: ${brr.getName}")
}
}
cr cr
} }
...@@ -922,6 +922,15 @@ private[spark] object SparkSubmitUtils { ...@@ -922,6 +922,15 @@ private[spark] object SparkSubmitUtils {
// A Module descriptor must be specified. Entries are dummy strings // A Module descriptor must be specified. Entries are dummy strings
val md = getModuleDescriptor val md = getModuleDescriptor
// clear ivy resolution from previous launches. The resolution file is usually at
// ~/.ivy2/org.apache.spark-spark-submit-parent-default.xml. In between runs, this file
// leads to confusion with Ivy when the files can no longer be found at the repository
// declared in that file/
val mdId = md.getModuleRevisionId
val previousResolution = new File(ivySettings.getDefaultCache,
s"${mdId.getOrganisation}-${mdId.getName}-$ivyConfName.xml")
if (previousResolution.exists) previousResolution.delete
md.setDefaultConf(ivyConfName) md.setDefaultConf(ivyConfName)
// Add exclusion rules for Spark and Scala Library // Add exclusion rules for Spark and Scala Library
......
...@@ -77,9 +77,9 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll { ...@@ -77,9 +77,9 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
assert(resolver2.getResolvers.size() === 7) assert(resolver2.getResolvers.size() === 7)
val expected = repos.split(",").map(r => s"$r/") val expected = repos.split(",").map(r => s"$r/")
resolver2.getResolvers.toArray.zipWithIndex.foreach { case (resolver: AbstractResolver, i) => resolver2.getResolvers.toArray.zipWithIndex.foreach { case (resolver: AbstractResolver, i) =>
if (i > 3) { if (i < 3) {
assert(resolver.getName === s"repo-${i - 3}") assert(resolver.getName === s"repo-${i + 1}")
assert(resolver.asInstanceOf[IBiblioResolver].getRoot === expected(i - 4)) assert(resolver.asInstanceOf[IBiblioResolver].getRoot === expected(i))
} }
} }
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment