From b4fbe140be158f576706f21fa69f40d6e14e4a43 Mon Sep 17 00:00:00 2001
From: Marcelo Vanzin <vanzin@cloudera.com>
Date: Mon, 11 Jul 2016 15:20:48 -0700
Subject: [PATCH] [SPARK-16349][SQL] Fall back to isolated class loader when
classes not found.
Some Hadoop classes needed by the Hive metastore client jars are not present
in Spark's packaging (for example, "org/apache/hadoop/mapred/MRVersion"). So
if the parent class loader fails to find a class, try to load it from the
isolated class loader, in case it's available there.
Tested by setting spark.sql.hive.metastore.jars to local paths with Hive/Hadoop
libraries and verifying that Spark can talk to the metastore.
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #14020 from vanzin/SPARK-16349.
---
.../spark/sql/hive/client/IsolatedClientLoader.scala | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index e1950d181d..a72a13b778 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -220,9 +220,15 @@ private[hive] class IsolatedClientLoader(
logDebug(s"hive class: $name - ${getResource(classToPath(name))}")
super.loadClass(name, resolve)
} else {
- // For shared classes, we delegate to baseClassLoader.
+ // For shared classes, we delegate to baseClassLoader, but fall back in case the
+ // class is not found.
logDebug(s"shared class: $name")
- baseClassLoader.loadClass(name)
+ try {
+ baseClassLoader.loadClass(name)
+ } catch {
+ case _: ClassNotFoundException =>
+ super.loadClass(name, resolve)
+ }
}
}
}
@@ -264,7 +270,7 @@ private[hive] class IsolatedClientLoader(
throw new ClassNotFoundException(
s"$cnf when creating Hive client using classpath: ${execJars.mkString(", ")}\n" +
"Please make sure that jars for your version of hive and hadoop are included in the " +
- s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS}.", e)
+ s"paths passed to ${HiveUtils.HIVE_METASTORE_JARS.key}.", e)
} else {
throw e
}
--
GitLab