From 16293311cdb25a62733a9aae4355659b971a3ce1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=A6=8F=E6=98=9F?= <fuxing@wacai.com>
Date: Thu, 3 Nov 2016 12:02:01 -0700
Subject: [PATCH] [SPARK-18237][HIVE] hive.exec.stagingdir have no effect
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
hive.exec.stagingdir have no effect in spark2.0.1，
Hive confs in hive-site.xml will be loaded in `hadoopConf`, so we should use `hadoopConf` in `InsertIntoHiveTable` instead of `SessionState.conf`
Author: ç¦æ˜Ÿ <fuxing@wacai.com>
Closes #15744 from ClassNotFoundExp/master.
---
.../apache/spark/sql/hive/execution/InsertIntoHiveTable.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
index 15be12cfc0..e333fc7feb 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
@@ -76,7 +76,8 @@ case class InsertIntoHiveTable(
def output: Seq[Attribute] = Seq.empty
- val stagingDir = sessionState.conf.getConfString("hive.exec.stagingdir", ".hive-staging")
+ val hadoopConf = sessionState.newHadoopConf()
+ val stagingDir = hadoopConf.get("hive.exec.stagingdir", ".hive-staging")
private def executionId: String = {
val rand: Random = new Random
@@ -163,7 +164,6 @@ case class InsertIntoHiveTable(
// instances within the closure, since Serializer is not serializable while TableDesc is.
val tableDesc = table.tableDesc
val tableLocation = table.hiveQlTable.getDataLocation
- val hadoopConf = sessionState.newHadoopConf()
val tmpLocation = getExternalTmpPath(tableLocation, hadoopConf)
val fileSinkConf = new FileSinkDesc(tmpLocation.toString, tableDesc, false)
val isCompressed = hadoopConf.get("hive.exec.compress.output", "false").toBoolean
--
GitLab