From 90abfd15f4b3f612a7b0ff65f03bf319c78a0243 Mon Sep 17 00:00:00 2001
From: Yuming Wang <wgyumg@gmail.com>
Date: Mon, 12 Dec 2016 23:38:36 +0100
Subject: [PATCH] [SPARK-18681][SQL] Fix filtering to compatible with partition
keys of type int
## What changes were proposed in this pull request?
Cloudera put `/var/run/cloudera-scm-agent/process/15000-hive-HIVEMETASTORE/hive-site.xml` as the configuration file for the Hive Metastore Server, where `hive.metastore.try.direct.sql=false`. But Spark isn't reading this configuration file and get default value `hive.metastore.try.direct.sql=true`. As mallman said, we should use `getMetaConf` method to obtain the original configuration from Hive Metastore Server. I have tested this method few times and the return value is always consistent with Hive Metastore Server.
## How was this patch tested?
The existing tests.
Author: Yuming Wang <wgyumg@gmail.com>
Closes #16122 from wangyum/SPARK-18681.
---
.../scala/org/apache/spark/sql/hive/client/HiveShim.scala | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 137ec26760..b052f1e7e4 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -594,8 +594,11 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
} else {
logDebug(s"Hive metastore filter is '$filter'.")
val tryDirectSqlConfVar = HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL
- val tryDirectSql =
- hive.getConf.getBoolean(tryDirectSqlConfVar.varname, tryDirectSqlConfVar.defaultBoolVal)
+ // We should get this config value from the metaStore. otherwise hit SPARK-18681.
+ // To be compatible with hive-0.12 and hive-0.13, In the future we can achieve this by:
+ // val tryDirectSql = hive.getMetaConf(tryDirectSqlConfVar.varname).toBoolean
+ val tryDirectSql = hive.getMSC.getConfigValue(tryDirectSqlConfVar.varname,
+ tryDirectSqlConfVar.defaultBoolVal.toString).toBoolean
try {
// Hive may throw an exception when calling this method in some circumstances, such as
// when filtering on a non-string partition column when the hive config key
--
GitLab