diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala index 56cd17816f7bd6ae86ab590a1e9e3eaba605ca0c..6fd2e0d24112b3e54614f78424d53f2207bcfdb6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCOptions.scala @@ -20,8 +20,6 @@ package org.apache.spark.sql.execution.datasources.jdbc import java.sql.{Connection, DriverManager} import java.util.Properties -import scala.collection.mutable.ArrayBuffer - import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap /** @@ -41,10 +39,23 @@ class JDBCOptions( JDBCOptions.JDBC_TABLE_NAME -> table))) } + /** + * Returns a property with all options. + */ + val asProperties: Properties = { + val properties = new Properties() + parameters.foreach { case (k, v) => properties.setProperty(k, v) } + properties + } + + /** + * Returns a property with all options except Spark internal data source options like `url`, + * `dbtable`, and `numPartition`. This should be used when invoking JDBC API like `Driver.connect` + * because each DBMS vendor has its own property list for JDBC driver. See SPARK-17776. + */ val asConnectionProperties: Properties = { val properties = new Properties() - // We should avoid to pass the options into properties. See SPARK-17776. - parameters.filterKeys(!jdbcOptionNames.contains(_)) + parameters.filterKeys(key => !jdbcOptionNames(key.toLowerCase)) .foreach { case (k, v) => properties.setProperty(k, v) } properties } @@ -126,10 +137,10 @@ class JDBCOptions( } object JDBCOptions { - private val jdbcOptionNames = ArrayBuffer.empty[String] + private val jdbcOptionNames = collection.mutable.Set[String]() private def newOption(name: String): String = { - jdbcOptionNames += name + jdbcOptionNames += name.toLowerCase name } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index 37df283a9e5b28b7857aa48886a4d240874503df..d5b11e7bec0bd2187d2a4cec96e4cd25d78dfe97 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -54,7 +54,6 @@ object JDBCRDD extends Logging { def resolveTable(options: JDBCOptions): StructType = { val url = options.url val table = options.table - val properties = options.asConnectionProperties val dialect = JdbcDialects.get(url) val conn: Connection = JdbcUtils.createConnectionFactory(options)() try { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala index 5ca1c7543cfa7a92589849cf4090782be501f4be..8b45dba04d29e09ae28d1b14efdcc2ba86196182 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala @@ -131,7 +131,7 @@ private[sql] case class JDBCRelation( override def insert(data: DataFrame, overwrite: Boolean): Unit = { val url = jdbcOptions.url val table = jdbcOptions.table - val properties = jdbcOptions.asConnectionProperties + val properties = jdbcOptions.asProperties data.write .mode(if (overwrite) SaveMode.Overwrite else SaveMode.Append) .jdbc(url, table, properties) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 7f312dbdddd852c8f31b45d7cf86eecc561c18d1..74ca66b1033c26edb7e2185ee03ef7657cebe0ef 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -26,6 +26,7 @@ import org.scalatest.{BeforeAndAfter, PrivateMethodTester} import org.apache.spark.SparkFunSuite import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap import org.apache.spark.sql.execution.DataSourceScanExec import org.apache.spark.sql.execution.command.ExplainCommand import org.apache.spark.sql.execution.datasources.LogicalRelation @@ -890,4 +891,13 @@ class JDBCSuite extends SparkFunSuite assert(sql("SELECT * FROM mixedCaseCols WHERE Id = 1 OR Name = 'mary'").collect().size == 2) assert(sql("SELECT * FROM mixedCaseCols WHERE Name = 'mary' AND Id = 2").collect().size == 1) } + + test("SPARK-18419: Fix `asConnectionProperties` to filter case-insensitively") { + val parameters = Map( + "url" -> "jdbc:mysql://localhost:3306/temp", + "dbtable" -> "t1", + "numPartitions" -> "10") + assert(new JDBCOptions(parameters).asConnectionProperties.isEmpty) + assert(new JDBCOptions(new CaseInsensitiveMap(parameters)).asConnectionProperties.isEmpty) + } }