Skip to content
Snippets Groups Projects
Commit 25826c77 authored by Andrew Ray's avatar Andrew Ray Committed by Sean Owen
Browse files

[SPARK-21330][SQL] Bad partitioning does not allow to read a JDBC table with...

[SPARK-21330][SQL] Bad partitioning does not allow to read a JDBC table with extreme values on the partition column

## What changes were proposed in this pull request?

An overflow of the difference of bounds on the partitioning column leads to no data being read. This
patch checks for this overflow.

## How was this patch tested?

New unit test.

Author: Andrew Ray <ray.andrew@gmail.com>

Closes #18800 from aray/SPARK-21330.
parent e3967dc5
No related branches found
No related tags found
No related merge requests found
......@@ -64,7 +64,8 @@ private[sql] object JDBCRelation extends Logging {
s"bound. Lower bound: $lowerBound; Upper bound: $upperBound")
val numPartitions =
if ((upperBound - lowerBound) >= partitioning.numPartitions) {
if ((upperBound - lowerBound) >= partitioning.numPartitions || /* check for overflow */
(upperBound - lowerBound) < 0) {
partitioning.numPartitions
} else {
logWarning("The number of partitions is reduced because the specified number of " +
......
......@@ -96,6 +96,15 @@ class JDBCSuite extends SparkFunSuite
| partitionColumn 'THEID', lowerBound '1', upperBound '4', numPartitions '3')
""".stripMargin.replaceAll("\n", " "))
sql(
s"""
|CREATE OR REPLACE TEMPORARY VIEW partsoverflow
|USING org.apache.spark.sql.jdbc
|OPTIONS (url '$url', dbtable 'TEST.PEOPLE', user 'testUser', password 'testPass',
| partitionColumn 'THEID', lowerBound '-9223372036854775808',
| upperBound '9223372036854775807', numPartitions '3')
""".stripMargin.replaceAll("\n", " "))
conn.prepareStatement("create table test.inttypes (a INT, b BOOLEAN, c TINYINT, "
+ "d SMALLINT, e BIGINT)").executeUpdate()
conn.prepareStatement("insert into test.inttypes values (1, false, 3, 4, 1234567890123)"
......@@ -376,6 +385,12 @@ class JDBCSuite extends SparkFunSuite
assert(ids(2) === 3)
}
test("overflow of partition bound difference does not give negative stride") {
val df = sql("SELECT * FROM partsoverflow")
checkNumPartitions(df, expectedNumPartitions = 3)
assert(df.collect().length == 3)
}
test("Register JDBC query with renamed fields") {
// Regression test for bug SPARK-7345
sql(
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment