Skip to content
Snippets Groups Projects
Commit fc44b694 authored by wangyang's avatar wangyang Committed by Wenchen Fan
Browse files

[SPARK-15379][SQL] check special invalid date

## What changes were proposed in this pull request?

When invalid date string like "2015-02-29 00:00:00" are cast as date or timestamp using spark sql, it used to not return null but another valid date (2015-03-01 in this case).
In this pr, invalid date string like "2016-02-29" and "2016-04-31" are returned as null when cast as date or timestamp.

## How was this patch tested?

Unit tests are added.

(If this patch involves UI changes, please attach a screenshot; otherwise, remove this)

Author: wangyang <wangyang@haizhi.com>

Closes #13169 from wangyang1992/invalid_date.
parent 3eff65f8
No related branches found
No related tags found
No related merge requests found
......@@ -58,6 +58,7 @@ object DateTimeUtils {
final val YearZero = -17999
final val toYearZero = to2001 + 7304850
final val TimeZoneGMT = TimeZone.getTimeZone("GMT")
final val MonthOf31Days = Set(1, 3, 5, 7, 8, 10, 12)
@transient lazy val defaultTimeZone = TimeZone.getDefault
......@@ -333,8 +334,7 @@ object DateTimeUtils {
digitsMilli += 1
}
if (!justTime && (segments(0) < 0 || segments(0) > 9999 || segments(1) < 1 ||
segments(1) > 12 || segments(2) < 1 || segments(2) > 31)) {
if (!justTime && isInvalidDate(segments(0), segments(1), segments(2))) {
return None
}
......@@ -414,10 +414,10 @@ object DateTimeUtils {
return None
}
segments(i) = currentSegmentValue
if (segments(0) < 0 || segments(0) > 9999 || segments(1) < 1 || segments(1) > 12 ||
segments(2) < 1 || segments(2) > 31) {
if (isInvalidDate(segments(0), segments(1), segments(2))) {
return None
}
val c = threadLocalGmtCalendar.get()
c.clear()
c.set(segments(0), segments(1) - 1, segments(2), 0, 0, 0)
......@@ -425,6 +425,25 @@ object DateTimeUtils {
Some((c.getTimeInMillis / MILLIS_PER_DAY).toInt)
}
/**
* Return true if the date is invalid.
*/
private def isInvalidDate(year: Int, month: Int, day: Int): Boolean = {
if (year < 0 || year > 9999 || month < 1 || month > 12 || day < 1 || day > 31) {
return true
}
if (month == 2) {
if (isLeapYear(year) && day > 29) {
return true
} else if (!isLeapYear(year) && day > 28) {
return true
}
} else if (!MonthOf31Days.contains(month) && day > 30) {
return true
}
false
}
/**
* Returns the microseconds since year zero (-17999) from microseconds since epoch.
*/
......
......@@ -353,6 +353,25 @@ class DateTimeUtilsSuite extends SparkFunSuite {
c.getTimeInMillis * 1000 + 123456)
}
test("SPARK-15379: special invalid date string") {
// Test stringToDate
assert(stringToDate(
UTF8String.fromString("2015-02-29 00:00:00")).isEmpty)
assert(stringToDate(
UTF8String.fromString("2015-04-31 00:00:00")).isEmpty)
assert(stringToDate(UTF8String.fromString("2015-02-29")).isEmpty)
assert(stringToDate(UTF8String.fromString("2015-04-31")).isEmpty)
// Test stringToTimestamp
assert(stringToTimestamp(
UTF8String.fromString("2015-02-29 00:00:00")).isEmpty)
assert(stringToTimestamp(
UTF8String.fromString("2015-04-31 00:00:00")).isEmpty)
assert(stringToTimestamp(UTF8String.fromString("2015-02-29")).isEmpty)
assert(stringToTimestamp(UTF8String.fromString("2015-04-31")).isEmpty)
}
test("hours") {
val c = Calendar.getInstance()
c.set(2015, 2, 18, 13, 2, 11)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment