Skip to content
Snippets Groups Projects
Commit 162d04a3 authored by Dongjoon Hyun's avatar Dongjoon Hyun Committed by Reynold Xin
Browse files

[SPARK-16602][SQL] `Nvl` function should support numeric-string cases

## What changes were proposed in this pull request?

`Nvl` function should support numeric-straing cases like Hive/Spark1.6. Currently, `Nvl` finds the tightest common types among numeric types. This PR extends that to consider `String` type, too.

```scala
- TypeCoercion.findTightestCommonTypeOfTwo(left.dataType, right.dataType).map { dtype =>
+ TypeCoercion.findTightestCommonTypeToString(left.dataType, right.dataType).map { dtype =>
```

**Before**
```scala
scala> sql("select nvl('0', 1)").collect()
org.apache.spark.sql.AnalysisException: cannot resolve `nvl("0", 1)` due to data type mismatch:
input to function coalesce should all be the same type, but it's [string, int]; line 1 pos 7
```

**After**
```scala
scala> sql("select nvl('0', 1)").collect()
res0: Array[org.apache.spark.sql.Row] = Array([0])
```

## How was this patch tested?

Pass the Jenkins tests.

Author: Dongjoon Hyun <dongjoon@apache.org>

Closes #14251 from dongjoon-hyun/SPARK-16602.
parent 0bd76e87
No related branches found
No related tags found
No related merge requests found
......@@ -100,7 +100,7 @@ object TypeCoercion {
}
/** Similar to [[findTightestCommonType]], but can promote all the way to StringType. */
private def findTightestCommonTypeToString(left: DataType, right: DataType): Option[DataType] = {
def findTightestCommonTypeToString(left: DataType, right: DataType): Option[DataType] = {
findTightestCommonTypeOfTwo(left, right).orElse((left, right) match {
case (StringType, t2: AtomicType) if t2 != BinaryType && t2 != BooleanType => Some(StringType)
case (t1: AtomicType, StringType) if t1 != BinaryType && t1 != BooleanType => Some(StringType)
......
......@@ -134,7 +134,7 @@ case class Nvl(left: Expression, right: Expression) extends RuntimeReplaceable {
override def replaceForTypeCoercion(): Expression = {
if (left.dataType != right.dataType) {
TypeCoercion.findTightestCommonTypeOfTwo(left.dataType, right.dataType).map { dtype =>
TypeCoercion.findTightestCommonTypeToString(left.dataType, right.dataType).map { dtype =>
copy(left = Cast(left, dtype), right = Cast(right, dtype))
}.getOrElse(this)
} else {
......
......@@ -77,6 +77,21 @@ class NullFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
}
}
test("SPARK-16602 Nvl should support numeric-string cases") {
val intLit = Literal.create(1, IntegerType)
val doubleLit = Literal.create(2.2, DoubleType)
val stringLit = Literal.create("c", StringType)
val nullLit = Literal.create(null, NullType)
assert(Nvl(intLit, doubleLit).replaceForTypeCoercion().dataType == DoubleType)
assert(Nvl(intLit, stringLit).replaceForTypeCoercion().dataType == StringType)
assert(Nvl(stringLit, doubleLit).replaceForTypeCoercion().dataType == StringType)
assert(Nvl(nullLit, intLit).replaceForTypeCoercion().dataType == IntegerType)
assert(Nvl(doubleLit, nullLit).replaceForTypeCoercion().dataType == DoubleType)
assert(Nvl(nullLit, stringLit).replaceForTypeCoercion().dataType == StringType)
}
test("AtLeastNNonNulls") {
val mix = Seq(Literal("x"),
Literal.create(null, StringType),
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment