Skip to content
Snippets Groups Projects
Commit 2848f4da authored by Reynold Xin's avatar Reynold Xin
Browse files

[SPARK-8809][SQL] Remove ConvertNaNs analyzer rule.

"NaN" from string to double is already handled by Cast expression itself.

Author: Reynold Xin <rxin@databricks.com>

Closes #7206 from rxin/convertnans and squashes the following commits:

3d99c33 [Reynold Xin] [SPARK-8809][SQL] Remove ConvertNaNs analyzer rule.
parent 9b23e92c
No related branches found
No related tags found
No related merge requests found
...@@ -35,7 +35,6 @@ object HiveTypeCoercion { ...@@ -35,7 +35,6 @@ object HiveTypeCoercion {
val typeCoercionRules = val typeCoercionRules =
PropagateTypes :: PropagateTypes ::
ConvertNaNs ::
InConversion :: InConversion ::
WidenTypes :: WidenTypes ::
PromoteStrings :: PromoteStrings ::
...@@ -148,38 +147,6 @@ object HiveTypeCoercion { ...@@ -148,38 +147,6 @@ object HiveTypeCoercion {
} }
} }
/**
* Converts string "NaN"s that are in binary operators with a NaN-able types (Float / Double) to
* the appropriate numeric equivalent.
*/
// TODO: remove this rule and make Cast handle Nan.
object ConvertNaNs extends Rule[LogicalPlan] {
private val StringNaN = Literal("NaN")
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
case q: LogicalPlan => q transformExpressions {
// Skip nodes who's children have not been resolved yet.
case e if !e.childrenResolved => e
/* Double Conversions */
case b @ BinaryOperator(StringNaN, right @ DoubleType()) =>
b.makeCopy(Array(Literal(Double.NaN), right))
case b @ BinaryOperator(left @ DoubleType(), StringNaN) =>
b.makeCopy(Array(left, Literal(Double.NaN)))
/* Float Conversions */
case b @ BinaryOperator(StringNaN, right @ FloatType()) =>
b.makeCopy(Array(Literal(Float.NaN), right))
case b @ BinaryOperator(left @ FloatType(), StringNaN) =>
b.makeCopy(Array(left, Literal(Float.NaN)))
/* Use float NaN by default to avoid unnecessary type widening */
case b @ BinaryOperator(left @ StringNaN, StringNaN) =>
b.makeCopy(Array(left, Literal(Float.NaN)))
}
}
}
/** /**
* Widens numeric types and converts strings to numbers when appropriate. * Widens numeric types and converts strings to numbers when appropriate.
* *
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment