Skip to content
Snippets Groups Projects
Commit 0e6368ff authored by Yin Huai's avatar Yin Huai Committed by Cheng Lian
Browse files

[SPARK-10197] [SQL] Add null check in wrapperFor (inside HiveInspectors).

https://issues.apache.org/jira/browse/SPARK-10197

Author: Yin Huai <yhuai@databricks.com>

Closes #8407 from yhuai/ORCSPARK-10197.
parent 7bc9a8c6
No related branches found
No related tags found
No related merge requests found
......@@ -370,17 +370,36 @@ private[hive] trait HiveInspectors {
protected def wrapperFor(oi: ObjectInspector, dataType: DataType): Any => Any = oi match {
case _: JavaHiveVarcharObjectInspector =>
(o: Any) =>
val s = o.asInstanceOf[UTF8String].toString
new HiveVarchar(s, s.size)
if (o != null) {
val s = o.asInstanceOf[UTF8String].toString
new HiveVarchar(s, s.size)
} else {
null
}
case _: JavaHiveDecimalObjectInspector =>
(o: Any) => HiveDecimal.create(o.asInstanceOf[Decimal].toJavaBigDecimal)
(o: Any) =>
if (o != null) {
HiveDecimal.create(o.asInstanceOf[Decimal].toJavaBigDecimal)
} else {
null
}
case _: JavaDateObjectInspector =>
(o: Any) => DateTimeUtils.toJavaDate(o.asInstanceOf[Int])
(o: Any) =>
if (o != null) {
DateTimeUtils.toJavaDate(o.asInstanceOf[Int])
} else {
null
}
case _: JavaTimestampObjectInspector =>
(o: Any) => DateTimeUtils.toJavaTimestamp(o.asInstanceOf[Long])
(o: Any) =>
if (o != null) {
DateTimeUtils.toJavaTimestamp(o.asInstanceOf[Long])
} else {
null
}
case soi: StandardStructObjectInspector =>
val schema = dataType.asInstanceOf[StructType]
......
......@@ -121,6 +121,35 @@ abstract class OrcSuite extends QueryTest with BeforeAndAfterAll {
sql("SELECT * FROM normal_orc_as_source"),
(6 to 10).map(i => Row(i, s"part-$i")))
}
test("write null values") {
sql("DROP TABLE IF EXISTS orcNullValues")
val df = sql(
"""
|SELECT
| CAST(null as TINYINT),
| CAST(null as SMALLINT),
| CAST(null as INT),
| CAST(null as BIGINT),
| CAST(null as FLOAT),
| CAST(null as DOUBLE),
| CAST(null as DECIMAL(7,2)),
| CAST(null as TIMESTAMP),
| CAST(null as DATE),
| CAST(null as STRING),
| CAST(null as VARCHAR(10))
|FROM orc_temp_table limit 1
""".stripMargin)
df.write.format("orc").saveAsTable("orcNullValues")
checkAnswer(
sql("SELECT * FROM orcNullValues"),
Row.fromSeq(Seq.fill(11)(null)))
sql("DROP TABLE IF EXISTS orcNullValues")
}
}
class OrcSourceSuite extends OrcSuite {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment