diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index 667ff649d12979e40f39e87f5cb66f51cb3a2fba..92d0888fc6eea5d2ec02665b2f0bdbcfecfbf01b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -517,7 +517,10 @@ case class StructToJson(options: Map[String, String], child: Expression) @transient lazy val gen = - new JacksonGenerator(child.dataType.asInstanceOf[StructType], writer) + new JacksonGenerator( + child.dataType.asInstanceOf[StructType], + writer, + new JSONOptions(options)) override def dataType: DataType = StringType diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala index 890cc5b560d02761dba0aef4c80cb34c3b061f4b..9c39b3c7f09bf1f4b789452ce39dce2bcf439fe9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql import org.apache.spark.sql.functions.{from_json, struct, to_json} import org.apache.spark.sql.test.SharedSQLContext -import org.apache.spark.sql.types.{CalendarIntervalType, IntegerType, StructType} +import org.apache.spark.sql.types.{CalendarIntervalType, IntegerType, StructType, TimestampType} class JsonFunctionsSuite extends QueryTest with SharedSQLContext { import testImplicits._ @@ -105,6 +105,16 @@ class JsonFunctionsSuite extends QueryTest with SharedSQLContext { Row(Row(1)) :: Nil) } + test("from_json with option") { + val df = Seq("""{"time": "26/08/2015 18:00"}""").toDS() + val schema = new StructType().add("time", TimestampType) + val options = Map("timestampFormat" -> "dd/MM/yyyy HH:mm") + + checkAnswer( + df.select(from_json($"value", schema, options)), + Row(Row(java.sql.Timestamp.valueOf("2015-08-26 18:00:00.0")))) + } + test("from_json missing columns") { val df = Seq("""{"a": 1}""").toDS() val schema = new StructType().add("b", IntegerType) @@ -131,6 +141,15 @@ class JsonFunctionsSuite extends QueryTest with SharedSQLContext { Row("""{"_1":1}""") :: Nil) } + test("to_json with option") { + val df = Seq(Tuple1(Tuple1(java.sql.Timestamp.valueOf("2015-08-26 18:00:00.0")))).toDF("a") + val options = Map("timestampFormat" -> "dd/MM/yyyy HH:mm") + + checkAnswer( + df.select(to_json($"a", options)), + Row("""{"_1":"26/08/2015 18:00"}""") :: Nil) + } + test("to_json unsupported type") { val df = Seq(Tuple1(Tuple1("interval -3 month 7 hours"))).toDF("a") .select(struct($"a._1".cast(CalendarIntervalType).as("a")).as("c"))