Skip to content
Snippets Groups Projects
Commit ae5b2d3e authored by Weiqing Yang's avatar Weiqing Yang Committed by Wenchen Fan
Browse files

[SPARK-18746][SQL] Add implicit encoder for BigDecimal, timestamp and date

## What changes were proposed in this pull request?
Add implicit encoders for BigDecimal, timestamp and date.

## How was this patch tested?
Add an unit test.  Pass build, unit tests, and some tests below .
Before:
```
scala> spark.createDataset(Seq(new java.math.BigDecimal(10)))
<console>:24: error: Unable to find encoder for type stored in a Dataset.  Primitive types (Int, String, etc) and Product types (case classes) are supported by importing spark.implicits._  Support for serializing other types will be added in future releases.
       spark.createDataset(Seq(new java.math.BigDecimal(10)))
                          ^

scala>
```
After:
```
scala> spark.createDataset(Seq(new java.math.BigDecimal(10)))
res0: org.apache.spark.sql.Dataset[java.math.BigDecimal] = [value: decimal(38,18)]
```

Author: Weiqing Yang <yangweiqing001@gmail.com>

Closes #16176 from weiqingy/SPARK-18746.
parent 594b14f1
No related branches found
No related tags found
No related merge requests found
......@@ -74,6 +74,19 @@ abstract class SQLImplicits {
/** @since 1.6.0 */
implicit def newStringEncoder: Encoder[String] = Encoders.STRING
/** @since 2.2.0 */
implicit def newJavaDecimalEncoder: Encoder[java.math.BigDecimal] = Encoders.DECIMAL
/** @since 2.2.0 */
implicit def newScalaDecimalEncoder: Encoder[scala.math.BigDecimal] = ExpressionEncoder()
/** @since 2.2.0 */
implicit def newDateEncoder: Encoder[java.sql.Date] = Encoders.DATE
/** @since 2.2.0 */
implicit def newTimeStampEncoder: Encoder[java.sql.Timestamp] = Encoders.TIMESTAMP
// Boxed primitives
/** @since 2.0.0 */
......@@ -141,7 +154,7 @@ abstract class SQLImplicits {
implicit def newFloatArrayEncoder: Encoder[Array[Float]] = ExpressionEncoder()
/** @since 1.6.1 */
implicit def newByteArrayEncoder: Encoder[Array[Byte]] = ExpressionEncoder()
implicit def newByteArrayEncoder: Encoder[Array[Byte]] = Encoders.BINARY
/** @since 1.6.1 */
implicit def newShortArrayEncoder: Encoder[Array[Short]] = ExpressionEncoder()
......
......@@ -26,7 +26,6 @@ import org.apache.spark.sql.execution.{LogicalRDD, RDDScanExec, SortExec}
import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchange}
import org.apache.spark.sql.execution.streaming.MemoryStream
import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSQLContext
import org.apache.spark.sql.types._
......@@ -1129,6 +1128,21 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
val ds2 = Seq(WithMap("hi", Map(42L -> "foo"))).toDS
checkDataset(ds2.map(t => t), WithMap("hi", Map(42L -> "foo")))
}
test("SPARK-18746: add implicit encoder for BigDecimal, date, timestamp") {
// For this implicit encoder, 18 is the default scale
assert(spark.range(1).map { x => new java.math.BigDecimal(1) }.head ==
new java.math.BigDecimal(1).setScale(18))
assert(spark.range(1).map { x => scala.math.BigDecimal(1, 18) }.head ==
scala.math.BigDecimal(1, 18))
assert(spark.range(1).map { x => new java.sql.Date(2016, 12, 12) }.head ==
new java.sql.Date(2016, 12, 12))
assert(spark.range(1).map { x => new java.sql.Timestamp(100000) }.head ==
new java.sql.Timestamp(100000))
}
}
case class WithImmutableMap(id: String, map_test: scala.collection.immutable.Map[Long, String])
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment