Skip to content
Snippets Groups Projects
Commit c0b13d55 authored by gatorsmile's avatar gatorsmile Committed by Michael Armbrust
Browse files

[SPARK-12195][SQL] Adding BigDecimal, Date and Timestamp into Encoder

This PR is to add three more data types into Encoder, including `BigDecimal`, `Date` and `Timestamp`.

marmbrus cloud-fan rxin Could you take a quick look at these three types? Not sure if it can be merged to 1.6. Thank you very much!

Author: gatorsmile <gatorsmile@gmail.com>

Closes #10188 from gatorsmile/dataTypesinEncoder.
parent 381f17b5
No related branches found
No related tags found
No related merge requests found
......@@ -97,6 +97,24 @@ object Encoders {
*/
def STRING: Encoder[java.lang.String] = ExpressionEncoder()
/**
* An encoder for nullable decimal type.
* @since 1.6.0
*/
def DECIMAL: Encoder[java.math.BigDecimal] = ExpressionEncoder()
/**
* An encoder for nullable date type.
* @since 1.6.0
*/
def DATE: Encoder[java.sql.Date] = ExpressionEncoder()
/**
* An encoder for nullable timestamp type.
* @since 1.6.0
*/
def TIMESTAMP: Encoder[java.sql.Timestamp] = ExpressionEncoder()
/**
* Creates an encoder for Java Bean of type T.
*
......
......@@ -18,6 +18,9 @@
package test.org.apache.spark.sql;
import java.io.Serializable;
import java.math.BigDecimal;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.*;
import scala.Tuple2;
......@@ -385,6 +388,20 @@ public class JavaDatasetSuite implements Serializable {
Assert.assertEquals(data3, ds3.collectAsList());
}
@Test
public void testPrimitiveEncoder() {
Encoder<Tuple5<Double, BigDecimal, Date, Timestamp, Float>> encoder =
Encoders.tuple(Encoders.DOUBLE(), Encoders.DECIMAL(), Encoders.DATE(), Encoders.TIMESTAMP(),
Encoders.FLOAT());
List<Tuple5<Double, BigDecimal, Date, Timestamp, Float>> data =
Arrays.asList(new Tuple5<Double, BigDecimal, Date, Timestamp, Float>(
1.7976931348623157E308, new BigDecimal("0.922337203685477589"),
Date.valueOf("1970-01-01"), new Timestamp(System.currentTimeMillis()), Float.MAX_VALUE));
Dataset<Tuple5<Double, BigDecimal, Date, Timestamp, Float>> ds =
context.createDataset(data, encoder);
Assert.assertEquals(data, ds.collectAsList());
}
@Test
public void testTypedAggregation() {
Encoder<Tuple2<String, Integer>> encoder = Encoders.tuple(Encoders.STRING(), Encoders.INT());
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment