Skip to content
Snippets Groups Projects
Commit ed8d1531 authored by Reynold Xin's avatar Reynold Xin
Browse files

[SPARK-11793][SQL] Dataset should set the resolved encoders internally for maps.

I also wrote a test case -- but unfortunately the test case is not working due to SPARK-11795.

Author: Reynold Xin <rxin@databricks.com>

Closes #9784 from rxin/SPARK-11503.
parent 75a29229
No related branches found
No related tags found
No related merge requests found
...@@ -199,11 +199,12 @@ class Dataset[T] private[sql]( ...@@ -199,11 +199,12 @@ class Dataset[T] private[sql](
* @since 1.6.0 * @since 1.6.0
*/ */
def mapPartitions[U : Encoder](func: Iterator[T] => Iterator[U]): Dataset[U] = { def mapPartitions[U : Encoder](func: Iterator[T] => Iterator[U]): Dataset[U] = {
encoderFor[T].assertUnresolved()
new Dataset[U]( new Dataset[U](
sqlContext, sqlContext,
MapPartitions[T, U]( MapPartitions[T, U](
func, func,
encoderFor[T], resolvedTEncoder,
encoderFor[U], encoderFor[U],
encoderFor[U].schema.toAttributes, encoderFor[U].schema.toAttributes,
logicalPlan)) logicalPlan))
......
...@@ -73,6 +73,17 @@ class DatasetSuite extends QueryTest with SharedSQLContext { ...@@ -73,6 +73,17 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
("a", 2), ("b", 3), ("c", 4)) ("a", 2), ("b", 3), ("c", 4))
} }
ignore("Dataset should set the resolved encoders internally for maps") {
// TODO: Enable this once we fix SPARK-11793.
val ds: Dataset[(ClassData, Long)] = Seq(ClassData("one", 1), ClassData("two", 2)).toDS()
.map(c => ClassData(c.a, c.b + 1))
.groupBy(p => p).count()
checkAnswer(
ds,
(ClassData("one", 1), 1L), (ClassData("two", 2), 1L))
}
test("select") { test("select") {
val ds = Seq(("a", 1) , ("b", 2), ("c", 3)).toDS() val ds = Seq(("a", 1) , ("b", 2), ("c", 3)).toDS()
checkAnswer( checkAnswer(
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment