From 1e799d617a28cd0eaa8f22d103ea8248c4655ae5 Mon Sep 17 00:00:00 2001
From: Ankur Dave <ankurdave@gmail.com>
Date: Fri, 11 Dec 2015 19:07:48 -0800
Subject: [PATCH] [SPARK-12298][SQL] Fix infinite loop in
DataFrame.sortWithinPartitions
Modifies the String overload to call the Column overload and ensures this is called in a test.
Author: Ankur Dave <ankurdave@gmail.com>
Closes #10271 from ankurdave/SPARK-12298.
---
sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala | 2 +-
.../src/test/scala/org/apache/spark/sql/DataFrameSuite.scala | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index da180a2ba0..497bd48266 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -609,7 +609,7 @@ class DataFrame private[sql](
*/
@scala.annotation.varargs
def sortWithinPartitions(sortCol: String, sortCols: String*): DataFrame = {
- sortWithinPartitions(sortCol, sortCols : _*)
+ sortWithinPartitions((sortCol +: sortCols).map(Column(_)) : _*)
}
/**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 5353fefaf4..c0bbf73ab1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -1090,8 +1090,8 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
}
// Distribute into one partition and order by. This partition should contain all the values.
- val df6 = data.repartition(1, $"a").sortWithinPartitions($"b".asc)
- // Walk each partition and verify that it is sorted descending and not globally sorted.
+ val df6 = data.repartition(1, $"a").sortWithinPartitions("b")
+ // Walk each partition and verify that it is sorted ascending and not globally sorted.
df6.rdd.foreachPartition { p =>
var previousValue: Int = -1
var allSequential: Boolean = true
--
GitLab