Skip to content
Snippets Groups Projects
Commit f84b228c authored by Venkata Ramana Gollamudi's avatar Venkata Ramana Gollamudi Committed by Michael Armbrust
Browse files

[SPARK-3593][SQL] Add support for sorting BinaryType

BinaryType is derived from NativeType and added Ordering support.

Author: Venkata Ramana G <ramana.gollamudihuawei.com>

Author: Venkata Ramana Gollamudi <ramana.gollamudi@huawei.com>

Closes #2617 from gvramana/binarytype_sort and squashes the following commits:

1cf26f3 [Venkata Ramana Gollamudi] Supported Sorting of BinaryType
parent f315fb7e
No related branches found
No related tags found
No related merge requests found
...@@ -157,8 +157,18 @@ case object StringType extends NativeType with PrimitiveType { ...@@ -157,8 +157,18 @@ case object StringType extends NativeType with PrimitiveType {
def simpleString: String = "string" def simpleString: String = "string"
} }
case object BinaryType extends DataType with PrimitiveType { case object BinaryType extends NativeType with PrimitiveType {
private[sql] type JvmType = Array[Byte] private[sql] type JvmType = Array[Byte]
@transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
private[sql] val ordering = new Ordering[JvmType] {
def compare(x: Array[Byte], y: Array[Byte]): Int = {
for (i <- 0 until x.length; if i < y.length) {
val res = x(i).compareTo(y(i))
if (res != 0) return res
}
return x.length - y.length
}
}
def simpleString: String = "binary" def simpleString: String = "binary"
} }
......
...@@ -190,6 +190,14 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll { ...@@ -190,6 +190,14 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
sql("SELECT * FROM testData2 ORDER BY a DESC, b ASC"), sql("SELECT * FROM testData2 ORDER BY a DESC, b ASC"),
Seq((3,1), (3,2), (2,1), (2,2), (1,1), (1,2))) Seq((3,1), (3,2), (2,1), (2,2), (1,1), (1,2)))
checkAnswer(
sql("SELECT b FROM binaryData ORDER BY a ASC"),
(1 to 5).map(Row(_)).toSeq)
checkAnswer(
sql("SELECT b FROM binaryData ORDER BY a DESC"),
(1 to 5).map(Row(_)).toSeq.reverse)
checkAnswer( checkAnswer(
sql("SELECT * FROM arrayData ORDER BY data[0] ASC"), sql("SELECT * FROM arrayData ORDER BY data[0] ASC"),
arrayData.collect().sortBy(_.data(0)).toSeq) arrayData.collect().sortBy(_.data(0)).toSeq)
......
...@@ -54,6 +54,16 @@ object TestData { ...@@ -54,6 +54,16 @@ object TestData {
TestData2(3, 2) :: Nil) TestData2(3, 2) :: Nil)
testData2.registerTempTable("testData2") testData2.registerTempTable("testData2")
case class BinaryData(a: Array[Byte], b: Int)
val binaryData: SchemaRDD =
TestSQLContext.sparkContext.parallelize(
BinaryData("12".getBytes(), 1) ::
BinaryData("22".getBytes(), 5) ::
BinaryData("122".getBytes(), 3) ::
BinaryData("121".getBytes(), 2) ::
BinaryData("123".getBytes(), 4) :: Nil)
binaryData.registerTempTable("binaryData")
// TODO: There is no way to express null primitives as case classes currently... // TODO: There is no way to express null primitives as case classes currently...
val testData3 = val testData3 =
logical.LocalRelation('a.int, 'b.int).loadData( logical.LocalRelation('a.int, 'b.int).loadData(
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment