Skip to content
Snippets Groups Projects
Commit d66642e3 authored by Reynold Xin's avatar Reynold Xin
Browse files

SPARK-1822: Some minor cleanup work on SchemaRDD.count()

Minor cleanup following #841.

Author: Reynold Xin <rxin@apache.org>

Closes #868 from rxin/schema-count and squashes the following commits:

5442651 [Reynold Xin] SPARK-1822: Some minor cleanup work on SchemaRDD.count()
parent 5c7faecd
No related branches found
No related tags found
No related merge requests found
......@@ -323,7 +323,10 @@ class SchemaRDD(RDD):
def count(self):
"""
Return the number of elements in this RDD.
Return the number of elements in this RDD. Unlike the base RDD
implementation of count, this implementation leverages the query
optimizer to compute the count on the SchemaRDD, which supports
features such as filter pushdown.
>>> srdd = sqlCtx.inferSchema(rdd)
>>> srdd.count()
......
......@@ -276,12 +276,12 @@ class SchemaRDD(
/**
* :: Experimental ::
* Overriding base RDD implementation to leverage query optimizer
* Return the number of elements in the RDD. Unlike the base RDD implementation of count, this
* implementation leverages the query optimizer to compute the count on the SchemaRDD, which
* supports features such as filter pushdown.
*/
@Experimental
override def count(): Long = {
groupBy()(Count(Literal(1))).collect().head.getLong(0)
}
override def count(): Long = groupBy()(Count(Literal(1))).collect().head.getLong(0)
/**
* :: Experimental ::
......
......@@ -124,7 +124,7 @@ class DslQuerySuite extends QueryTest {
}
test("zero count") {
assert(testData4.count() === 0)
assert(emptyTableData.count() === 0)
}
test("inner join where, one match per row") {
......
......@@ -47,7 +47,7 @@ object TestData {
(1, null) ::
(2, 2) :: Nil)
val testData4 = logical.LocalRelation('a.int, 'b.int)
val emptyTableData = logical.LocalRelation('a.int, 'b.int)
case class UpperCaseData(N: Int, L: String)
val upperCaseData =
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment