Skip to content
Snippets Groups Projects
Commit 20768dad authored by Davies Liu's avatar Davies Liu
Browse files

Revert "[SPARK-16186] [SQL] Support partition batch pruning with `IN`...

Revert "[SPARK-16186] [SQL] Support partition batch pruning with `IN` predicate in InMemoryTableScanExec"

This reverts commit a65bcbc2.
parent a65bcbc2
No related branches found
No related tags found
No related merge requests found
...@@ -79,10 +79,6 @@ private[sql] case class InMemoryTableScanExec( ...@@ -79,10 +79,6 @@ private[sql] case class InMemoryTableScanExec(
case IsNull(a: Attribute) => statsFor(a).nullCount > 0 case IsNull(a: Attribute) => statsFor(a).nullCount > 0
case IsNotNull(a: Attribute) => statsFor(a).count - statsFor(a).nullCount > 0 case IsNotNull(a: Attribute) => statsFor(a).count - statsFor(a).nullCount > 0
case In(a: AttributeReference, list: Seq[Expression]) if list.forall(_.isInstanceOf[Literal]) =>
list.map(l => statsFor(a).lowerBound <= l.asInstanceOf[Literal] &&
l.asInstanceOf[Literal] <= statsFor(a).upperBound).reduce(_ || _)
} }
val partitionFilters: Seq[Expression] = { val partitionFilters: Seq[Expression] = {
......
...@@ -65,18 +65,11 @@ class PartitionBatchPruningSuite ...@@ -65,18 +65,11 @@ class PartitionBatchPruningSuite
}, 5).toDF() }, 5).toDF()
pruningData.createOrReplaceTempView("pruningData") pruningData.createOrReplaceTempView("pruningData")
spark.catalog.cacheTable("pruningData") spark.catalog.cacheTable("pruningData")
val pruningStringData = sparkContext.makeRDD((100 to 200).map { key =>
StringData(key.toString)
}, 5).toDF()
pruningStringData.createOrReplaceTempView("pruningStringData")
spark.catalog.cacheTable("pruningStringData")
} }
override protected def afterEach(): Unit = { override protected def afterEach(): Unit = {
try { try {
spark.catalog.uncacheTable("pruningData") spark.catalog.uncacheTable("pruningData")
spark.catalog.uncacheTable("pruningStringData")
} finally { } finally {
super.afterEach() super.afterEach()
} }
...@@ -117,23 +110,9 @@ class PartitionBatchPruningSuite ...@@ -117,23 +110,9 @@ class PartitionBatchPruningSuite
88 to 100 88 to 100
} }
// Support `IN` predicate // With unsupported predicate
checkBatchPruning("SELECT key FROM pruningData WHERE key IN (1)", 1, 1)(Seq(1))
checkBatchPruning("SELECT key FROM pruningData WHERE key IN (1, 2)", 1, 1)(Seq(1, 2))
checkBatchPruning("SELECT key FROM pruningData WHERE key IN (1, 11)", 1, 2)(Seq(1, 11))
checkBatchPruning("SELECT key FROM pruningData WHERE key IN (1, 21, 41, 61, 81)", 5, 5)(
Seq(1, 21, 41, 61, 81))
checkBatchPruning("SELECT CAST(s AS INT) FROM pruningStringData WHERE s = '100'", 1, 1)(Seq(100))
checkBatchPruning("SELECT CAST(s AS INT) FROM pruningStringData WHERE s < '102'", 1, 1)(
Seq(100, 101))
checkBatchPruning(
"SELECT CAST(s AS INT) FROM pruningStringData WHERE s IN ('99', '150', '201')", 1, 1)(
Seq(150))
// With unsupported `InSet` predicate
{ {
val seq = (1 to 30).mkString(", ") val seq = (1 to 30).mkString(", ")
checkBatchPruning(s"SELECT key FROM pruningData WHERE key IN ($seq)", 5, 10)(1 to 30)
checkBatchPruning(s"SELECT key FROM pruningData WHERE NOT (key IN ($seq))", 5, 10)(31 to 100) checkBatchPruning(s"SELECT key FROM pruningData WHERE NOT (key IN ($seq))", 5, 10)(31 to 100)
checkBatchPruning(s"SELECT key FROM pruningData WHERE NOT (key IN ($seq)) AND key > 88", 1, 2) { checkBatchPruning(s"SELECT key FROM pruningData WHERE NOT (key IN ($seq)) AND key > 88", 1, 2) {
89 to 100 89 to 100
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment