diff --git a/repl/scala-2.10/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/scala-2.10/src/test/scala/org/apache/spark/repl/ReplSuite.scala
index 50fd43a418bcaf8409b6cdcabe1cbcb600892c46..f150fec7db9455c484ac8819f41d7c3f20f43aab 100644
--- a/repl/scala-2.10/src/test/scala/org/apache/spark/repl/ReplSuite.scala
+++ b/repl/scala-2.10/src/test/scala/org/apache/spark/repl/ReplSuite.scala
@@ -267,6 +267,17 @@ class ReplSuite extends SparkFunSuite {
assertDoesNotContain("Exception", output)
}
+ test("SPARK-8461 SQL with codegen") {
+ val output = runInterpreter("local",
+ """
+ |val sqlContext = new org.apache.spark.sql.SQLContext(sc)
+ |sqlContext.setConf("spark.sql.codegen", "true")
+ |sqlContext.range(0, 100).filter('id > 50).count()
+ """.stripMargin)
+ assertContains("Long = 49", output)
+ assertDoesNotContain("java.lang.ClassNotFoundException", output)
+ }
+
test("SPARK-2632 importing a method from non serializable class and not using it.") {
val output = runInterpreter("local",
"""
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index ab850d17a6dd3b230c04aa42ba6a3ad6d3c5c99b..bd5475d2066fc4e0e28030166a895d9672c3e61d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -203,6 +203,11 @@ class CodeGenContext {
def isPrimitiveType(dt: DataType): Boolean = primitiveTypes.contains(dt)
}
+
+abstract class GeneratedClass {
+ def generate(expressions: Array[Expression]): Any
+}
+
/**
* A base class for generators of byte code to perform expression evaluation. Includes a set of
* helpers for referring to Catalyst types and building trees that perform evaluation of individual
@@ -214,11 +219,6 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
protected val mutableRowType: String = classOf[MutableRow].getName
protected val genericMutableRowType: String = classOf[GenericMutableRow].getName
- /**
- * Can be flipped on manually in the console to add (expensive) expression evaluation trace code.
- */
- var debugLogging = false
-
/**
* Generates a class for a given input expression. Called when there is not cached code
* already available.
@@ -239,10 +239,14 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
*
* It will track the time used to compile
*/
- protected def compile(code: String): Class[_] = {
+ protected def compile(code: String): GeneratedClass = {
val startTime = System.nanoTime()
- val clazz = try {
- new ClassBodyEvaluator(code).getClazz()
+ val evaluator = new ClassBodyEvaluator()
+ evaluator.setParentClassLoader(getClass.getClassLoader)
+ evaluator.setDefaultImports(Array("org.apache.spark.sql.catalyst.InternalRow"))
+ evaluator.setExtendedClass(classOf[GeneratedClass])
+ try {
+ evaluator.cook(code)
} catch {
case e: Exception =>
logError(s"failed to compile:\n $code", e)
@@ -251,7 +255,7 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
val endTime = System.nanoTime()
def timeMs: Double = (endTime - startTime).toDouble / 1000000
logDebug(s"Code (${code.size} bytes) compiled in $timeMs ms")
- clazz
+ evaluator.getClazz().newInstance().asInstanceOf[GeneratedClass]
}
/**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala
index 573a9ea0a54713f0d2f8b23f7e581c09ddf7e2dc..e75e82d380541bb836423e6e12f86ad257923c38 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala
@@ -47,9 +47,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu
"""
}.mkString("\n")
val code = s"""
- import org.apache.spark.sql.catalyst.InternalRow;
-
- public SpecificProjection generate($exprType[] expr) {
+ public Object generate($exprType[] expr) {
return new SpecificProjection(expr);
}
@@ -85,10 +83,8 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu
logDebug(s"code for ${expressions.mkString(",")}:\n$code")
val c = compile(code)
- // fetch the only one method `generate(Expression[])`
- val m = c.getDeclaredMethods()(0)
() => {
- m.invoke(c.newInstance(), ctx.references.toArray).asInstanceOf[BaseMutableProjection]
+ c.generate(ctx.references.toArray).asInstanceOf[MutableProjection]
}
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
index 3e9ee60f3303759acbc9af5edab908c2a442a427..7ed2c5addec9b84c89b4c4f476e8a349cbbf31ad 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
@@ -76,8 +76,6 @@ object GenerateOrdering
}.mkString("\n")
val code = s"""
- import org.apache.spark.sql.catalyst.InternalRow;
-
public SpecificOrdering generate($exprType[] expr) {
return new SpecificOrdering(expr);
}
@@ -100,9 +98,6 @@ object GenerateOrdering
logDebug(s"Generated Ordering: $code")
- val c = compile(code)
- // fetch the only one method `generate(Expression[])`
- val m = c.getDeclaredMethods()(0)
- m.invoke(c.newInstance(), ctx.references.toArray).asInstanceOf[BaseOrdering]
+ compile(code).generate(ctx.references.toArray).asInstanceOf[BaseOrdering]
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala
index dad4364bdd94afa6f1606bec8083fc59df6d60fd..3ebc2c147579bef644d323f6302eab8f7f1dfa8f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala
@@ -17,7 +17,6 @@
package org.apache.spark.sql.catalyst.expressions.codegen
-import org.apache.spark.sql.catalyst
import org.apache.spark.sql.catalyst.expressions._
/**
@@ -41,8 +40,6 @@ object GeneratePredicate extends CodeGenerator[Expression, (InternalRow) => Bool
val ctx = newCodeGenContext()
val eval = predicate.gen(ctx)
val code = s"""
- import org.apache.spark.sql.catalyst.InternalRow;
-
public SpecificPredicate generate($exprType[] expr) {
return new SpecificPredicate(expr);
}
@@ -62,10 +59,7 @@ object GeneratePredicate extends CodeGenerator[Expression, (InternalRow) => Bool
logDebug(s"Generated predicate '$predicate':\n$code")
- val c = compile(code)
- // fetch the only one method `generate(Expression[])`
- val m = c.getDeclaredMethods()(0)
- val p = m.invoke(c.newInstance(), ctx.references.toArray).asInstanceOf[Predicate]
+ val p = compile(code).generate(ctx.references.toArray).asInstanceOf[Predicate]
(r: InternalRow) => p.eval(r)
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala
index 8b5dc194be31f97bf380bba7a57ab3fd3f456c96..2e20eda1a300254a533e09bd4a27c1843d0cea54 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala
@@ -147,8 +147,6 @@ object GenerateProjection extends CodeGenerator[Seq[Expression], Projection] {
}.mkString("\n")
val code = s"""
- import org.apache.spark.sql.catalyst.InternalRow;
-
public SpecificProjection generate($exprType[] expr) {
return new SpecificProjection(expr);
}
@@ -220,9 +218,6 @@ object GenerateProjection extends CodeGenerator[Seq[Expression], Projection] {
logDebug(s"MutableRow, initExprs: ${expressions.mkString(",")} code:\n${code}")
- val c = compile(code)
- // fetch the only one method `generate(Expression[])`
- val m = c.getDeclaredMethods()(0)
- m.invoke(c.newInstance(), ctx.references.toArray).asInstanceOf[Projection]
+ compile(code).generate(ctx.references.toArray).asInstanceOf[Projection]
}
}