From ef43b4ed87894982678fcc6f2c61cf1487ee9e14 Mon Sep 17 00:00:00 2001
From: Davies Liu <davies@databricks.com>
Date: Fri, 17 Jun 2016 00:34:33 -0700
Subject: [PATCH] [SPARK-15811][SQL] fix the Python UDF in Scala 2.10
## What changes were proposed in this pull request?
Iterator can't be serialized in Scala 2.10, we should force it into a array to make sure that .
## How was this patch tested?
Build with Scala 2.10 and ran all the Python unit tests manually (will be covered by a jenkins build).
Author: Davies Liu <davies@databricks.com>
Closes #13717 from davies/fix_udf_210.
---
.../apache/spark/sql/execution/python/ExtractPythonUDFs.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
index 668470ee6a..87583c8234 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
@@ -132,7 +132,7 @@ private[spark] object ExtractPythonUDFs extends Rule[SparkPlan] {
val validUdfs = udfs.filter { case udf =>
// Check to make sure that the UDF can be evaluated with only the input of this child.
udf.references.subsetOf(child.outputSet)
- }
+ }.toArray // Turn it into an array since iterators cannot be serialized in Scala 2.10
if (validUdfs.nonEmpty) {
val resultAttrs = udfs.zipWithIndex.map { case (u, i) =>
AttributeReference(s"pythonUDF$i", u.dataType)()
--
GitLab