From 1a91be807802ec88c068f1090dafe8fbfb1c6d5c Mon Sep 17 00:00:00 2001
From: Reynold Xin <rxin@databricks.com>
Date: Mon, 28 Dec 2015 20:43:06 -0800
Subject: [PATCH] [SPARK-12547][SQL] Tighten scala style checker enforcement
for UDF registration
We use scalastyle:off to turn off style checks in certain places where it is not possible to follow the style guide. This is usually ok. However, in udf registration, we disable the checker for a large amount of code simply because some of them exceed 100 char line limit. It is better to just disable the line limit check rather than everything.
In this pull request, I only disabled line length check, and fixed a problem (lack explicit types for public methods).
Author: Reynold Xin <rxin@databricks.com>
Closes #10501 from rxin/SPARK-12547.
---
.../apache/spark/sql/UDFRegistration.scala | 53 +++++++++----------
.../org/apache/spark/sql/functions.scala | 6 ++-
2 files changed, 30 insertions(+), 29 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
index 051694c0d4..42c373ea72 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql
-import java.util.{List => JList, Map => JMap}
-
import scala.reflect.runtime.universe.TypeTag
import scala.util.Try
@@ -69,7 +67,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
udaf
}
- // scalastyle:off
+ // scalastyle:off line.size.limit
/* register 0-22 were generated by this script
@@ -102,7 +100,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
| * Register a user-defined function with ${i} arguments.
| * @since 1.3.0
| */
- |def register(name: String, f: UDF$i[$extTypeArgs, _], returnType: DataType) = {
+ |def register(name: String, f: UDF$i[$extTypeArgs, _], returnType: DataType): Unit = {
| functionRegistry.registerFunction(
| name,
| (e: Seq[Expression]) => ScalaUDF(f$anyCast.call($anyParams), returnType, e))
@@ -416,7 +414,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 1 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF1[_, _], returnType: DataType) = {
+ def register(name: String, f: UDF1[_, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF1[Any, Any]].call(_: Any), returnType, e))
@@ -426,7 +424,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 2 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF2[_, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF2[_, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF2[Any, Any, Any]].call(_: Any, _: Any), returnType, e))
@@ -436,7 +434,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 3 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF3[_, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF3[_, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF3[Any, Any, Any, Any]].call(_: Any, _: Any, _: Any), returnType, e))
@@ -446,7 +444,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 4 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF4[_, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF4[_, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF4[Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -456,7 +454,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 5 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF5[_, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF5[_, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -466,7 +464,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 6 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF6[_, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF6[_, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -476,7 +474,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 7 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -486,7 +484,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 8 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -496,7 +494,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 9 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -506,7 +504,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 10 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -516,7 +514,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 11 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF11[_, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF11[_, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF11[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -526,7 +524,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 12 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF12[_, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF12[_, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF12[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -536,7 +534,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 13 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF13[_, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF13[_, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -546,7 +544,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 14 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF14[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF14[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -556,7 +554,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 15 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF15[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -566,7 +564,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 16 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -576,7 +574,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 17 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF17[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -586,7 +584,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 18 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF18[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -596,7 +594,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 19 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF19[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -606,7 +604,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 20 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF20[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -616,7 +614,7 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 21 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF21[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
@@ -626,11 +624,12 @@ class UDFRegistration private[sql] (sqlContext: SQLContext) extends Logging {
* Register a user-defined function with 22 arguments.
* @since 1.3.0
*/
- def register(name: String, f: UDF22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType) = {
+ def register(name: String, f: UDF22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
functionRegistry.registerFunction(
name,
(e: Seq[Expression]) => ScalaUDF(f.asInstanceOf[UDF22[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any), returnType, e))
}
- // scalastyle:on
+ // scalastyle:on line.size.limit
+
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 65733dcf83..487191638f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2512,7 +2512,8 @@ object functions extends LegacyFunctions {
//////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////
- // scalastyle:off
+ // scalastyle:off line.size.limit
+ // scalastyle:off parameter.number
/* Use the following code to generate:
(0 to 10).map { x =>
@@ -2839,7 +2840,8 @@ object functions extends LegacyFunctions {
ScalaUDF(f, returnType, Seq(arg1.expr, arg2.expr, arg3.expr, arg4.expr, arg5.expr, arg6.expr, arg7.expr, arg8.expr, arg9.expr, arg10.expr))
}
- // scalastyle:on
+ // scalastyle:on parameter.number
+ // scalastyle:on line.size.limit
/**
* Call an user-defined function.
--
GitLab