diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index cb4c3f24b27218f3233495bcd497a479d563cbba..03e36c7871bcfa316b7303f8c039e55a596dbead 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -355,9 +355,9 @@ abstract class BinaryExpression extends Expression {
* @param f accepts two variable names and returns Java code to compute the output.
*/
protected def defineCodeGen(
- ctx: CodeGenContext,
- ev: GeneratedExpressionCode,
- f: (String, String) => String): String = {
+ ctx: CodeGenContext,
+ ev: GeneratedExpressionCode,
+ f: (String, String) => String): String = {
nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
s"${ev.primitive} = ${f(eval1, eval2)};"
})
@@ -372,9 +372,9 @@ abstract class BinaryExpression extends Expression {
* and returns Java code to compute the output.
*/
protected def nullSafeCodeGen(
- ctx: CodeGenContext,
- ev: GeneratedExpressionCode,
- f: (String, String) => String): String = {
+ ctx: CodeGenContext,
+ ev: GeneratedExpressionCode,
+ f: (String, String) => String): String = {
val eval1 = left.gen(ctx)
val eval2 = right.gen(ctx)
val resultCode = f(eval1.primitive, eval2.primitive)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala
index b00a1b26fa285294475f6abe26c79ef4579ad5d2..c37afc13f2d174e8581a3e9632d87aea4a6a5d7f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala
@@ -276,8 +276,6 @@ case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitC
override def dataType: DataType = DateType
- override def prettyName: String = "last_day"
-
override def nullSafeEval(date: Any): Any = {
val days = date.asInstanceOf[Int]
DateTimeUtils.getLastDayOfMonth(days)
@@ -289,12 +287,16 @@ case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitC
s"$dtu.getLastDayOfMonth($sd)"
})
}
+
+ override def prettyName: String = "last_day"
}
/**
* Returns the first date which is later than startDate and named as dayOfWeek.
* For example, NextDay(2015-07-27, Sunday) would return 2015-08-02, which is the first
- * sunday later than 2015-07-27.
+ * Sunday later than 2015-07-27.
+ *
+ * Allowed "dayOfWeek" is defined in [[DateTimeUtils.getDayOfWeekFromString]].
*/
case class NextDay(startDate: Expression, dayOfWeek: Expression)
extends BinaryExpression with ImplicitCastInputTypes {
@@ -318,22 +320,32 @@ case class NextDay(startDate: Expression, dayOfWeek: Expression)
override protected def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = {
nullSafeCodeGen(ctx, ev, (sd, dowS) => {
- val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
- val dow = ctx.freshName("dow")
- val genDow = if (right.foldable) {
- val dowVal = DateTimeUtils.getDayOfWeekFromString(
- dayOfWeek.eval(InternalRow.empty).asInstanceOf[UTF8String])
- s"int $dow = $dowVal;"
- } else {
- s"int $dow = $dtu.getDayOfWeekFromString($dowS);"
- }
- genDow + s"""
- if ($dow == -1) {
- ${ev.isNull} = true;
+ val dateTimeUtilClass = DateTimeUtils.getClass.getName.stripSuffix("$")
+ val dayOfWeekTerm = ctx.freshName("dayOfWeek")
+ if (dayOfWeek.foldable) {
+ val input = dayOfWeek.eval().asInstanceOf[UTF8String]
+ if ((input eq null) || DateTimeUtils.getDayOfWeekFromString(input) == -1) {
+ s"""
+ |${ev.isNull} = true;
+ """.stripMargin
} else {
- ${ev.primitive} = $dtu.getNextDateForDayOfWeek($sd, $dow);
+ val dayOfWeekValue = DateTimeUtils.getDayOfWeekFromString(input)
+ s"""
+ |${ev.primitive} = $dateTimeUtilClass.getNextDateForDayOfWeek($sd, $dayOfWeekValue);
+ """.stripMargin
}
- """
+ } else {
+ s"""
+ |int $dayOfWeekTerm = $dateTimeUtilClass.getDayOfWeekFromString($dowS);
+ |if ($dayOfWeekTerm == -1) {
+ | ${ev.isNull} = true;
+ |} else {
+ | ${ev.primitive} = $dateTimeUtilClass.getNextDateForDayOfWeek($sd, $dayOfWeekTerm);
+ |}
+ """.stripMargin
+ }
})
}
+
+ override def prettyName: String = "next_day"
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 85060b789355679783710ac05b982c01e902c4a9..064a1720c36e8268aa6b290e8bae1618284f35e0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -118,7 +118,7 @@ case class Literal protected (value: Any, dataType: DataType)
super.genCode(ctx, ev)
} else {
ev.isNull = "false"
- ev.primitive = s"${value}"
+ ev.primitive = s"${value}D"
""
}
case ByteType | ShortType =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 2e28fb9af9b65a4f5f69998722d5afd34468f560..8b0b80c26db17e89abd97045e5e13f8c97f2443a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -575,7 +575,7 @@ object DateTimeUtils {
}
/**
- * Returns Day of week from String. Starting from Thursday, marked as 0.
+ * Returns day of week from String. Starting from Thursday, marked as 0.
* (Because 1970-01-01 is Thursday).
*/
def getDayOfWeekFromString(string: UTF8String): Int = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 4d2d33765a269250f3b8aecff6e609815d097740..30c5769424bd737afc6a046f48582873c8bb54ba 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -32,6 +32,19 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
val d = new Date(sdf.parse("2015-04-08 13:10:15").getTime)
val ts = new Timestamp(sdf.parse("2013-11-08 13:10:15").getTime)
+ test("datetime function current_date") {
+ val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis())
+ val cd = CurrentDate().eval(EmptyRow).asInstanceOf[Int]
+ val d1 = DateTimeUtils.millisToDays(System.currentTimeMillis())
+ assert(d0 <= cd && cd <= d1 && d1 - d0 <= 1)
+ }
+
+ test("datetime function current_timestamp") {
+ val ct = DateTimeUtils.toJavaTimestamp(CurrentTimestamp().eval(EmptyRow).asInstanceOf[Long])
+ val t1 = System.currentTimeMillis()
+ assert(math.abs(t1 - ct.getTime) < 5000)
+ }
+
test("DayOfYear") {
val sdfDay = new SimpleDateFormat("D")
(2002 to 2004).foreach { y =>
@@ -264,14 +277,28 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
}
test("next_day") {
+ def testNextDay(input: String, dayOfWeek: String, output: String): Unit = {
+ checkEvaluation(
+ NextDay(Literal(Date.valueOf(input)), NonFoldableLiteral(dayOfWeek)),
+ DateTimeUtils.fromJavaDate(Date.valueOf(output)))
+ checkEvaluation(
+ NextDay(Literal(Date.valueOf(input)), Literal(dayOfWeek)),
+ DateTimeUtils.fromJavaDate(Date.valueOf(output)))
+ }
+ testNextDay("2015-07-23", "Mon", "2015-07-27")
+ testNextDay("2015-07-23", "mo", "2015-07-27")
+ testNextDay("2015-07-23", "Tue", "2015-07-28")
+ testNextDay("2015-07-23", "tu", "2015-07-28")
+ testNextDay("2015-07-23", "we", "2015-07-29")
+ testNextDay("2015-07-23", "wed", "2015-07-29")
+ testNextDay("2015-07-23", "Thu", "2015-07-30")
+ testNextDay("2015-07-23", "TH", "2015-07-30")
+ testNextDay("2015-07-23", "Fri", "2015-07-24")
+ testNextDay("2015-07-23", "fr", "2015-07-24")
+
+ checkEvaluation(NextDay(Literal(Date.valueOf("2015-07-23")), Literal("xx")), null)
+ checkEvaluation(NextDay(Literal.create(null, DateType), Literal("xx")), null)
checkEvaluation(
- NextDay(Literal(Date.valueOf("2015-07-23")), Literal("Thu")),
- DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-30")))
- checkEvaluation(
- NextDay(Literal(Date.valueOf("2015-07-23")), Literal("THURSDAY")),
- DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-30")))
- checkEvaluation(
- NextDay(Literal(Date.valueOf("2015-07-23")), Literal("th")),
- DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-30")))
+ NextDay(Literal(Date.valueOf("2015-07-23")), Literal.create(null, StringType)), null)
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DatetimeFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DatetimeFunctionsSuite.scala
deleted file mode 100644
index 1618c24871c605145f1da17b378f0966129ea01d..0000000000000000000000000000000000000000
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DatetimeFunctionsSuite.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.expressions
-
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
-
-class DatetimeFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
- test("datetime function current_date") {
- val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis())
- val cd = CurrentDate().eval(EmptyRow).asInstanceOf[Int]
- val d1 = DateTimeUtils.millisToDays(System.currentTimeMillis())
- assert(d0 <= cd && cd <= d1 && d1 - d0 <= 1)
- }
-
- test("datetime function current_timestamp") {
- val ct = DateTimeUtils.toJavaTimestamp(CurrentTimestamp().eval(EmptyRow).asInstanceOf[Long])
- val t1 = System.currentTimeMillis()
- assert(math.abs(t1 - ct.getTime) < 5000)
- }
-
-}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
index 136368bf5b368b5fdf1d1b61b5835013c3d7f005..0c8611d5ddefae69f32f49933540e164bb54500d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
@@ -82,6 +82,7 @@ trait ExpressionEvalHelper {
s"""
|Code generation of $expression failed:
|$e
+ |${e.getStackTraceString}
""".stripMargin)
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NonFoldableLiteral.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NonFoldableLiteral.scala
new file mode 100644
index 0000000000000000000000000000000000000000..0559fb80e7fcecdf2699e2eed5c9bdb80cdfe178
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/NonFoldableLiteral.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.codegen._
+import org.apache.spark.sql.types._
+
+
+/**
+ * A literal value that is not foldable. Used in expression codegen testing to test code path
+ * that behave differently based on foldable values.
+ */
+case class NonFoldableLiteral(value: Any, dataType: DataType)
+ extends LeafExpression with CodegenFallback {
+
+ override def foldable: Boolean = false
+ override def nullable: Boolean = true
+
+ override def toString: String = if (value != null) value.toString else "null"
+
+ override def eval(input: InternalRow): Any = value
+
+ override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = {
+ Literal.create(value, dataType).genCode(ctx, ev)
+ }
+}
+
+
+object NonFoldableLiteral {
+ def apply(value: Any): NonFoldableLiteral = {
+ val lit = Literal(value)
+ NonFoldableLiteral(lit.value, lit.dataType)
+ }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index d18558b510f0ba096a22bddfec976810ca4239fd..cec61b66b157cfa24b8cf167df63343bbc158dc8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2033,7 +2033,10 @@ object functions {
def hour(columnName: String): Column = hour(Column(columnName))
/**
- * Returns the last day of the month which the given date belongs to.
+ * Given a date column, returns the last day of the month which the given date belongs to.
+ * For example, input "2015-07-27" returns "2015-07-31" since July 31 is the last day of the
+ * month in July 2015.
+ *
* @group datetime_funcs
* @since 1.5.0
*/
@@ -2054,14 +2057,19 @@ object functions {
def minute(columnName: String): Column = minute(Column(columnName))
/**
- * Returns the first date which is later than given date sd and named as dow.
- * For example, `next_day('2015-07-27', "Sunday")` would return 2015-08-02, which is the
- * first Sunday later than 2015-07-27. The parameter dayOfWeek could be 2-letter, 3-letter,
- * or full name of the day of the week (e.g. Mo, tue, FRIDAY).
+ * Given a date column, returns the first date which is later than the value of the date column
+ * that is on the specified day of the week.
+ *
+ * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first
+ * Sunday after 2015-07-27.
+ *
+ * Day of the week parameter is case insensitive, and accepts:
+ * "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun".
+ *
* @group datetime_funcs
* @since 1.5.0
*/
- def next_day(sd: Column, dayOfWeek: String): Column = NextDay(sd.expr, lit(dayOfWeek).expr)
+ def next_day(date: Column, dayOfWeek: String): Column = NextDay(date.expr, lit(dayOfWeek).expr)
/**
* Extracts the seconds as an integer from a given date/timestamp/string.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index 001fcd035c82a3170474ae0be43390d538d5fd63..36820cbbc7e5ece87092884ac629569ad49802f9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql
import java.sql.{Timestamp, Date}
import java.text.SimpleDateFormat
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.functions._
class DateFunctionsSuite extends QueryTest {
@@ -27,6 +28,26 @@ class DateFunctionsSuite extends QueryTest {
import ctx.implicits._
+ test("function current_date") {
+ val df1 = Seq((1, 2), (3, 1)).toDF("a", "b")
+ val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis())
+ val d1 = DateTimeUtils.fromJavaDate(df1.select(current_date()).collect().head.getDate(0))
+ val d2 = DateTimeUtils.fromJavaDate(
+ ctx.sql("""SELECT CURRENT_DATE()""").collect().head.getDate(0))
+ val d3 = DateTimeUtils.millisToDays(System.currentTimeMillis())
+ assert(d0 <= d1 && d1 <= d2 && d2 <= d3 && d3 - d0 <= 1)
+ }
+
+ test("function current_timestamp") {
+ val df1 = Seq((1, 2), (3, 1)).toDF("a", "b")
+ checkAnswer(df1.select(countDistinct(current_timestamp())), Row(1))
+ // Execution in one query should return the same value
+ checkAnswer(ctx.sql("""SELECT CURRENT_TIMESTAMP() = CURRENT_TIMESTAMP()"""),
+ Row(true))
+ assert(math.abs(ctx.sql("""SELECT CURRENT_TIMESTAMP()""").collect().head.getTimestamp(
+ 0).getTime - System.currentTimeMillis()) < 5000)
+ }
+
val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
val sdfDate = new SimpleDateFormat("yyyy-MM-dd")
val d = new Date(sdf.parse("2015-04-08 13:10:15").getTime)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatetimeExpressionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatetimeExpressionsSuite.scala
deleted file mode 100644
index 44b915304533cddd56efe8ed1e89e97545525764..0000000000000000000000000000000000000000
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatetimeExpressionsSuite.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql
-
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.functions._
-
-class DatetimeExpressionsSuite extends QueryTest {
- private lazy val ctx = org.apache.spark.sql.test.TestSQLContext
-
- import ctx.implicits._
-
- lazy val df1 = Seq((1, 2), (3, 1)).toDF("a", "b")
-
- test("function current_date") {
- val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis())
- val d1 = DateTimeUtils.fromJavaDate(df1.select(current_date()).collect().head.getDate(0))
- val d2 = DateTimeUtils.fromJavaDate(
- ctx.sql("""SELECT CURRENT_DATE()""").collect().head.getDate(0))
- val d3 = DateTimeUtils.millisToDays(System.currentTimeMillis())
- assert(d0 <= d1 && d1 <= d2 && d2 <= d3 && d3 - d0 <= 1)
- }
-
- test("function current_timestamp") {
- checkAnswer(df1.select(countDistinct(current_timestamp())), Row(1))
- // Execution in one query should return the same value
- checkAnswer(ctx.sql("""SELECT CURRENT_TIMESTAMP() = CURRENT_TIMESTAMP()"""),
- Row(true))
- assert(math.abs(ctx.sql("""SELECT CURRENT_TIMESTAMP()""").collect().head.getTimestamp(
- 0).getTime - System.currentTimeMillis()) < 5000)
- }
-
-}