diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index c336a0c8eab7a14b59265035267ffa4c05c9a1aa..87719d9ee2bc40ecc17f0e9f3f43673f3996d24a 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -653,7 +653,6 @@ quotedIdentifier
number
: MINUS? DECIMAL_VALUE #decimalLiteral
- | MINUS? SCIENTIFIC_DECIMAL_VALUE #scientificDecimalLiteral
| MINUS? INTEGER_VALUE #integerLiteral
| MINUS? BIGINT_LITERAL #bigIntLiteral
| MINUS? SMALLINT_LITERAL #smallIntLiteral
@@ -944,12 +943,8 @@ INTEGER_VALUE
;
DECIMAL_VALUE
- : DECIMAL_DIGITS {isValidDecimal()}?
- ;
-
-SCIENTIFIC_DECIMAL_VALUE
: DIGIT+ EXPONENT
- | DECIMAL_DIGITS EXPONENT {isValidDecimal()}?
+ | DECIMAL_DIGITS EXPONENT? {isValidDecimal()}?
;
DOUBLE_LITERAL
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index cd0c70a49150dd3142301bb0378e94c112f06a02..bf3f30279a6fe9eab6cfa5963047f98e33158991 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -1282,14 +1282,6 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging {
}
}
- /**
- * Create a double literal for a number denoted in scientific notation.
- */
- override def visitScientificDecimalLiteral(
- ctx: ScientificDecimalLiteralContext): Literal = withOrigin(ctx) {
- Literal(ctx.getText.toDouble)
- }
-
/**
* Create a decimal literal for a regular decimal number.
*/
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 3718ac5f1e77b7630630480adc63c8112fa5d558..0fb1138478a9b5ee423c3f552adf07987455431d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -352,6 +352,10 @@ class ExpressionParserSuite extends PlanTest {
}
test("literals") {
+ def testDecimal(value: String): Unit = {
+ assertEqual(value, Literal(BigDecimal(value).underlying))
+ }
+
// NULL
assertEqual("null", Literal(null))
@@ -362,20 +366,18 @@ class ExpressionParserSuite extends PlanTest {
// Integral should have the narrowest possible type
assertEqual("787324", Literal(787324))
assertEqual("7873247234798249234", Literal(7873247234798249234L))
- assertEqual("78732472347982492793712334",
- Literal(BigDecimal("78732472347982492793712334").underlying()))
+ testDecimal("78732472347982492793712334")
// Decimal
- assertEqual("7873247234798249279371.2334",
- Literal(BigDecimal("7873247234798249279371.2334").underlying()))
+ testDecimal("7873247234798249279371.2334")
// Scientific Decimal
- assertEqual("9.0e1", 90d)
- assertEqual(".9e+2", 90d)
- assertEqual("0.9e+2", 90d)
- assertEqual("900e-1", 90d)
- assertEqual("900.0E-1", 90d)
- assertEqual("9.e+1", 90d)
+ testDecimal("9.0e1")
+ testDecimal(".9e+2")
+ testDecimal("0.9e+2")
+ testDecimal("900e-1")
+ testDecimal("900.0E-1")
+ testDecimal("9.e+1")
intercept(".e3")
// Tiny Int Literal
@@ -395,8 +397,6 @@ class ExpressionParserSuite extends PlanTest {
assertEqual("10.0D", Literal(10.0D))
intercept("-1.8E308D", s"does not fit in range")
intercept("1.8E308D", s"does not fit in range")
- // TODO we need to figure out if we should throw an exception here!
- assertEqual("1E309", Literal(Double.PositiveInfinity))
// BigDecimal Literal
assertEqual("90912830918230182310293801923652346786BD",
diff --git a/sql/core/src/test/resources/sql-tests/inputs/literals.sql b/sql/core/src/test/resources/sql-tests/inputs/literals.sql
index 40dceb19cfc5be9ac2dc5e6d9efd5307f5f4d154..37b4b7606d12bbf6603f00d0f57311f495a9e6a9 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/literals.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/literals.sql
@@ -50,14 +50,14 @@ select 1D, 1.2D, 1e10, 1.5e5, .10D, 0.10D, .1e5, .9e+2, 0.9e+2, 900e-1, 9.e+1;
select -1D, -1.2D, -1e10, -1.5e5, -.10D, -0.10D, -.1e5;
-- negative double
select .e3;
--- inf and -inf
+-- very large decimals (overflowing double).
select 1E309, -1E309;
-- decimal parsing
select 0.3, -0.8, .5, -.18, 0.1111, .1111;
--- super large scientific notation numbers should still be valid doubles
-select 123456789012345678901234567890123456789e10, 123456789012345678901234567890123456789.1e10;
+-- super large scientific notation double literals should still be valid doubles
+select 123456789012345678901234567890123456789e10d, 123456789012345678901234567890123456789.1e10d;
-- string
select "Hello Peter!", 'hello lee!';
@@ -103,3 +103,5 @@ select x'2379ACFe';
-- invalid hexadecimal binary literal
select X'XuZ';
+-- Hive literal_double test.
+SELECT 3.14, -3.14, 3.14e8, 3.14e-8, -3.14e8, -3.14e-8, 3.14e+8, 3.14E8, 3.14E-8;
diff --git a/sql/core/src/test/resources/sql-tests/results/arithmetic.sql.out b/sql/core/src/test/resources/sql-tests/results/arithmetic.sql.out
index 6abe048af477dc828e2118434461231238e5dcd3..ce42c016a7100214304a15b992429a20ff40e63d 100644
--- a/sql/core/src/test/resources/sql-tests/results/arithmetic.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/arithmetic.sql.out
@@ -29,7 +29,7 @@ struct<-5.2:decimal(2,1)>
-- !query 3
select +6.8e0
-- !query 3 schema
-struct<6.8:double>
+struct<6.8:decimal(2,1)>
-- !query 3 output
6.8
diff --git a/sql/core/src/test/resources/sql-tests/results/literals.sql.out b/sql/core/src/test/resources/sql-tests/results/literals.sql.out
index e2d8daef9868f561b8a30613c53600aa9b042307..95d4413148f644b614e3f379bcfd6173bdc86149 100644
--- a/sql/core/src/test/resources/sql-tests/results/literals.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/literals.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 42
+-- Number of queries: 43
-- !query 0
@@ -167,17 +167,17 @@ select 1234567890123456789012345678901234567890.0
-- !query 17
select 1D, 1.2D, 1e10, 1.5e5, .10D, 0.10D, .1e5, .9e+2, 0.9e+2, 900e-1, 9.e+1
-- !query 17 schema
-struct<1.0:double,1.2:double,1.0E10:double,150000.0:double,0.1:double,0.1:double,10000.0:double,90.0:double,90.0:double,90.0:double,90.0:double>
+struct<1.0:double,1.2:double,1E+10:decimal(1,-10),1.5E+5:decimal(2,-4),0.1:double,0.1:double,1E+4:decimal(1,-4),9E+1:decimal(1,-1),9E+1:decimal(1,-1),90.0:decimal(3,1),9E+1:decimal(1,-1)>
-- !query 17 output
-1.0 1.2 1.0E10 150000.0 0.1 0.1 10000.0 90.0 90.0 90.0 90.0
+1.0 1.2 10000000000 150000 0.1 0.1 10000 90 90 90 90
-- !query 18
select -1D, -1.2D, -1e10, -1.5e5, -.10D, -0.10D, -.1e5
-- !query 18 schema
-struct<-1.0:double,-1.2:double,-1.0E10:double,-150000.0:double,-0.1:double,-0.1:double,-10000.0:double>
+struct<-1.0:double,-1.2:double,-1E+10:decimal(1,-10),-1.5E+5:decimal(2,-4),-0.1:double,-0.1:double,-1E+4:decimal(1,-4)>
-- !query 18 output
--1.0 -1.2 -1.0E10 -150000.0 -0.1 -0.1 -10000.0
+-1.0 -1.2 -10000000000 -150000 -0.1 -0.1 -10000
-- !query 19
@@ -197,9 +197,9 @@ select .e3
-- !query 20
select 1E309, -1E309
-- !query 20 schema
-struct<Infinity:double,-Infinity:double>
+struct<1E+309:decimal(1,-309),-1E+309:decimal(1,-309)>
-- !query 20 output
-Infinity -Infinity
+1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 -1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
-- !query 21
@@ -211,7 +211,7 @@ struct<0.3:decimal(1,1),-0.8:decimal(1,1),0.5:decimal(1,1),-0.18:decimal(2,2),0.
-- !query 22
-select 123456789012345678901234567890123456789e10, 123456789012345678901234567890123456789.1e10
+select 123456789012345678901234567890123456789e10d, 123456789012345678901234567890123456789.1e10d
-- !query 22 schema
struct<1.2345678901234568E48:double,1.2345678901234568E48:double>
-- !query 22 output
@@ -408,3 +408,11 @@ contains illegal character for hexBinary: 0XuZ(line 1, pos 7)
== SQL ==
select X'XuZ'
-------^^^
+
+
+-- !query 42
+SELECT 3.14, -3.14, 3.14e8, 3.14e-8, -3.14e8, -3.14e-8, 3.14e+8, 3.14E8, 3.14E-8
+-- !query 42 schema
+struct<3.14:decimal(3,2),-3.14:decimal(3,2),3.14E+8:decimal(3,-6),3.14E-8:decimal(10,10),-3.14E+8:decimal(3,-6),-3.14E-8:decimal(10,10),3.14E+8:decimal(3,-6),3.14E+8:decimal(3,-6),3.14E-8:decimal(10,10)>
+-- !query 42 output
+3.14 -3.14 314000000 0.0000000314 -314000000 -0.0000000314 314000000 314000000 0.0000000314
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index bebcb8f8016b14e924ecb7d8593d3b4109cb34ef..f5d10de8cd2bff59f672277bb98fe513eeb34371 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -555,6 +555,9 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
"varchar_2",
"varchar_join1",
+ // This test assumes we parse scientific decimals as doubles (we parse them as decimals)
+ "literal_double",
+
// These tests are duplicates of joinXYZ
"auto_join0",
"auto_join1",
@@ -832,7 +835,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
"leftsemijoin_mr",
"limit_pushdown_negative",
"lineage1",
- "literal_double",
"literal_ints",
"literal_string",
"load_dyn_part1",