From 8e96acf71c7bf1686aeca842f626f66c1cc8117f Mon Sep 17 00:00:00 2001
From: Xiao Li <gatorsmile@gmail.com>
Date: Sat, 10 Jun 2017 10:28:14 -0700
Subject: [PATCH] [SPARK-20211][SQL] Fix the Precision and Scale of Decimal
 Values when the Input is BigDecimal between -1.0 and 1.0

### What changes were proposed in this pull request?
The precision and scale of decimal values are wrong when the input is BigDecimal between -1.0 and 1.0.

The BigDecimal's precision is the digit count starts from the leftmost nonzero digit based on the [JAVA's BigDecimal definition](https://docs.oracle.com/javase/7/docs/api/java/math/BigDecimal.html). However, our Decimal decision follows the database decimal standard, which is the total number of digits, including both to the left and the right of the decimal point. Thus, this PR is to fix the issue by doing the conversion.

Before this PR, the following queries failed:
```SQL
select 1 > 0.0001
select floor(0.0001)
select ceil(0.0001)
```

### How was this patch tested?
Added test cases.

Author: Xiao Li <gatorsmile@gmail.com>

Closes #18244 from gatorsmile/bigdecimal.
---
 .../org/apache/spark/sql/types/Decimal.scala  | 10 +++-
 .../apache/spark/sql/types/DecimalSuite.scala | 10 ++++
 .../resources/sql-tests/inputs/operators.sql  |  7 +++
 .../sql-tests/results/operators.sql.out       | 58 ++++++++++++++++---
 4 files changed, 75 insertions(+), 10 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 80916ee9c5..1f1fb51add 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -126,7 +126,15 @@ final class Decimal extends Ordered[Decimal] with Serializable {
   def set(decimal: BigDecimal): Decimal = {
     this.decimalVal = decimal
     this.longVal = 0L
-    this._precision = decimal.precision
+    if (decimal.precision <= decimal.scale) {
+      // For Decimal, we expect the precision is equal to or large than the scale, however,
+      // in BigDecimal, the digit count starts from the leftmost nonzero digit of the exact
+      // result. For example, the precision of 0.01 equals to 1 based on the definition, but
+      // the scale is 2. The expected precision should be 3.
+      this._precision = decimal.scale + 1
+    } else {
+      this._precision = decimal.precision
+    }
     this._scale = decimal.scale
     this
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
index 93c231e30b..144f3d688d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala
@@ -32,6 +32,16 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester {
 
   test("creating decimals") {
     checkDecimal(new Decimal(), "0", 1, 0)
+    checkDecimal(Decimal(BigDecimal("0.09")), "0.09", 3, 2)
+    checkDecimal(Decimal(BigDecimal("0.9")), "0.9", 2, 1)
+    checkDecimal(Decimal(BigDecimal("0.90")), "0.90", 3, 2)
+    checkDecimal(Decimal(BigDecimal("0.0")), "0.0", 2, 1)
+    checkDecimal(Decimal(BigDecimal("0")), "0", 1, 0)
+    checkDecimal(Decimal(BigDecimal("1.0")), "1.0", 2, 1)
+    checkDecimal(Decimal(BigDecimal("-0.09")), "-0.09", 3, 2)
+    checkDecimal(Decimal(BigDecimal("-0.9")), "-0.9", 2, 1)
+    checkDecimal(Decimal(BigDecimal("-0.90")), "-0.90", 3, 2)
+    checkDecimal(Decimal(BigDecimal("-1.0")), "-1.0", 2, 1)
     checkDecimal(Decimal(BigDecimal("10.030")), "10.030", 5, 3)
     checkDecimal(Decimal(BigDecimal("10.030"), 4, 1), "10.0", 4, 1)
     checkDecimal(Decimal(BigDecimal("-9.95"), 4, 1), "-10.0", 4, 1)
diff --git a/sql/core/src/test/resources/sql-tests/inputs/operators.sql b/sql/core/src/test/resources/sql-tests/inputs/operators.sql
index 7e3b86b76a..75a0256ad7 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/operators.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/operators.sql
@@ -65,8 +65,15 @@ select ceiling(0);
 select ceiling(1);
 select ceil(1234567890123456);
 select ceiling(1234567890123456);
+select ceil(0.01);
+select ceiling(-0.10);
 
 -- floor
 select floor(0);
 select floor(1);
 select floor(1234567890123456);
+select floor(0.01);
+select floor(-0.10);
+
+-- comparison operator
+select 1 > 0.00001
diff --git a/sql/core/src/test/resources/sql-tests/results/operators.sql.out b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
index 28cfb74419..57e8a612fa 100644
--- a/sql/core/src/test/resources/sql-tests/results/operators.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 45
+-- Number of queries: 50
 
 
 -- !query 0
@@ -351,24 +351,64 @@ struct<CEIL(1234567890123456):bigint>
 
 
 -- !query 42
-select floor(0)
+select ceil(0.01)
 -- !query 42 schema
-struct<FLOOR(CAST(0 AS DOUBLE)):bigint>
+struct<CEIL(0.01):decimal(1,0)>
 -- !query 42 output
-0
+1
 
 
 -- !query 43
-select floor(1)
+select ceiling(-0.10)
 -- !query 43 schema
-struct<FLOOR(CAST(1 AS DOUBLE)):bigint>
+struct<CEIL(-0.10):decimal(1,0)>
 -- !query 43 output
-1
+0
 
 
 -- !query 44
-select floor(1234567890123456)
+select floor(0)
 -- !query 44 schema
-struct<FLOOR(1234567890123456):bigint>
+struct<FLOOR(CAST(0 AS DOUBLE)):bigint>
 -- !query 44 output
+0
+
+
+-- !query 45
+select floor(1)
+-- !query 45 schema
+struct<FLOOR(CAST(1 AS DOUBLE)):bigint>
+-- !query 45 output
+1
+
+
+-- !query 46
+select floor(1234567890123456)
+-- !query 46 schema
+struct<FLOOR(1234567890123456):bigint>
+-- !query 46 output
 1234567890123456
+
+
+-- !query 47
+select floor(0.01)
+-- !query 47 schema
+struct<FLOOR(0.01):decimal(1,0)>
+-- !query 47 output
+0
+
+
+-- !query 48
+select floor(-0.10)
+-- !query 48 schema
+struct<FLOOR(-0.10):decimal(1,0)>
+-- !query 48 output
+-1
+
+
+-- !query 49
+select 1 > 0.00001
+-- !query 49 schema
+struct<(CAST(1 AS BIGINT) > 0):boolean>
+-- !query 49 output
+true
-- 
GitLab