From 1e07a71924ef1420c96a3a0a8cb5be2f3a830037 Mon Sep 17 00:00:00 2001
From: actuaryzhang <actuaryzhang10@gmail.com>
Date: Mon, 23 Jan 2017 00:53:44 -0800
Subject: [PATCH] [SPARK-19155][ML] Make family case insensitive in GLM
## What changes were proposed in this pull request?
This is a supplement to PR #16516 which did not make the value from `getFamily` case insensitive. Current tests of poisson/binomial glm with weight fail when specifying 'Poisson' or 'Binomial', because the calculation of `dispersion` and `pValue` checks the value of family retrieved from `getFamily`
```
model.getFamily == Binomial.name || model.getFamily == Poisson.name
```
## How was this patch tested?
Update existing tests for 'Poisson' and 'Binomial'.
yanboliang felixcheung imatiach-msft
Author: actuaryzhang <actuaryzhang10@gmail.com>
Closes #16675 from actuaryzhang/family.
(cherry picked from commit f067acefabebf04939d03a639a2aaa654e1bc8f9)
Signed-off-by: Yanbo Liang <ybliang8@gmail.com>
---
.../spark/ml/regression/GeneralizedLinearRegression.scala | 6 ++++--
.../ml/regression/GeneralizedLinearRegressionSuite.scala | 4 ++--
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
index 1e7ba91e01..676be61795 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
@@ -1027,7 +1027,8 @@ class GeneralizedLinearRegressionSummary private[regression] (
*/
@Since("2.0.0")
lazy val dispersion: Double = if (
- model.getFamily == Binomial.name || model.getFamily == Poisson.name) {
+ model.getFamily.toLowerCase == Binomial.name ||
+ model.getFamily.toLowerCase == Poisson.name) {
1.0
} else {
val rss = pearsonResiduals.agg(sum(pow(col("pearsonResiduals"), 2.0))).first().getDouble(0)
@@ -1130,7 +1131,8 @@ class GeneralizedLinearRegressionTrainingSummary private[regression] (
@Since("2.0.0")
lazy val pValues: Array[Double] = {
if (isNormalSolver) {
- if (model.getFamily == Binomial.name || model.getFamily == Poisson.name) {
+ if (model.getFamily.toLowerCase == Binomial.name ||
+ model.getFamily.toLowerCase == Poisson.name) {
tValues.map { x => 2.0 * (1.0 - dist.Gaussian(0.0, 1.0).cdf(math.abs(x))) }
} else {
tValues.map { x =>
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala
index 415d426af3..95b443dd25 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala
@@ -757,7 +757,7 @@ class GeneralizedLinearRegressionSuite
0.5554219 -0.4034267 0.6567520 -0.2611382
*/
val trainer = new GeneralizedLinearRegression()
- .setFamily("binomial")
+ .setFamily("Binomial")
.setWeightCol("weight")
.setFitIntercept(false)
@@ -874,7 +874,7 @@ class GeneralizedLinearRegressionSuite
-0.4378554 0.2189277 0.1459518 -0.1094638
*/
val trainer = new GeneralizedLinearRegression()
- .setFamily("poisson")
+ .setFamily("Poisson")
.setWeightCol("weight")
.setFitIntercept(true)
--
GitLab