From 9afcf127d31b5477a539dde6e5f01861532a1c4c Mon Sep 17 00:00:00 2001
From: Peng <peng.meng@intel.com>
Date: Wed, 24 May 2017 19:54:17 +0800
Subject: [PATCH] [SPARK-20764][ML][PYSPARK][FOLLOWUP] Fix visibility
discrepancy with numInstances and degreesOfFreedom in LR and GLR - Python
version
## What changes were proposed in this pull request?
Add test cases for PR-18062
## How was this patch tested?
The existing UT
Author: Peng <peng.meng@intel.com>
Closes #18068 from mpjlu/moreTest.
---
python/pyspark/ml/tests.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/python/pyspark/ml/tests.py b/python/pyspark/ml/tests.py
index 51a3e8efe8..a3393c6248 100755
--- a/python/pyspark/ml/tests.py
+++ b/python/pyspark/ml/tests.py
@@ -1066,6 +1066,7 @@ class TrainingSummaryTest(SparkSessionTestCase):
self.assertAlmostEqual(s.r2, 1.0, 2)
self.assertTrue(isinstance(s.residuals, DataFrame))
self.assertEqual(s.numInstances, 2)
+ self.assertEqual(s.degreesOfFreedom, 1)
devResiduals = s.devianceResiduals
self.assertTrue(isinstance(devResiduals, list) and isinstance(devResiduals[0], float))
coefStdErr = s.coefficientStandardErrors
@@ -1075,7 +1076,8 @@ class TrainingSummaryTest(SparkSessionTestCase):
pValues = s.pValues
self.assertTrue(isinstance(pValues, list) and isinstance(pValues[0], float))
# test evaluation (with training dataset) produces a summary with same values
- # one check is enough to verify a summary is returned, Scala version runs full test
+ # one check is enough to verify a summary is returned
+ # The child class LinearRegressionTrainingSummary runs full test
sameSummary = model.evaluate(df)
self.assertAlmostEqual(sameSummary.explainedVariance, s.explainedVariance)
@@ -1093,6 +1095,7 @@ class TrainingSummaryTest(SparkSessionTestCase):
self.assertEqual(s.numIterations, 1) # this should default to a single iteration of WLS
self.assertTrue(isinstance(s.predictions, DataFrame))
self.assertEqual(s.predictionCol, "prediction")
+ self.assertEqual(s.numInstances, 2)
self.assertTrue(isinstance(s.residuals(), DataFrame))
self.assertTrue(isinstance(s.residuals("pearson"), DataFrame))
coefStdErr = s.coefficientStandardErrors
@@ -1111,7 +1114,8 @@ class TrainingSummaryTest(SparkSessionTestCase):
self.assertTrue(isinstance(s.nullDeviance, float))
self.assertTrue(isinstance(s.dispersion, float))
# test evaluation (with training dataset) produces a summary with same values
- # one check is enough to verify a summary is returned, Scala version runs full test
+ # one check is enough to verify a summary is returned
+ # The child class GeneralizedLinearRegressionTrainingSummary runs full test
sameSummary = model.evaluate(df)
self.assertAlmostEqual(sameSummary.deviance, s.deviance)
--
GitLab