Skip to content
Snippets Groups Projects
Commit fbad7228 authored by Xiangrui Meng's avatar Xiangrui Meng
Browse files

[SPARK-3077][MLLIB] fix some chisq-test

- promote nullHypothesis field in ChiSqTestResult to TestResult. Every test should have a null hypothesis
- correct null hypothesis statement for independence test
- p-value: 0.01 -> 0.1

Author: Xiangrui Meng <meng@databricks.com>

Closes #1982 from mengxr/fix-chisq and squashes the following commits:

5f0de02 [Xiangrui Meng] make ChiSqTestResult constructor package private
bc74ea1 [Xiangrui Meng] update chisq-test
parent bc95fe08
No related branches found
No related tags found
No related merge requests found
......@@ -56,7 +56,7 @@ private[stat] object ChiSqTest extends Logging {
object NullHypothesis extends Enumeration {
type NullHypothesis = Value
val goodnessOfFit = Value("observed follows the same distribution as expected.")
val independence = Value("observations in each column are statistically independent.")
val independence = Value("the occurrence of the outcomes is statistically independent.")
}
// Method identification based on input methodName string
......
......@@ -44,6 +44,11 @@ trait TestResult[DF] {
*/
def statistic: Double
/**
* Null hypothesis of the test.
*/
def nullHypothesis: String
/**
* String explaining the hypothesis test result.
* Specific classes implementing this trait should override this method to output test-specific
......@@ -53,13 +58,13 @@ trait TestResult[DF] {
// String explaining what the p-value indicates.
val pValueExplain = if (pValue <= 0.01) {
"Very strong presumption against null hypothesis."
s"Very strong presumption against null hypothesis: $nullHypothesis."
} else if (0.01 < pValue && pValue <= 0.05) {
"Strong presumption against null hypothesis."
} else if (0.05 < pValue && pValue <= 0.01) {
"Low presumption against null hypothesis."
s"Strong presumption against null hypothesis: $nullHypothesis."
} else if (0.05 < pValue && pValue <= 0.1) {
s"Low presumption against null hypothesis: $nullHypothesis."
} else {
"No presumption against null hypothesis."
s"No presumption against null hypothesis: $nullHypothesis."
}
s"degrees of freedom = ${degreesOfFreedom.toString} \n" +
......@@ -70,19 +75,18 @@ trait TestResult[DF] {
/**
* :: Experimental ::
* Object containing the test results for the chi squared hypothesis test.
* Object containing the test results for the chi-squared hypothesis test.
*/
@Experimental
class ChiSqTestResult(override val pValue: Double,
class ChiSqTestResult private[stat] (override val pValue: Double,
override val degreesOfFreedom: Int,
override val statistic: Double,
val method: String,
val nullHypothesis: String) extends TestResult[Int] {
override val nullHypothesis: String) extends TestResult[Int] {
override def toString: String = {
"Chi squared test summary: \n" +
s"method: $method \n" +
s"null hypothesis: $nullHypothesis \n" +
super.toString
"Chi squared test summary:\n" +
s"method: $method\n" +
super.toString
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment