Skip to content
Snippets Groups Projects
Commit b4a4421b authored by Sean Owen's avatar Sean Owen Committed by DB Tsai
Browse files

[SPARK-11918][ML] Better error from WLS for cases like singular input

## What changes were proposed in this pull request?

Update error handling for Cholesky decomposition to provide a little more info when input is singular.

## How was this patch tested?

New test case; jenkins tests.

Author: Sean Owen <sowen@cloudera.com>

Closes #15177 from srowen/SPARK-11918.
parent d7ee1221
No related branches found
No related tags found
No related merge requests found
......@@ -36,8 +36,7 @@ private[spark] object CholeskyDecomposition {
val k = bx.length
val info = new intW(0)
lapack.dppsv("U", k, 1, A, bx, k, info)
val code = info.`val`
assert(code == 0, s"lapack.dppsv returned $code.")
checkReturnValue(info, "dppsv")
bx
}
......@@ -52,8 +51,20 @@ private[spark] object CholeskyDecomposition {
def inverse(UAi: Array[Double], k: Int): Array[Double] = {
val info = new intW(0)
lapack.dpptri("U", k, UAi, info)
val code = info.`val`
assert(code == 0, s"lapack.dpptri returned $code.")
checkReturnValue(info, "dpptri")
UAi
}
private def checkReturnValue(info: intW, method: String): Unit = {
info.`val` match {
case code if code < 0 =>
throw new IllegalStateException(s"LAPACK.$method returned $code; arg ${-code} is illegal")
case code if code > 0 =>
throw new IllegalArgumentException(
s"LAPACK.$method returned $code because A is not positive definite. Is A derived from " +
"a singular matrix (e.g. collinear column values)?")
case _ => // do nothing
}
}
}
......@@ -60,6 +60,26 @@ class WeightedLeastSquaresSuite extends SparkFunSuite with MLlibTestSparkContext
), 2)
}
test("two collinear features result in error with no regularization") {
val singularInstances = sc.parallelize(Seq(
Instance(1.0, 1.0, Vectors.dense(1.0, 2.0)),
Instance(2.0, 1.0, Vectors.dense(2.0, 4.0)),
Instance(3.0, 1.0, Vectors.dense(3.0, 6.0)),
Instance(4.0, 1.0, Vectors.dense(4.0, 8.0))
), 2)
intercept[IllegalArgumentException] {
new WeightedLeastSquares(
false, regParam = 0.0, standardizeFeatures = false,
standardizeLabel = false).fit(singularInstances)
}
// Should not throw an exception
new WeightedLeastSquares(
false, regParam = 1.0, standardizeFeatures = false,
standardizeLabel = false).fit(singularInstances)
}
test("WLS against lm") {
/*
R code:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment