diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index 44b3478e0c3dd8e07b98f125d12af47aa43e57d4..d7dde329ed004a1382529fe1467716938247809e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -22,7 +22,7 @@ import java.util.Locale
import scala.collection.mutable
import breeze.linalg.{DenseVector => BDV}
-import breeze.optimize.{CachedDiffFunction, DiffFunction, LBFGS => BreezeLBFGS, OWLQN => BreezeOWLQN}
+import breeze.optimize.{CachedDiffFunction, DiffFunction, LBFGS => BreezeLBFGS, LBFGSB => BreezeLBFGSB, OWLQN => BreezeOWLQN}
import org.apache.hadoop.fs.Path
import org.apache.spark.SparkException
@@ -178,11 +178,86 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
}
}
+ /**
+ * The lower bounds on coefficients if fitting under bound constrained optimization.
+ * The bound matrix must be compatible with the shape (1, number of features) for binomial
+ * regression, or (number of classes, number of features) for multinomial regression.
+ * Otherwise, it throws exception.
+ *
+ * @group param
+ */
+ @Since("2.2.0")
+ val lowerBoundsOnCoefficients: Param[Matrix] = new Param(this, "lowerBoundsOnCoefficients",
+ "The lower bounds on coefficients if fitting under bound constrained optimization.")
+
+ /** @group getParam */
+ @Since("2.2.0")
+ def getLowerBoundsOnCoefficients: Matrix = $(lowerBoundsOnCoefficients)
+
+ /**
+ * The upper bounds on coefficients if fitting under bound constrained optimization.
+ * The bound matrix must be compatible with the shape (1, number of features) for binomial
+ * regression, or (number of classes, number of features) for multinomial regression.
+ * Otherwise, it throws exception.
+ *
+ * @group param
+ */
+ @Since("2.2.0")
+ val upperBoundsOnCoefficients: Param[Matrix] = new Param(this, "upperBoundsOnCoefficients",
+ "The upper bounds on coefficients if fitting under bound constrained optimization.")
+
+ /** @group getParam */
+ @Since("2.2.0")
+ def getUpperBoundsOnCoefficients: Matrix = $(upperBoundsOnCoefficients)
+
+ /**
+ * The lower bounds on intercepts if fitting under bound constrained optimization.
+ * The bounds vector size must be equal with 1 for binomial regression, or the number
+ * of classes for multinomial regression. Otherwise, it throws exception.
+ *
+ * @group param
+ */
+ @Since("2.2.0")
+ val lowerBoundsOnIntercepts: Param[Vector] = new Param(this, "lowerBoundsOnIntercepts",
+ "The lower bounds on intercepts if fitting under bound constrained optimization.")
+
+ /** @group getParam */
+ @Since("2.2.0")
+ def getLowerBoundsOnIntercepts: Vector = $(lowerBoundsOnIntercepts)
+
+ /**
+ * The upper bounds on intercepts if fitting under bound constrained optimization.
+ * The bound vector size must be equal with 1 for binomial regression, or the number
+ * of classes for multinomial regression. Otherwise, it throws exception.
+ *
+ * @group param
+ */
+ @Since("2.2.0")
+ val upperBoundsOnIntercepts: Param[Vector] = new Param(this, "upperBoundsOnIntercepts",
+ "The upper bounds on intercepts if fitting under bound constrained optimization.")
+
+ /** @group getParam */
+ @Since("2.2.0")
+ def getUpperBoundsOnIntercepts: Vector = $(upperBoundsOnIntercepts)
+
+ protected def usingBoundConstrainedOptimization: Boolean = {
+ isSet(lowerBoundsOnCoefficients) || isSet(upperBoundsOnCoefficients) ||
+ isSet(lowerBoundsOnIntercepts) || isSet(upperBoundsOnIntercepts)
+ }
+
override protected def validateAndTransformSchema(
schema: StructType,
fitting: Boolean,
featuresDataType: DataType): StructType = {
checkThresholdConsistency()
+ if (usingBoundConstrainedOptimization) {
+ require($(elasticNetParam) == 0.0, "Fitting under bound constrained optimization only " +
+ s"supports L2 regularization, but got elasticNetParam = $getElasticNetParam.")
+ }
+ if (!$(fitIntercept)) {
+ require(!isSet(lowerBoundsOnIntercepts) && !isSet(upperBoundsOnIntercepts),
+ "Pls don't set bounds on intercepts if fitting without intercept.")
+ }
super.validateAndTransformSchema(schema, fitting, featuresDataType)
}
}
@@ -217,6 +292,9 @@ class LogisticRegression @Since("1.2.0") (
* For alpha in (0,1), the penalty is a combination of L1 and L2.
* Default is 0.0 which is an L2 penalty.
*
+ * Note: Fitting under bound constrained optimization only supports L2 regularization,
+ * so throws exception if this param is non-zero value.
+ *
* @group setParam
*/
@Since("1.4.0")
@@ -312,6 +390,71 @@ class LogisticRegression @Since("1.2.0") (
def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
setDefault(aggregationDepth -> 2)
+ /**
+ * Set the lower bounds on coefficients if fitting under bound constrained optimization.
+ *
+ * @group setParam
+ */
+ @Since("2.2.0")
+ def setLowerBoundsOnCoefficients(value: Matrix): this.type = set(lowerBoundsOnCoefficients, value)
+
+ /**
+ * Set the upper bounds on coefficients if fitting under bound constrained optimization.
+ *
+ * @group setParam
+ */
+ @Since("2.2.0")
+ def setUpperBoundsOnCoefficients(value: Matrix): this.type = set(upperBoundsOnCoefficients, value)
+
+ /**
+ * Set the lower bounds on intercepts if fitting under bound constrained optimization.
+ *
+ * @group setParam
+ */
+ @Since("2.2.0")
+ def setLowerBoundsOnIntercepts(value: Vector): this.type = set(lowerBoundsOnIntercepts, value)
+
+ /**
+ * Set the upper bounds on intercepts if fitting under bound constrained optimization.
+ *
+ * @group setParam
+ */
+ @Since("2.2.0")
+ def setUpperBoundsOnIntercepts(value: Vector): this.type = set(upperBoundsOnIntercepts, value)
+
+ private def assertBoundConstrainedOptimizationParamsValid(
+ numCoefficientSets: Int,
+ numFeatures: Int): Unit = {
+ if (isSet(lowerBoundsOnCoefficients)) {
+ require($(lowerBoundsOnCoefficients).numRows == numCoefficientSets &&
+ $(lowerBoundsOnCoefficients).numCols == numFeatures)
+ }
+ if (isSet(upperBoundsOnCoefficients)) {
+ require($(upperBoundsOnCoefficients).numRows == numCoefficientSets &&
+ $(upperBoundsOnCoefficients).numCols == numFeatures)
+ }
+ if (isSet(lowerBoundsOnIntercepts)) {
+ require($(lowerBoundsOnIntercepts).size == numCoefficientSets)
+ }
+ if (isSet(upperBoundsOnIntercepts)) {
+ require($(upperBoundsOnIntercepts).size == numCoefficientSets)
+ }
+ if (isSet(lowerBoundsOnCoefficients) && isSet(upperBoundsOnCoefficients)) {
+ require($(lowerBoundsOnCoefficients).toArray.zip($(upperBoundsOnCoefficients).toArray)
+ .forall(x => x._1 <= x._2), "LowerBoundsOnCoefficients should always " +
+ "less than or equal to upperBoundsOnCoefficients, but found: " +
+ s"lowerBoundsOnCoefficients = $getLowerBoundsOnCoefficients, " +
+ s"upperBoundsOnCoefficients = $getUpperBoundsOnCoefficients.")
+ }
+ if (isSet(lowerBoundsOnIntercepts) && isSet(upperBoundsOnIntercepts)) {
+ require($(lowerBoundsOnIntercepts).toArray.zip($(upperBoundsOnIntercepts).toArray)
+ .forall(x => x._1 <= x._2), "LowerBoundsOnIntercepts should always " +
+ "less than or equal to upperBoundsOnIntercepts, but found: " +
+ s"lowerBoundsOnIntercepts = $getLowerBoundsOnIntercepts, " +
+ s"upperBoundsOnIntercepts = $getUpperBoundsOnIntercepts.")
+ }
+ }
+
private var optInitialModel: Option[LogisticRegressionModel] = None
private[spark] def setInitialModel(model: LogisticRegressionModel): this.type = {
@@ -378,6 +521,11 @@ class LogisticRegression @Since("1.2.0") (
}
val numCoefficientSets = if (isMultinomial) numClasses else 1
+ // Check params interaction is valid if fitting under bound constrained optimization.
+ if (usingBoundConstrainedOptimization) {
+ assertBoundConstrainedOptimizationParamsValid(numCoefficientSets, numFeatures)
+ }
+
if (isDefined(thresholds)) {
require($(thresholds).length == numClasses, this.getClass.getSimpleName +
".train() called with non-matching numClasses and thresholds.length." +
@@ -397,7 +545,7 @@ class LogisticRegression @Since("1.2.0") (
val isConstantLabel = histogram.count(_ != 0.0) == 1
- if ($(fitIntercept) && isConstantLabel) {
+ if ($(fitIntercept) && isConstantLabel && !usingBoundConstrainedOptimization) {
logWarning(s"All labels are the same value and fitIntercept=true, so the coefficients " +
s"will be zeros. Training is not needed.")
val constantLabelIndex = Vectors.dense(histogram).argmax
@@ -434,8 +582,53 @@ class LogisticRegression @Since("1.2.0") (
$(standardization), bcFeaturesStd, regParamL2, multinomial = isMultinomial,
$(aggregationDepth))
+ val numCoeffsPlusIntercepts = numFeaturesPlusIntercept * numCoefficientSets
+
+ val (lowerBounds, upperBounds): (Array[Double], Array[Double]) = {
+ if (usingBoundConstrainedOptimization) {
+ val lowerBounds = Array.fill[Double](numCoeffsPlusIntercepts)(Double.NegativeInfinity)
+ val upperBounds = Array.fill[Double](numCoeffsPlusIntercepts)(Double.PositiveInfinity)
+ val isSetLowerBoundsOnCoefficients = isSet(lowerBoundsOnCoefficients)
+ val isSetUpperBoundsOnCoefficients = isSet(upperBoundsOnCoefficients)
+ val isSetLowerBoundsOnIntercepts = isSet(lowerBoundsOnIntercepts)
+ val isSetUpperBoundsOnIntercepts = isSet(upperBoundsOnIntercepts)
+
+ var i = 0
+ while (i < numCoeffsPlusIntercepts) {
+ val coefficientSetIndex = i % numCoefficientSets
+ val featureIndex = i / numCoefficientSets
+ if (featureIndex < numFeatures) {
+ if (isSetLowerBoundsOnCoefficients) {
+ lowerBounds(i) = $(lowerBoundsOnCoefficients)(
+ coefficientSetIndex, featureIndex) * featuresStd(featureIndex)
+ }
+ if (isSetUpperBoundsOnCoefficients) {
+ upperBounds(i) = $(upperBoundsOnCoefficients)(
+ coefficientSetIndex, featureIndex) * featuresStd(featureIndex)
+ }
+ } else {
+ if (isSetLowerBoundsOnIntercepts) {
+ lowerBounds(i) = $(lowerBoundsOnIntercepts)(coefficientSetIndex)
+ }
+ if (isSetUpperBoundsOnIntercepts) {
+ upperBounds(i) = $(upperBoundsOnIntercepts)(coefficientSetIndex)
+ }
+ }
+ i += 1
+ }
+ (lowerBounds, upperBounds)
+ } else {
+ (null, null)
+ }
+ }
+
val optimizer = if ($(elasticNetParam) == 0.0 || $(regParam) == 0.0) {
- new BreezeLBFGS[BDV[Double]]($(maxIter), 10, $(tol))
+ if (lowerBounds != null && upperBounds != null) {
+ new BreezeLBFGSB(
+ BDV[Double](lowerBounds), BDV[Double](upperBounds), $(maxIter), 10, $(tol))
+ } else {
+ new BreezeLBFGS[BDV[Double]]($(maxIter), 10, $(tol))
+ }
} else {
val standardizationParam = $(standardization)
def regParamL1Fun = (index: Int) => {
@@ -546,6 +739,26 @@ class LogisticRegression @Since("1.2.0") (
math.log(histogram(1) / histogram(0)))
}
+ if (usingBoundConstrainedOptimization) {
+ // Make sure all initial values locate in the corresponding bound.
+ var i = 0
+ while (i < numCoeffsPlusIntercepts) {
+ val coefficientSetIndex = i % numCoefficientSets
+ val featureIndex = i / numCoefficientSets
+ if (initialCoefWithInterceptMatrix(coefficientSetIndex, featureIndex) < lowerBounds(i))
+ {
+ initialCoefWithInterceptMatrix.update(
+ coefficientSetIndex, featureIndex, lowerBounds(i))
+ } else if (
+ initialCoefWithInterceptMatrix(coefficientSetIndex, featureIndex) > upperBounds(i))
+ {
+ initialCoefWithInterceptMatrix.update(
+ coefficientSetIndex, featureIndex, upperBounds(i))
+ }
+ i += 1
+ }
+ }
+
val states = optimizer.iterations(new CachedDiffFunction(costFun),
new BDV[Double](initialCoefWithInterceptMatrix.toArray))
@@ -599,7 +812,7 @@ class LogisticRegression @Since("1.2.0") (
if (isIntercept) interceptVec.toArray(classIndex) = value
}
- if ($(regParam) == 0.0 && isMultinomial) {
+ if ($(regParam) == 0.0 && isMultinomial && !usingBoundConstrainedOptimization) {
/*
When no regularization is applied, the multinomial coefficients lack identifiability
because we do not use a pivot class. We can add any constant value to the coefficients
@@ -620,7 +833,7 @@ class LogisticRegression @Since("1.2.0") (
}
// center the intercepts when using multinomial algorithm
- if ($(fitIntercept) && isMultinomial) {
+ if ($(fitIntercept) && isMultinomial && !usingBoundConstrainedOptimization) {
val interceptArray = interceptVec.toArray
val interceptMean = interceptArray.sum / interceptArray.length
(0 until interceptVec.size).foreach { i => interceptArray(i) -= interceptMean }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
index 83f575e83828fd298974ecfa141f07510a8e5231..bf6bfe30bfe20429811df98c7d7d2504d5dd6bc7 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
@@ -26,7 +26,7 @@ import org.apache.spark.{SparkException, SparkFunSuite}
import org.apache.spark.ml.attribute.NominalAttribute
import org.apache.spark.ml.classification.LogisticRegressionSuite._
import org.apache.spark.ml.feature.{Instance, LabeledPoint}
-import org.apache.spark.ml.linalg.{DenseMatrix, Matrices, SparseMatrix, Vector, Vectors}
+import org.apache.spark.ml.linalg.{DenseMatrix, Matrices, Matrix, SparseMatrix, Vector, Vectors}
import org.apache.spark.ml.param.{ParamMap, ParamsSuite}
import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTestingUtils}
import org.apache.spark.ml.util.TestingUtils._
@@ -150,6 +150,54 @@ class LogisticRegressionSuite
assert(!model.hasSummary)
}
+ test("logistic regression: illegal params") {
+ val lowerBoundsOnCoefficients = Matrices.dense(1, 4, Array(1.0, 0.0, 1.0, 0.0))
+ val upperBoundsOnCoefficients1 = Matrices.dense(1, 4, Array(0.0, 1.0, 1.0, 0.0))
+ val upperBoundsOnCoefficients2 = Matrices.dense(1, 3, Array(1.0, 0.0, 1.0))
+ val lowerBoundsOnIntercepts = Vectors.dense(1.0)
+
+ // Work well when only set bound in one side.
+ new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .fit(binaryDataset)
+
+ withClue("bound constrained optimization only supports L2 regularization") {
+ intercept[IllegalArgumentException] {
+ new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setElasticNetParam(1.0)
+ .fit(binaryDataset)
+ }
+ }
+
+ withClue("lowerBoundsOnCoefficients should less than or equal to upperBoundsOnCoefficients") {
+ intercept[IllegalArgumentException] {
+ new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients1)
+ .fit(binaryDataset)
+ }
+ }
+
+ withClue("the coefficients bound matrix mismatched with shape (1, number of features)") {
+ intercept[IllegalArgumentException] {
+ new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients2)
+ .fit(binaryDataset)
+ }
+ }
+
+ withClue("bounds on intercepts should not be set if fitting without intercept") {
+ intercept[IllegalArgumentException] {
+ new LogisticRegression()
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setFitIntercept(false)
+ .fit(binaryDataset)
+ }
+ }
+ }
+
test("empty probabilityCol") {
val lr = new LogisticRegression().setProbabilityCol("")
val model = lr.fit(smallBinaryDataset)
@@ -610,6 +658,107 @@ class LogisticRegressionSuite
assert(model2.coefficients ~= coefficientsR relTol 1E-3)
}
+ test("binary logistic regression with intercept without regularization with bound") {
+ // Bound constrained optimization with bound on one side.
+ val upperBoundsOnCoefficients = Matrices.dense(1, 4, Array(1.0, 0.0, 1.0, 0.0))
+ val upperBoundsOnIntercepts = Vectors.dense(1.0)
+
+ val trainer1 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected1 = Vectors.dense(0.06079437, 0.0, -0.26351059, -0.59102199)
+ val interceptExpected1 = 1.0
+
+ assert(model1.intercept ~== interceptExpected1 relTol 1E-3)
+ assert(model1.coefficients ~= coefficientsExpected1 relTol 1E-3)
+
+ // Without regularization, with or without standardization will converge to the same solution.
+ assert(model2.intercept ~== interceptExpected1 relTol 1E-3)
+ assert(model2.coefficients ~= coefficientsExpected1 relTol 1E-3)
+
+ // Bound constrained optimization with bound on both side.
+ val lowerBoundsOnCoefficients = Matrices.dense(1, 4, Array(0.0, -1.0, 0.0, -1.0))
+ val lowerBoundsOnIntercepts = Vectors.dense(0.0)
+
+ val trainer3 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer4 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model3 = trainer3.fit(binaryDataset)
+ val model4 = trainer4.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected3 = Vectors.dense(0.0, 0.0, 0.0, -0.71708632)
+ val interceptExpected3 = 0.58776113
+
+ assert(model3.intercept ~== interceptExpected3 relTol 1E-3)
+ assert(model3.coefficients ~= coefficientsExpected3 relTol 1E-3)
+
+ // Without regularization, with or without standardization will converge to the same solution.
+ assert(model4.intercept ~== interceptExpected3 relTol 1E-3)
+ assert(model4.coefficients ~= coefficientsExpected3 relTol 1E-3)
+
+ // Bound constrained optimization with infinite bound on both side.
+ val trainer5 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(Matrices.dense(1, 4, Array.fill(4)(Double.PositiveInfinity)))
+ .setUpperBoundsOnIntercepts(Vectors.dense(Double.PositiveInfinity))
+ .setLowerBoundsOnCoefficients(Matrices.dense(1, 4, Array.fill(4)(Double.NegativeInfinity)))
+ .setLowerBoundsOnIntercepts(Vectors.dense(Double.NegativeInfinity))
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer6 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(Matrices.dense(1, 4, Array.fill(4)(Double.PositiveInfinity)))
+ .setUpperBoundsOnIntercepts(Vectors.dense(Double.PositiveInfinity))
+ .setLowerBoundsOnCoefficients(Matrices.dense(1, 4, Array.fill(4)(Double.NegativeInfinity)))
+ .setLowerBoundsOnIntercepts(Vectors.dense(Double.NegativeInfinity))
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model5 = trainer5.fit(binaryDataset)
+ val model6 = trainer6.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ // It should be same as unbound constrained optimization with LBFGS.
+ val coefficientsExpected5 = Vectors.dense(-0.5734389, 0.8911736, -0.3878645, -0.8060570)
+ val interceptExpected5 = 2.7355261
+
+ assert(model5.intercept ~== interceptExpected5 relTol 1E-3)
+ assert(model5.coefficients ~= coefficientsExpected5 relTol 1E-3)
+
+ // Without regularization, with or without standardization will converge to the same solution.
+ assert(model6.intercept ~== interceptExpected5 relTol 1E-3)
+ assert(model6.coefficients ~= coefficientsExpected5 relTol 1E-3)
+ }
+
test("binary logistic regression without intercept without regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(false).setStandardization(true)
.setWeightCol("weight")
@@ -650,6 +799,34 @@ class LogisticRegressionSuite
assert(model2.coefficients ~= coefficientsR relTol 1E-2)
}
+ test("binary logistic regression without intercept without regularization with bound") {
+ val upperBoundsOnCoefficients = Matrices.dense(1, 4, Array(1.0, 0.0, 1.0, 0.0)).toSparse
+
+ val trainer1 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setFitIntercept(false)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setFitIntercept(false)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected = Vectors.dense(0.20847553, 0.0, -0.24240289, -0.55568071)
+
+ assert(model1.intercept ~== 0.0 relTol 1E-3)
+ assert(model1.coefficients ~= coefficientsExpected relTol 1E-3)
+
+ // Without regularization, with or without standardization will converge to the same solution.
+ assert(model2.intercept ~== 0.0 relTol 1E-3)
+ assert(model2.coefficients ~= coefficientsExpected relTol 1E-3)
+ }
+
test("binary logistic regression with intercept with L1 regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(true)
.setElasticNetParam(1.0).setRegParam(0.12).setStandardization(true).setWeightCol("weight")
@@ -815,6 +992,40 @@ class LogisticRegressionSuite
assert(model2.coefficients ~= coefficientsR relTol 1E-3)
}
+ test("binary logistic regression with intercept with L2 regularization with bound") {
+ val upperBoundsOnCoefficients = Matrices.dense(1, 4, Array(1.0, 0.0, 1.0, 0.0))
+ val upperBoundsOnIntercepts = Vectors.dense(1.0)
+
+ val trainer1 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setRegParam(1.37)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setRegParam(1.37)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpectedWithStd = Vectors.dense(-0.06985003, 0.0, -0.04794278, -0.10168595)
+ val interceptExpectedWithStd = 0.45750141
+ val coefficientsExpected = Vectors.dense(-0.0494524, 0.0, -0.11360797, -0.06313577)
+ val interceptExpected = 0.53722967
+
+ assert(model1.intercept ~== interceptExpectedWithStd relTol 1E-3)
+ assert(model1.coefficients ~= coefficientsExpectedWithStd relTol 1E-3)
+ assert(model2.intercept ~== interceptExpected relTol 1E-3)
+ assert(model2.coefficients ~= coefficientsExpected relTol 1E-3)
+ }
+
test("binary logistic regression without intercept with L2 regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(false)
.setElasticNetParam(0.0).setRegParam(1.37).setStandardization(true).setWeightCol("weight")
@@ -864,6 +1075,35 @@ class LogisticRegressionSuite
assert(model2.coefficients ~= coefficientsR relTol 1E-2)
}
+ test("binary logistic regression without intercept with L2 regularization with bound") {
+ val upperBoundsOnCoefficients = Matrices.dense(1, 4, Array(1.0, 0.0, 1.0, 0.0))
+
+ val trainer1 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setRegParam(1.37)
+ .setFitIntercept(false)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setRegParam(1.37)
+ .setFitIntercept(false)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpectedWithStd = Vectors.dense(-0.00796538, 0.0, -0.0394228, -0.0873314)
+ val coefficientsExpected = Vectors.dense(0.01105972, 0.0, -0.08574949, -0.05079558)
+
+ assert(model1.intercept ~== 0.0 relTol 1E-3)
+ assert(model1.coefficients ~= coefficientsExpectedWithStd relTol 1E-3)
+ assert(model2.intercept ~== 0.0 relTol 1E-3)
+ assert(model2.coefficients ~= coefficientsExpected relTol 1E-3)
+ }
+
test("binary logistic regression with intercept with ElasticNet regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(true).setMaxIter(200)
.setElasticNetParam(0.38).setRegParam(0.21).setStandardization(true).setWeightCol("weight")
@@ -1084,7 +1324,6 @@ class LogisticRegressionSuite
}
test("multinomial logistic regression with intercept without regularization") {
-
val trainer1 = (new LogisticRegression).setFitIntercept(true)
.setElasticNetParam(0.0).setRegParam(0.0).setStandardization(true).setWeightCol("weight")
val trainer2 = (new LogisticRegression).setFitIntercept(true)
@@ -1152,6 +1391,110 @@ class LogisticRegressionSuite
assert(model2.interceptVector.toArray.sum ~== 0.0 absTol eps)
}
+ test("multinomial logistic regression with intercept without regularization with bound") {
+ // Bound constrained optimization with bound on one side.
+ val lowerBoundsOnCoefficients = Matrices.dense(3, 4, Array.fill(12)(1.0))
+ val lowerBoundsOnIntercepts = Vectors.dense(Array.fill(3)(1.0))
+
+ val trainer1 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(multinomialDataset)
+ val model2 = trainer2.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected1 = new DenseMatrix(3, 4, Array(
+ 2.52076464, 2.73596057, 1.87984904, 2.73264492,
+ 1.93302281, 3.71363303, 1.50681746, 1.93398782,
+ 2.37839917, 1.93601818, 1.81924758, 2.45191255), isTransposed = true)
+ val interceptsExpected1 = Vectors.dense(1.00010477, 3.44237083, 4.86740286)
+
+ checkCoefficientsEquivalent(model1.coefficientMatrix, coefficientsExpected1)
+ assert(model1.interceptVector ~== interceptsExpected1 relTol 0.01)
+ checkCoefficientsEquivalent(model2.coefficientMatrix, coefficientsExpected1)
+ assert(model2.interceptVector ~== interceptsExpected1 relTol 0.01)
+
+ // Bound constrained optimization with bound on both side.
+ val upperBoundsOnCoefficients = Matrices.dense(3, 4, Array.fill(12)(2.0))
+ val upperBoundsOnIntercepts = Vectors.dense(Array.fill(3)(2.0))
+
+ val trainer3 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer4 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setUpperBoundsOnCoefficients(upperBoundsOnCoefficients)
+ .setUpperBoundsOnIntercepts(upperBoundsOnIntercepts)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model3 = trainer3.fit(multinomialDataset)
+ val model4 = trainer4.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected3 = new DenseMatrix(3, 4, Array(
+ 1.61967097, 1.16027835, 1.45131448, 1.97390431,
+ 1.30529317, 2.0, 1.12985473, 1.26652854,
+ 1.61647195, 1.0, 1.40642959, 1.72985589), isTransposed = true)
+ val interceptsExpected3 = Vectors.dense(1.0, 2.0, 2.0)
+
+ checkCoefficientsEquivalent(model3.coefficientMatrix, coefficientsExpected3)
+ assert(model3.interceptVector ~== interceptsExpected3 relTol 0.01)
+ checkCoefficientsEquivalent(model4.coefficientMatrix, coefficientsExpected3)
+ assert(model4.interceptVector ~== interceptsExpected3 relTol 0.01)
+
+ // Bound constrained optimization with infinite bound on both side.
+ val trainer5 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(Matrices.dense(3, 4, Array.fill(12)(Double.NegativeInfinity)))
+ .setLowerBoundsOnIntercepts(Vectors.dense(Array.fill(3)(Double.NegativeInfinity)))
+ .setUpperBoundsOnCoefficients(Matrices.dense(3, 4, Array.fill(12)(Double.PositiveInfinity)))
+ .setUpperBoundsOnIntercepts(Vectors.dense(Array.fill(3)(Double.PositiveInfinity)))
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer6 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(Matrices.dense(3, 4, Array.fill(12)(Double.NegativeInfinity)))
+ .setLowerBoundsOnIntercepts(Vectors.dense(Array.fill(3)(Double.NegativeInfinity)))
+ .setUpperBoundsOnCoefficients(Matrices.dense(3, 4, Array.fill(12)(Double.PositiveInfinity)))
+ .setUpperBoundsOnIntercepts(Vectors.dense(Array.fill(3)(Double.PositiveInfinity)))
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model5 = trainer5.fit(multinomialDataset)
+ val model6 = trainer6.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ // It should be same as unbound constrained optimization with LBFGS.
+ val coefficientsExpected5 = new DenseMatrix(3, 4, Array(
+ 0.24337896, -0.05916156, 0.14446790, 0.35976165,
+ -0.3443375, 0.9181331, -0.2283959, -0.4388066,
+ 0.10095851, -0.85897154, 0.08392798, 0.07904499), isTransposed = true)
+ val interceptsExpected5 = Vectors.dense(-2.10320093, 0.3394473, 1.76375361)
+
+ checkCoefficientsEquivalent(model5.coefficientMatrix, coefficientsExpected5)
+ assert(model5.interceptVector ~== interceptsExpected5 relTol 0.01)
+ checkCoefficientsEquivalent(model6.coefficientMatrix, coefficientsExpected5)
+ assert(model6.interceptVector ~== interceptsExpected5 relTol 0.01)
+ }
+
test("multinomial logistic regression without intercept without regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(false)
@@ -1220,6 +1563,35 @@ class LogisticRegressionSuite
assert(model2.interceptVector.toArray.sum ~== 0.0 absTol eps)
}
+ test("multinomial logistic regression without intercept without regularization with bound") {
+ val lowerBoundsOnCoefficients = Matrices.dense(3, 4, Array.fill(12)(1.0))
+
+ val trainer1 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setFitIntercept(false)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setFitIntercept(false)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(multinomialDataset)
+ val model2 = trainer2.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpected = new DenseMatrix(3, 4, Array(
+ 1.62410051, 1.38219391, 1.34486618, 1.74641729,
+ 1.23058989, 2.71787825, 1.0, 1.00007073,
+ 1.79478632, 1.14360459, 1.33011603, 1.55093897), isTransposed = true)
+
+ checkCoefficientsEquivalent(model1.coefficientMatrix, coefficientsExpected)
+ assert(model1.interceptVector.toArray === Array.fill(3)(0.0))
+ checkCoefficientsEquivalent(model2.coefficientMatrix, coefficientsExpected)
+ assert(model2.interceptVector.toArray === Array.fill(3)(0.0))
+ }
+
test("multinomial logistic regression with intercept with L1 regularization") {
// use tighter constraints because OWL-QN solver takes longer to converge
@@ -1518,6 +1890,46 @@ class LogisticRegressionSuite
assert(model2.interceptVector.toArray.sum ~== 0.0 absTol eps)
}
+ test("multinomial logistic regression with intercept with L2 regularization with bound") {
+ val lowerBoundsOnCoefficients = Matrices.dense(3, 4, Array.fill(12)(1.0))
+ val lowerBoundsOnIntercepts = Vectors.dense(Array.fill(3)(1.0))
+
+ val trainer1 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setRegParam(0.1)
+ .setFitIntercept(true)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setLowerBoundsOnIntercepts(lowerBoundsOnIntercepts)
+ .setRegParam(0.1)
+ .setFitIntercept(true)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(multinomialDataset)
+ val model2 = trainer2.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpectedWithStd = new DenseMatrix(3, 4, Array(
+ 1.0, 1.0, 1.0, 1.01647497,
+ 1.0, 1.44105616, 1.0, 1.0,
+ 1.0, 1.0, 1.0, 1.0), isTransposed = true)
+ val interceptsExpectedWithStd = Vectors.dense(2.52055893, 1.0, 2.560682)
+ val coefficientsExpected = new DenseMatrix(3, 4, Array(
+ 1.0, 1.0, 1.03189386, 1.0,
+ 1.0, 1.0, 1.0, 1.0,
+ 1.0, 1.0, 1.0, 1.0), isTransposed = true)
+ val interceptsExpected = Vectors.dense(1.06418835, 1.0, 1.20494701)
+
+ assert(model1.coefficientMatrix ~== coefficientsExpectedWithStd relTol 0.01)
+ assert(model1.interceptVector ~== interceptsExpectedWithStd relTol 0.01)
+ assert(model2.coefficientMatrix ~== coefficientsExpected relTol 0.01)
+ assert(model2.interceptVector ~== interceptsExpected relTol 0.01)
+ }
+
test("multinomial logistic regression without intercept with L2 regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(false)
.setElasticNetParam(0.0).setRegParam(0.1).setStandardization(true).setWeightCol("weight")
@@ -1615,6 +2027,41 @@ class LogisticRegressionSuite
assert(model2.interceptVector.toArray.sum ~== 0.0 absTol eps)
}
+ test("multinomial logistic regression without intercept with L2 regularization with bound") {
+ val lowerBoundsOnCoefficients = Matrices.dense(3, 4, Array.fill(12)(1.0))
+
+ val trainer1 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setRegParam(0.1)
+ .setFitIntercept(false)
+ .setStandardization(true)
+ .setWeightCol("weight")
+ val trainer2 = new LogisticRegression()
+ .setLowerBoundsOnCoefficients(lowerBoundsOnCoefficients)
+ .setRegParam(0.1)
+ .setFitIntercept(false)
+ .setStandardization(false)
+ .setWeightCol("weight")
+
+ val model1 = trainer1.fit(multinomialDataset)
+ val model2 = trainer2.fit(multinomialDataset)
+
+ // The solution is generated by https://github.com/yanboliang/bound-optimization.
+ val coefficientsExpectedWithStd = new DenseMatrix(3, 4, Array(
+ 1.01324653, 1.0, 1.0, 1.0415767,
+ 1.0, 1.0, 1.0, 1.0,
+ 1.02244888, 1.0, 1.0, 1.0), isTransposed = true)
+ val coefficientsExpected = new DenseMatrix(3, 4, Array(
+ 1.0, 1.0, 1.03932259, 1.0,
+ 1.0, 1.0, 1.0, 1.0,
+ 1.0, 1.0, 1.03274649, 1.0), isTransposed = true)
+
+ assert(model1.coefficientMatrix ~== coefficientsExpectedWithStd absTol 0.01)
+ assert(model1.interceptVector.toArray === Array.fill(3)(0.0))
+ assert(model2.coefficientMatrix ~== coefficientsExpected absTol 0.01)
+ assert(model2.interceptVector.toArray === Array.fill(3)(0.0))
+ }
+
test("multinomial logistic regression with intercept with elasticnet regularization") {
val trainer1 = (new LogisticRegression).setFitIntercept(true).setWeightCol("weight")
.setElasticNetParam(0.5).setRegParam(0.1).setStandardization(true)
@@ -2273,4 +2720,19 @@ object LogisticRegressionSuite {
val testData = (0 until nPoints).map(i => LabeledPoint(y(i), x(i)))
testData
}
+
+ /**
+ * When no regularization is applied, the multinomial coefficients lack identifiability
+ * because we do not use a pivot class. We can add any constant value to the coefficients
+ * and get the same likelihood. If fitting under bound constrained optimization, we don't
+ * choose the mean centered coefficients like what we do for unbound problems, since they
+ * may out of the bounds. We use this function to check whether two coefficients are equivalent.
+ */
+ def checkCoefficientsEquivalent(coefficients1: Matrix, coefficients2: Matrix): Unit = {
+ coefficients1.colIter.zip(coefficients2.colIter).foreach { case (col1: Vector, col2: Vector) =>
+ (col1.asBreeze - col2.asBreeze).toArray.toSeq.sliding(2).foreach {
+ case Seq(v1, v2) => assert(v1 ~= v2 absTol 1E-3)
+ }
+ }
+ }
}