diff --git a/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala b/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
index 9da3ff65c744e96306a8b28bef5568e62ad284b2..11a4722722ea19e8d4d902d61203e04077ec21ca 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
@@ -97,12 +97,9 @@ class Pipeline(override val uid: String) extends Estimator[PipelineModel] {
/** @group getParam */
def getStages: Array[PipelineStage] = $(stages).clone()
- override def validateParams(paramMap: ParamMap): Unit = {
- val map = extractParamMap(paramMap)
- getStages.foreach {
- case pStage: Params => pStage.validateParams(map)
- case _ =>
- }
+ override def validateParams(): Unit = {
+ super.validateParams()
+ $(stages).foreach(_.validateParams())
}
/**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
index 1771177e1ea916d65ec35e144b4bb8d2be64182e..abb1b35bedea5851883c16dff75431f71051fd57 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
@@ -36,8 +36,8 @@ final class RegressionEvaluator(override val uid: String)
def this() = this(Identifiable.randomUID("regEval"))
/**
- * param for metric name in evaluation
- * @group param supports mse, rmse, r2, mae as valid metric names.
+ * param for metric name in evaluation (supports `"rmse"` (default), `"mse"`, `"r2"`, and `"mae"`)
+ * @group param
*/
val metricName: Param[String] = {
val allowedParams = ParamValidators.inArray(Array("mse", "rmse", "r2", "mae"))
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
index 1afa59c994c2b3f48ced5234424b205b9b4bdd42..473488dce9b0dc5693a998777730dce6f0718b02 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
@@ -333,19 +333,6 @@ trait Params extends Identifiable with Serializable {
.map(m => m.invoke(this).asInstanceOf[Param[_]])
}
- /**
- * Validates parameter values stored internally plus the input parameter map.
- * Raises an exception if any parameter is invalid.
- *
- * This only needs to check for interactions between parameters.
- * Parameter value checks which do not depend on other parameters are handled by
- * [[Param.validate()]]. This method does not handle input/output column parameters;
- * those are checked during schema validation.
- */
- def validateParams(paramMap: ParamMap): Unit = {
- copy(paramMap).validateParams()
- }
-
/**
* Validates parameter values stored internally.
* Raise an exception if any parameter value is invalid.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala
index 2e5a6295611805b3071eaa7278ee14268f21b830..6434b64aed15df4e1c6779dcb79a43f6f1d3da7e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala
@@ -102,12 +102,6 @@ class CrossValidator(override val uid: String) extends Estimator[CrossValidatorM
/** @group setParam */
def setNumFolds(value: Int): this.type = set(numFolds, value)
- override def validateParams(paramMap: ParamMap): Unit = {
- getEstimatorParamMaps.foreach { eMap =>
- getEstimator.validateParams(eMap ++ paramMap)
- }
- }
-
override def fit(dataset: DataFrame): CrossValidatorModel = {
val schema = dataset.schema
transformSchema(schema, logging = true)
@@ -147,6 +141,14 @@ class CrossValidator(override val uid: String) extends Estimator[CrossValidatorM
override def transformSchema(schema: StructType): StructType = {
$(estimator).transformSchema(schema)
}
+
+ override def validateParams(): Unit = {
+ super.validateParams()
+ val est = $(estimator)
+ for (paramMap <- $(estimatorParamMaps)) {
+ est.copy(paramMap).validateParams()
+ }
+ }
}
/**
@@ -159,8 +161,8 @@ class CrossValidatorModel private[ml] (
val bestModel: Model[_])
extends Model[CrossValidatorModel] with CrossValidatorParams {
- override def validateParams(paramMap: ParamMap): Unit = {
- bestModel.validateParams(paramMap)
+ override def validateParams(): Unit = {
+ bestModel.validateParams()
}
override def transform(dataset: DataFrame): DataFrame = {
@@ -171,4 +173,9 @@ class CrossValidatorModel private[ml] (
override def transformSchema(schema: StructType): StructType = {
bestModel.transformSchema(schema)
}
+
+ override def copy(extra: ParamMap): CrossValidatorModel = {
+ val copied = new CrossValidatorModel(uid, bestModel.copy(extra).asInstanceOf[Model[_]])
+ copyValues(copied, extra)
+ }
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/param/ParamsSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/param/ParamsSuite.scala
index d270ad7613af140f00c5b4e36b99865469bf68ff..04f2af4727ea4a615cfe331ed6145777d47b2283 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/param/ParamsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/param/ParamsSuite.scala
@@ -135,7 +135,7 @@ class ParamsSuite extends FunSuite {
intercept[IllegalArgumentException] {
solver.validateParams()
}
- solver.validateParams(ParamMap(inputCol -> "input"))
+ solver.copy(ParamMap(inputCol -> "input")).validateParams()
solver.setInputCol("input")
assert(solver.isSet(inputCol))
assert(solver.isDefined(inputCol))
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
index 05313d440fbf6511cc6a32092ca6d1c03d1f9996..65972ec79b9a5c6ba6750daaf39ab88e47559c69 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
@@ -19,11 +19,15 @@ package org.apache.spark.ml.tuning
import org.scalatest.FunSuite
+import org.apache.spark.ml.{Estimator, Model}
import org.apache.spark.ml.classification.LogisticRegression
-import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator
+import org.apache.spark.ml.evaluation.{BinaryClassificationEvaluator, Evaluator}
+import org.apache.spark.ml.param.ParamMap
+import org.apache.spark.ml.param.shared.HasInputCol
import org.apache.spark.mllib.classification.LogisticRegressionSuite.generateLogisticInput
import org.apache.spark.mllib.util.MLlibTestSparkContext
-import org.apache.spark.sql.{SQLContext, DataFrame}
+import org.apache.spark.sql.{DataFrame, SQLContext}
+import org.apache.spark.sql.types.StructType
class CrossValidatorSuite extends FunSuite with MLlibTestSparkContext {
@@ -53,4 +57,48 @@ class CrossValidatorSuite extends FunSuite with MLlibTestSparkContext {
assert(parent.getRegParam === 0.001)
assert(parent.getMaxIter === 10)
}
+
+ test("validateParams should check estimatorParamMaps") {
+ import CrossValidatorSuite._
+
+ val est = new MyEstimator("est")
+ val eval = new MyEvaluator
+ val paramMaps = new ParamGridBuilder()
+ .addGrid(est.inputCol, Array("input1", "input2"))
+ .build()
+
+ val cv = new CrossValidator()
+ .setEstimator(est)
+ .setEstimatorParamMaps(paramMaps)
+ .setEvaluator(eval)
+
+ cv.validateParams() // This should pass.
+
+ val invalidParamMaps = paramMaps :+ ParamMap(est.inputCol -> "")
+ cv.setEstimatorParamMaps(invalidParamMaps)
+ intercept[IllegalArgumentException] {
+ cv.validateParams()
+ }
+ }
+}
+
+object CrossValidatorSuite {
+
+ abstract class MyModel extends Model[MyModel]
+
+ class MyEstimator(override val uid: String) extends Estimator[MyModel] with HasInputCol {
+
+ override def validateParams(): Unit = require($(inputCol).nonEmpty)
+
+ override def fit(dataset: DataFrame): MyModel = ???
+
+ override def transformSchema(schema: StructType): StructType = ???
+ }
+
+ class MyEvaluator extends Evaluator {
+
+ override def evaluate(dataset: DataFrame): Double = ???
+
+ override val uid: String = "eval"
+ }
}