diff --git a/R/pkg/R/mllib.R b/R/pkg/R/mllib.R
index b0d73dd93a79dfc2b73aa4682f66bde7e1c939aa..7ff859741b4a034bb21b181cd2f3b79a7a38095c 100644
--- a/R/pkg/R/mllib.R
+++ b/R/pkg/R/mllib.R
@@ -91,12 +91,26 @@ setMethod("predict", signature(object = "PipelineModel"),
#'}
setMethod("summary", signature(x = "PipelineModel"),
function(x, ...) {
+ modelName <- callJStatic("org.apache.spark.ml.api.r.SparkRWrappers",
+ "getModelName", x@model)
features <- callJStatic("org.apache.spark.ml.api.r.SparkRWrappers",
"getModelFeatures", x@model)
coefficients <- callJStatic("org.apache.spark.ml.api.r.SparkRWrappers",
"getModelCoefficients", x@model)
- coefficients <- as.matrix(unlist(coefficients))
- colnames(coefficients) <- c("Estimate")
- rownames(coefficients) <- unlist(features)
- return(list(coefficients = coefficients))
+ if (modelName == "LinearRegressionModel") {
+ devianceResiduals <- callJStatic("org.apache.spark.ml.api.r.SparkRWrappers",
+ "getModelDevianceResiduals", x@model)
+ devianceResiduals <- matrix(devianceResiduals, nrow = 1)
+ colnames(devianceResiduals) <- c("Min", "Max")
+ rownames(devianceResiduals) <- rep("", times = 1)
+ coefficients <- matrix(coefficients, ncol = 4)
+ colnames(coefficients) <- c("Estimate", "Std. Error", "t value", "Pr(>|t|)")
+ rownames(coefficients) <- unlist(features)
+ return(list(DevianceResiduals = devianceResiduals, Coefficients = coefficients))
+ } else {
+ coefficients <- as.matrix(unlist(coefficients))
+ colnames(coefficients) <- c("Estimate")
+ rownames(coefficients) <- unlist(features)
+ return(list(coefficients = coefficients))
+ }
})
diff --git a/R/pkg/inst/tests/test_mllib.R b/R/pkg/inst/tests/test_mllib.R
index 4761e285a2479c71d5cf2f832ca541aafaf57418..2606407bdcb44682cc3099101bf1e437d5412479 100644
--- a/R/pkg/inst/tests/test_mllib.R
+++ b/R/pkg/inst/tests/test_mllib.R
@@ -71,12 +71,23 @@ test_that("feature interaction vs native glm", {
test_that("summary coefficients match with native glm", {
training <- createDataFrame(sqlContext, iris)
- stats <- summary(glm(Sepal_Width ~ Sepal_Length + Species, data = training, solver = "l-bfgs"))
- coefs <- as.vector(stats$coefficients)
+ stats <- summary(glm(Sepal_Width ~ Sepal_Length + Species, data = training, solver = "normal"))
+ coefs <- unlist(stats$Coefficients)
+ devianceResiduals <- unlist(stats$DevianceResiduals)
+
rCoefs <- as.vector(coef(glm(Sepal.Width ~ Sepal.Length + Species, data = iris)))
- expect_true(all(abs(rCoefs - coefs) < 1e-6))
+ rStdError <- c(0.23536, 0.04630, 0.07207, 0.09331)
+ rTValue <- c(7.123, 7.557, -13.644, -10.798)
+ rPValue <- c(0.0, 0.0, 0.0, 0.0)
+ rDevianceResiduals <- c(-0.95096, 0.72918)
+
+ expect_true(all(abs(rCoefs - coefs[1:4]) < 1e-6))
+ expect_true(all(abs(rStdError - coefs[5:8]) < 1e-5))
+ expect_true(all(abs(rTValue - coefs[9:12]) < 1e-3))
+ expect_true(all(abs(rPValue - coefs[13:16]) < 1e-6))
+ expect_true(all(abs(rDevianceResiduals - devianceResiduals) < 1e-5))
expect_true(all(
- as.character(stats$features) ==
+ rownames(stats$Coefficients) ==
c("(Intercept)", "Sepal_Length", "Species_versicolor", "Species_virginica")))
})
@@ -85,14 +96,20 @@ test_that("summary coefficients match with native glm of family 'binomial'", {
training <- filter(df, df$Species != "setosa")
stats <- summary(glm(Species ~ Sepal_Length + Sepal_Width, data = training,
family = "binomial"))
- coefs <- as.vector(stats$coefficients)
+ coefs <- as.vector(stats$Coefficients)
rTraining <- iris[iris$Species %in% c("versicolor","virginica"),]
rCoefs <- as.vector(coef(glm(Species ~ Sepal.Length + Sepal.Width, data = rTraining,
family = binomial(link = "logit"))))
+ rStdError <- c(3.0974, 0.5169, 0.8628)
+ rTValue <- c(-4.212, 3.680, 0.469)
+ rPValue <- c(0.000, 0.000, 0.639)
- expect_true(all(abs(rCoefs - coefs) < 1e-4))
+ expect_true(all(abs(rCoefs - coefs[1:3]) < 1e-4))
+ expect_true(all(abs(rStdError - coefs[4:6]) < 1e-4))
+ expect_true(all(abs(rTValue - coefs[7:9]) < 1e-3))
+ expect_true(all(abs(rPValue - coefs[10:12]) < 1e-3))
expect_true(all(
- as.character(stats$features) ==
+ rownames(stats$Coefficients) ==
c("(Intercept)", "Sepal_Length", "Sepal_Width")))
})
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/SparkRWrappers.scala b/mllib/src/main/scala/org/apache/spark/ml/r/SparkRWrappers.scala
index 5be2f869362118196d0fcf694c7589847a736bc4..4d82b90bfdf20f5511f8bd889e8a871d799a1793 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/r/SparkRWrappers.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/r/SparkRWrappers.scala
@@ -52,11 +52,36 @@ private[r] object SparkRWrappers {
}
def getModelCoefficients(model: PipelineModel): Array[Double] = {
+ model.stages.last match {
+ case m: LinearRegressionModel => {
+ val coefficientStandardErrorsR = Array(m.summary.coefficientStandardErrors.last) ++
+ m.summary.coefficientStandardErrors.dropRight(1)
+ val tValuesR = Array(m.summary.tValues.last) ++ m.summary.tValues.dropRight(1)
+ val pValuesR = Array(m.summary.pValues.last) ++ m.summary.pValues.dropRight(1)
+ if (m.getFitIntercept) {
+ Array(m.intercept) ++ m.coefficients.toArray ++ coefficientStandardErrorsR ++
+ tValuesR ++ pValuesR
+ } else {
+ m.coefficients.toArray ++ coefficientStandardErrorsR ++ tValuesR ++ pValuesR
+ }
+ }
+ case m: LogisticRegressionModel => {
+ if (m.getFitIntercept) {
+ Array(m.intercept) ++ m.coefficients.toArray
+ } else {
+ m.coefficients.toArray
+ }
+ }
+ }
+ }
+
+ def getModelDevianceResiduals(model: PipelineModel): Array[Double] = {
model.stages.last match {
case m: LinearRegressionModel =>
- Array(m.intercept) ++ m.coefficients.toArray
+ m.summary.devianceResiduals
case m: LogisticRegressionModel =>
- Array(m.intercept) ++ m.coefficients.toArray
+ throw new UnsupportedOperationException(
+ "No deviance residuals available for LogisticRegressionModel")
}
}
@@ -65,11 +90,28 @@ private[r] object SparkRWrappers {
case m: LinearRegressionModel =>
val attrs = AttributeGroup.fromStructField(
m.summary.predictions.schema(m.summary.featuresCol))
- Array("(Intercept)") ++ attrs.attributes.get.map(_.name.get)
+ if (m.getFitIntercept) {
+ Array("(Intercept)") ++ attrs.attributes.get.map(_.name.get)
+ } else {
+ attrs.attributes.get.map(_.name.get)
+ }
case m: LogisticRegressionModel =>
val attrs = AttributeGroup.fromStructField(
m.summary.predictions.schema(m.summary.featuresCol))
- Array("(Intercept)") ++ attrs.attributes.get.map(_.name.get)
+ if (m.getFitIntercept) {
+ Array("(Intercept)") ++ attrs.attributes.get.map(_.name.get)
+ } else {
+ attrs.attributes.get.map(_.name.get)
+ }
+ }
+ }
+
+ def getModelName(model: PipelineModel): String = {
+ model.stages.last match {
+ case m: LinearRegressionModel =>
+ "LinearRegressionModel"
+ case m: LogisticRegressionModel =>
+ "LogisticRegressionModel"
}
}
}