From dc4c351837879dab26ad8fb471dc51c06832a9e4 Mon Sep 17 00:00:00 2001
From: Felix Cheung <felixcheung_m@hotmail.com>
Date: Sun, 11 Jun 2017 00:00:33 -0700
Subject: [PATCH] [SPARK-20877][SPARKR] refactor tests to basic tests only for
CRAN
## What changes were proposed in this pull request?
Move all existing tests to non-installed directory so that it will never run by installing SparkR package
For a follow-up PR:
- remove all skip_on_cran() calls in tests
- clean up test timer
- improve or change basic tests that do run on CRAN (if anyone has suggestion)
It looks like `R CMD build pkg` will still put pkg\tests (ie. the full tests) into the source package but `R CMD INSTALL` on such source package does not install these tests (and so `R CMD check` does not run them)
## How was this patch tested?
- [x] unit tests, Jenkins
- [x] AppVeyor
- [x] make a source package, install it, `R CMD check` it - verify the full tests are not installed or run
Author: Felix Cheung <felixcheung_m@hotmail.com>
Closes #18264 from felixcheung/rtestset.
---
R/pkg/inst/tests/testthat/test_basic.R | 90 +++++++++++++++++++
.../testthat => tests/fulltests}/jarTest.R | 0
.../fulltests}/packageInAJarTest.R | 0
.../testthat => tests/fulltests}/test_Serde.R | 0
.../fulltests}/test_Windows.R | 0
.../fulltests}/test_binaryFile.R | 0
.../fulltests}/test_binary_function.R | 0
.../fulltests}/test_broadcast.R | 0
.../fulltests}/test_client.R | 0
.../fulltests}/test_context.R | 0
.../fulltests}/test_includePackage.R | 0
.../fulltests}/test_jvm_api.R | 0
.../fulltests}/test_mllib_classification.R | 0
.../fulltests}/test_mllib_clustering.R | 0
.../fulltests}/test_mllib_fpm.R | 0
.../fulltests}/test_mllib_recommendation.R | 0
.../fulltests}/test_mllib_regression.R | 0
.../fulltests}/test_mllib_stat.R | 0
.../fulltests}/test_mllib_tree.R | 0
.../fulltests}/test_parallelize_collect.R | 0
.../testthat => tests/fulltests}/test_rdd.R | 0
.../fulltests}/test_shuffle.R | 0
.../fulltests}/test_sparkR.R | 0
.../fulltests}/test_sparkSQL.R | 0
.../fulltests}/test_streaming.R | 0
.../testthat => tests/fulltests}/test_take.R | 0
.../fulltests}/test_textFile.R | 0
.../testthat => tests/fulltests}/test_utils.R | 0
R/pkg/tests/run-all.R | 8 ++
29 files changed, 98 insertions(+)
create mode 100644 R/pkg/inst/tests/testthat/test_basic.R
rename R/pkg/{inst/tests/testthat => tests/fulltests}/jarTest.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/packageInAJarTest.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_Serde.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_Windows.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_binaryFile.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_binary_function.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_broadcast.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_client.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_context.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_includePackage.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_jvm_api.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_mllib_classification.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_mllib_clustering.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_mllib_fpm.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_mllib_recommendation.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_mllib_regression.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_mllib_stat.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_mllib_tree.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_parallelize_collect.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_rdd.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_shuffle.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_sparkR.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_sparkSQL.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_streaming.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_take.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_textFile.R (100%)
rename R/pkg/{inst/tests/testthat => tests/fulltests}/test_utils.R (100%)
diff --git a/R/pkg/inst/tests/testthat/test_basic.R b/R/pkg/inst/tests/testthat/test_basic.R
new file mode 100644
index 0000000000..de47162d53
--- /dev/null
+++ b/R/pkg/inst/tests/testthat/test_basic.R
@@ -0,0 +1,90 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+context("basic tests for CRAN")
+
+test_that("create DataFrame from list or data.frame", {
+ sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE)
+
+ i <- 4
+ df <- createDataFrame(data.frame(dummy = 1:i))
+ expect_equal(count(df), i)
+
+ l <- list(list(a = 1, b = 2), list(a = 3, b = 4))
+ df <- createDataFrame(l)
+ expect_equal(columns(df), c("a", "b"))
+
+ a <- 1:3
+ b <- c("a", "b", "c")
+ ldf <- data.frame(a, b)
+ df <- createDataFrame(ldf)
+ expect_equal(columns(df), c("a", "b"))
+ expect_equal(dtypes(df), list(c("a", "int"), c("b", "string")))
+ expect_equal(count(df), 3)
+ ldf2 <- collect(df)
+ expect_equal(ldf$a, ldf2$a)
+
+ mtcarsdf <- createDataFrame(mtcars)
+ expect_equivalent(collect(mtcarsdf), mtcars)
+
+ bytes <- as.raw(c(1, 2, 3))
+ df <- createDataFrame(list(list(bytes)))
+ expect_equal(collect(df)[[1]][[1]], bytes)
+
+ sparkR.session.stop()
+})
+
+test_that("spark.glm and predict", {
+ sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE)
+
+ training <- suppressWarnings(createDataFrame(iris))
+ # gaussian family
+ model <- spark.glm(training, Sepal_Width ~ Sepal_Length + Species)
+ prediction <- predict(model, training)
+ expect_equal(typeof(take(select(prediction, "prediction"), 1)$prediction), "double")
+ vals <- collect(select(prediction, "prediction"))
+ rVals <- predict(glm(Sepal.Width ~ Sepal.Length + Species, data = iris), iris)
+ expect_true(all(abs(rVals - vals) < 1e-6), rVals - vals)
+
+ # Gamma family
+ x <- runif(100, -1, 1)
+ y <- rgamma(100, rate = 10 / exp(0.5 + 1.2 * x), shape = 10)
+ df <- as.DataFrame(as.data.frame(list(x = x, y = y)))
+ model <- glm(y ~ x, family = Gamma, df)
+ out <- capture.output(print(summary(model)))
+ expect_true(any(grepl("Dispersion parameter for gamma family", out)))
+
+ # tweedie family
+ model <- spark.glm(training, Sepal_Width ~ Sepal_Length + Species,
+ family = "tweedie", var.power = 1.2, link.power = 0.0)
+ prediction <- predict(model, training)
+ expect_equal(typeof(take(select(prediction, "prediction"), 1)$prediction), "double")
+ vals <- collect(select(prediction, "prediction"))
+
+ # manual calculation of the R predicted values to avoid dependence on statmod
+ #' library(statmod)
+ #' rModel <- glm(Sepal.Width ~ Sepal.Length + Species, data = iris,
+ #' family = tweedie(var.power = 1.2, link.power = 0.0))
+ #' print(coef(rModel))
+
+ rCoef <- c(0.6455409, 0.1169143, -0.3224752, -0.3282174)
+ rVals <- exp(as.numeric(model.matrix(Sepal.Width ~ Sepal.Length + Species,
+ data = iris) %*% rCoef))
+ expect_true(all(abs(rVals - vals) < 1e-5), rVals - vals)
+
+ sparkR.session.stop()
+})
diff --git a/R/pkg/inst/tests/testthat/jarTest.R b/R/pkg/tests/fulltests/jarTest.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/jarTest.R
rename to R/pkg/tests/fulltests/jarTest.R
diff --git a/R/pkg/inst/tests/testthat/packageInAJarTest.R b/R/pkg/tests/fulltests/packageInAJarTest.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/packageInAJarTest.R
rename to R/pkg/tests/fulltests/packageInAJarTest.R
diff --git a/R/pkg/inst/tests/testthat/test_Serde.R b/R/pkg/tests/fulltests/test_Serde.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_Serde.R
rename to R/pkg/tests/fulltests/test_Serde.R
diff --git a/R/pkg/inst/tests/testthat/test_Windows.R b/R/pkg/tests/fulltests/test_Windows.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_Windows.R
rename to R/pkg/tests/fulltests/test_Windows.R
diff --git a/R/pkg/inst/tests/testthat/test_binaryFile.R b/R/pkg/tests/fulltests/test_binaryFile.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_binaryFile.R
rename to R/pkg/tests/fulltests/test_binaryFile.R
diff --git a/R/pkg/inst/tests/testthat/test_binary_function.R b/R/pkg/tests/fulltests/test_binary_function.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_binary_function.R
rename to R/pkg/tests/fulltests/test_binary_function.R
diff --git a/R/pkg/inst/tests/testthat/test_broadcast.R b/R/pkg/tests/fulltests/test_broadcast.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_broadcast.R
rename to R/pkg/tests/fulltests/test_broadcast.R
diff --git a/R/pkg/inst/tests/testthat/test_client.R b/R/pkg/tests/fulltests/test_client.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_client.R
rename to R/pkg/tests/fulltests/test_client.R
diff --git a/R/pkg/inst/tests/testthat/test_context.R b/R/pkg/tests/fulltests/test_context.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_context.R
rename to R/pkg/tests/fulltests/test_context.R
diff --git a/R/pkg/inst/tests/testthat/test_includePackage.R b/R/pkg/tests/fulltests/test_includePackage.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_includePackage.R
rename to R/pkg/tests/fulltests/test_includePackage.R
diff --git a/R/pkg/inst/tests/testthat/test_jvm_api.R b/R/pkg/tests/fulltests/test_jvm_api.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_jvm_api.R
rename to R/pkg/tests/fulltests/test_jvm_api.R
diff --git a/R/pkg/inst/tests/testthat/test_mllib_classification.R b/R/pkg/tests/fulltests/test_mllib_classification.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_mllib_classification.R
rename to R/pkg/tests/fulltests/test_mllib_classification.R
diff --git a/R/pkg/inst/tests/testthat/test_mllib_clustering.R b/R/pkg/tests/fulltests/test_mllib_clustering.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_mllib_clustering.R
rename to R/pkg/tests/fulltests/test_mllib_clustering.R
diff --git a/R/pkg/inst/tests/testthat/test_mllib_fpm.R b/R/pkg/tests/fulltests/test_mllib_fpm.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_mllib_fpm.R
rename to R/pkg/tests/fulltests/test_mllib_fpm.R
diff --git a/R/pkg/inst/tests/testthat/test_mllib_recommendation.R b/R/pkg/tests/fulltests/test_mllib_recommendation.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_mllib_recommendation.R
rename to R/pkg/tests/fulltests/test_mllib_recommendation.R
diff --git a/R/pkg/inst/tests/testthat/test_mllib_regression.R b/R/pkg/tests/fulltests/test_mllib_regression.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_mllib_regression.R
rename to R/pkg/tests/fulltests/test_mllib_regression.R
diff --git a/R/pkg/inst/tests/testthat/test_mllib_stat.R b/R/pkg/tests/fulltests/test_mllib_stat.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_mllib_stat.R
rename to R/pkg/tests/fulltests/test_mllib_stat.R
diff --git a/R/pkg/inst/tests/testthat/test_mllib_tree.R b/R/pkg/tests/fulltests/test_mllib_tree.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_mllib_tree.R
rename to R/pkg/tests/fulltests/test_mllib_tree.R
diff --git a/R/pkg/inst/tests/testthat/test_parallelize_collect.R b/R/pkg/tests/fulltests/test_parallelize_collect.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_parallelize_collect.R
rename to R/pkg/tests/fulltests/test_parallelize_collect.R
diff --git a/R/pkg/inst/tests/testthat/test_rdd.R b/R/pkg/tests/fulltests/test_rdd.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_rdd.R
rename to R/pkg/tests/fulltests/test_rdd.R
diff --git a/R/pkg/inst/tests/testthat/test_shuffle.R b/R/pkg/tests/fulltests/test_shuffle.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_shuffle.R
rename to R/pkg/tests/fulltests/test_shuffle.R
diff --git a/R/pkg/inst/tests/testthat/test_sparkR.R b/R/pkg/tests/fulltests/test_sparkR.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_sparkR.R
rename to R/pkg/tests/fulltests/test_sparkR.R
diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_sparkSQL.R
rename to R/pkg/tests/fulltests/test_sparkSQL.R
diff --git a/R/pkg/inst/tests/testthat/test_streaming.R b/R/pkg/tests/fulltests/test_streaming.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_streaming.R
rename to R/pkg/tests/fulltests/test_streaming.R
diff --git a/R/pkg/inst/tests/testthat/test_take.R b/R/pkg/tests/fulltests/test_take.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_take.R
rename to R/pkg/tests/fulltests/test_take.R
diff --git a/R/pkg/inst/tests/testthat/test_textFile.R b/R/pkg/tests/fulltests/test_textFile.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_textFile.R
rename to R/pkg/tests/fulltests/test_textFile.R
diff --git a/R/pkg/inst/tests/testthat/test_utils.R b/R/pkg/tests/fulltests/test_utils.R
similarity index 100%
rename from R/pkg/inst/tests/testthat/test_utils.R
rename to R/pkg/tests/fulltests/test_utils.R
diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R
index f0bef4f6d2..d48e36c880 100644
--- a/R/pkg/tests/run-all.R
+++ b/R/pkg/tests/run-all.R
@@ -43,3 +43,11 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) {
}
test_package("SparkR")
+
+if (identical(Sys.getenv("NOT_CRAN"), "true")) {
+ # for testthat 1.0.2 later, change reporter from "summary" to default_reporter()
+ testthat:::run_tests("SparkR",
+ file.path(sparkRDir, "pkg", "tests", "fulltests"),
+ NULL,
+ "summary")
+}
--
GitLab