Skip to content
Snippets Groups Projects
Commit fc47bb69 authored by GuoQiang Li's avatar GuoQiang Li Committed by Xiangrui Meng
Browse files

[SPARK-2544][MLLIB] Improve ALS algorithm resource usage

Author: GuoQiang Li <witgo@qq.com>
Author: witgo <witgo@qq.com>

Closes #929 from witgo/improve_als and squashes the following commits:

ea25033 [GuoQiang Li] checkpoint products 3,6,9 ...
154dccf [GuoQiang Li] checkpoint products only
c5779ff [witgo] Improve ALS algorithm resource usage
parent e3d85b7e
No related branches found
No related tags found
No related merge requests found
......@@ -255,6 +255,9 @@ class ALS private (
rank, lambda, alpha, YtY)
previousProducts.unpersist()
logInfo("Re-computing U given I (Iteration %d/%d)".format(iter, iterations))
if (sc.checkpointDir.isDefined && (iter % 3 == 0)) {
products.checkpoint()
}
products.setName(s"products-$iter").persist()
val XtX = Some(sc.broadcast(computeYtY(products)))
val previousUsers = users
......@@ -268,6 +271,9 @@ class ALS private (
logInfo("Re-computing I given U (Iteration %d/%d)".format(iter, iterations))
products = updateFeatures(numProductBlocks, users, userOutLinks, productInLinks,
rank, lambda, alpha, YtY = None)
if (sc.checkpointDir.isDefined && (iter % 3 == 0)) {
products.checkpoint()
}
products.setName(s"products-$iter")
logInfo("Re-computing U given I (Iteration %d/%d)".format(iter, iterations))
users = updateFeatures(numUserBlocks, products, productOutLinks, userInLinks,
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment