Skip to content
Snippets Groups Projects
Commit cbfbc019 authored by jerryshao's avatar jerryshao
Browse files

Fix configure didn't work small problem in ALS

parent 1d7bef0c
No related branches found
No related tags found
No related merge requests found
...@@ -22,7 +22,7 @@ import scala.util.Random ...@@ -22,7 +22,7 @@ import scala.util.Random
import scala.util.Sorting import scala.util.Sorting
import org.apache.spark.broadcast.Broadcast import org.apache.spark.broadcast.Broadcast
import org.apache.spark.{Logging, HashPartitioner, Partitioner, SparkContext} import org.apache.spark.{Logging, HashPartitioner, Partitioner, SparkContext, SparkConf}
import org.apache.spark.storage.StorageLevel import org.apache.spark.storage.StorageLevel
import org.apache.spark.rdd.RDD import org.apache.spark.rdd.RDD
import org.apache.spark.serializer.KryoRegistrator import org.apache.spark.serializer.KryoRegistrator
...@@ -578,12 +578,13 @@ object ALS { ...@@ -578,12 +578,13 @@ object ALS {
val implicitPrefs = if (args.length >= 7) args(6).toBoolean else false val implicitPrefs = if (args.length >= 7) args(6).toBoolean else false
val alpha = if (args.length >= 8) args(7).toDouble else 1 val alpha = if (args.length >= 8) args(7).toDouble else 1
val blocks = if (args.length == 9) args(8).toInt else -1 val blocks = if (args.length == 9) args(8).toInt else -1
val sc = new SparkContext(master, "ALS") val conf = new SparkConf()
sc.conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
sc.conf.set("spark.kryo.registrator", classOf[ALSRegistrator].getName) .set("spark.kryo.registrator", classOf[ALSRegistrator].getName)
sc.conf.set("spark.kryo.referenceTracking", "false") .set("spark.kryo.referenceTracking", "false")
sc.conf.set("spark.kryoserializer.buffer.mb", "8") .set("spark.kryoserializer.buffer.mb", "8")
sc.conf.set("spark.locality.wait", "10000") .set("spark.locality.wait", "10000")
val sc = new SparkContext(master, "ALS", conf)
val ratings = sc.textFile(ratingsFile).map { line => val ratings = sc.textFile(ratingsFile).map { line =>
val fields = line.split(',') val fields = line.split(',')
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment