From 4432568aac1d4a44fa1a7c3469f095eb7a6ce945 Mon Sep 17 00:00:00 2001
From: Yuhao Yang <hhbyyh@gmail.com>
Date: Mon, 19 Jan 2015 10:10:15 -0800
Subject: [PATCH] [SPARK-5282][mllib]: RowMatrix easily gets int overflow in
the memory size warning
JIRA: https://issues.apache.org/jira/browse/SPARK-5282
fix the possible int overflow in the memory computation warning
Author: Yuhao Yang <hhbyyh@gmail.com>
Closes #4069 from hhbyyh/addscStop and squashes the following commits:
e54e5c8 [Yuhao Yang] change to MB based number
7afac23 [Yuhao Yang] 5282: fix int overflow in the warning
---
.../org/apache/spark/mllib/linalg/distributed/RowMatrix.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
index d5abba6a4b..02075edbab 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
@@ -131,8 +131,8 @@ class RowMatrix(
throw new IllegalArgumentException(s"Argument with more than 65535 cols: $cols")
}
if (cols > 10000) {
- val mem = cols * cols * 8
- logWarning(s"$cols columns will require at least $mem bytes of memory!")
+ val memMB = (cols.toLong * cols) / 125000
+ logWarning(s"$cols columns will require at least $memMB megabytes of memory!")
}
}
--
GitLab