diff --git a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala index d91c3294ddb8bd4292533e40e25520fd17dc3bec..968a72d5adae9c49ea9598349765292c3a6ce6dd 100644 --- a/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala +++ b/core/src/main/scala/org/apache/spark/util/SizeEstimator.scala @@ -75,7 +75,8 @@ private[spark] object SizeEstimator extends Logging { // Sets object size, pointer size based on architecture and CompressedOops settings // from the JVM. private def initialize() { - is64bit = System.getProperty("os.arch").contains("64") + val arch = System.getProperty("os.arch") + is64bit = arch.contains("64") || arch.contains("s390x") isCompressedOops = getIsCompressedOops objectSize = if (!is64bit) 8 else { @@ -97,6 +98,11 @@ private[spark] object SizeEstimator extends Logging { return System.getProperty("spark.test.useCompressedOops").toBoolean } + // java.vm.info provides compressed ref info for IBM JDKs + if (System.getProperty("java.vendor").contains("IBM")) { + return System.getProperty("java.vm.info").contains("Compressed Ref") + } + try { val hotSpotMBeanName = "com.sun.management:type=HotSpotDiagnostic" val server = ManagementFactory.getPlatformMBeanServer() diff --git a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala index 133a76f28e0003da66b96e73c1c235e1ea679542..04f0f3749d6b97cbfc649eec5b395c9e7c5d321f 100644 --- a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala @@ -45,6 +45,10 @@ class DummyClass6 extends DummyClass5 { val y: Boolean = true } +class DummyClass7 { + val x: DummyClass1 = new DummyClass1 +} + object DummyString { def apply(str: String) : DummyString = new DummyString(str.toArray) } @@ -197,4 +201,12 @@ class SizeEstimatorSuite assertResult(24)(SizeEstimator.estimate(new DummyClass5)) assertResult(32)(SizeEstimator.estimate(new DummyClass6)) } + + test("check 64-bit detection for s390x arch") { + System.setProperty("os.arch", "s390x") + val initialize = PrivateMethod[Unit]('initialize) + SizeEstimator invokePrivate initialize() + // Class should be 32 bytes on s390x if recognised as 64 bit platform + assertResult(32)(SizeEstimator.estimate(new DummyClass7)) + } }