From ae9e42479253a9cd30423476405377f2d7952137 Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Thu, 17 Aug 2017 13:00:37 -0700
Subject: [PATCH] [SQL][MINOR][TEST] Set spark.unsafe.exceptionOnMemoryLeak to
true
## What changes were proposed in this pull request?
When running IntelliJ, we are unable to capture the exception of memory leak detection.
> org.apache.spark.executor.Executor: Managed memory leak detected
Explicitly setting `spark.unsafe.exceptionOnMemoryLeak` in SparkConf when building the SparkSession, instead of reading it from system properties.
## How was this patch tested?
N/A
Author: gatorsmile <gatorsmile@gmail.com>
Closes #18967 from gatorsmile/setExceptionOnMemoryLeak.
---
.../scala/org/apache/spark/sql/test/SharedSQLContext.scala | 4 +++-
.../main/scala/org/apache/spark/sql/hive/test/TestHive.scala | 3 ++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala
index 5ec76a4f0e..1f073d5f64 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSQLContext.scala
@@ -31,7 +31,9 @@ import org.apache.spark.sql.{SparkSession, SQLContext}
trait SharedSQLContext extends SQLTestUtils with BeforeAndAfterEach with Eventually {
protected def sparkConf = {
- new SparkConf().set("spark.hadoop.fs.file.impl", classOf[DebugFilesystem].getName)
+ new SparkConf()
+ .set("spark.hadoop.fs.file.impl", classOf[DebugFilesystem].getName)
+ .set("spark.unsafe.exceptionOnMemoryLeak", "true")
}
/**
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
index 9e15baa4b2..10c9a2de65 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -55,7 +55,8 @@ object TestHive
"org.apache.spark.sql.hive.execution.PairSerDe")
.set("spark.sql.warehouse.dir", TestHiveContext.makeWarehouseDir().toURI.getPath)
// SPARK-8910
- .set("spark.ui.enabled", "false")))
+ .set("spark.ui.enabled", "false")
+ .set("spark.unsafe.exceptionOnMemoryLeak", "true")))
case class TestHiveVersion(hiveClient: HiveClient)
--
GitLab