From 6f61e1f961826a6c9e98a66d10b271b7e3c7dd55 Mon Sep 17 00:00:00 2001
From: Cheng Lian <lian@databricks.com>
Date: Fri, 5 Dec 2014 10:27:40 -0800
Subject: [PATCH] [SPARK-4761][SQL] Enables Kryo by default in Spark SQL Thrift
server
Enables Kryo and disables reference tracking by default in Spark SQL Thrift server. Configurations explicitly defined by users in `spark-defaults.conf` are respected (the Thrift server is started by `spark-submit`, which handles configuration properties properly).
<!-- Reviewable:start -->
[<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/apache/spark/3621)
<!-- Reviewable:end -->
Author: Cheng Lian <lian@databricks.com>
Closes #3621 from liancheng/kryo-by-default and squashes the following commits:
70c2775 [Cheng Lian] Enables Kryo by default in Spark SQL Thrift server
---
.../spark/sql/hive/thriftserver/SparkSQLEnv.scala | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
index 89732c939b..158c225159 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
@@ -32,11 +32,21 @@ private[hive] object SparkSQLEnv extends Logging {
def init() {
if (hiveContext == null) {
- val sparkConf = new SparkConf()
+ val sparkConf = new SparkConf(loadDefaults = true)
+ val maybeSerializer = sparkConf.getOption("spark.serializer")
+ val maybeKryoReferenceTracking = sparkConf.getOption("spark.kryo.referenceTracking")
+
+ sparkConf
.setAppName(s"SparkSQL::${java.net.InetAddress.getLocalHost.getHostName}")
.set("spark.sql.hive.version", HiveShim.version)
- sparkContext = new SparkContext(sparkConf)
+ .set(
+ "spark.serializer",
+ maybeSerializer.getOrElse("org.apache.spark.serializer.KryoSerializer"))
+ .set(
+ "spark.kryo.referenceTracking",
+ maybeKryoReferenceTracking.getOrElse("false"))
+ sparkContext = new SparkContext(sparkConf)
sparkContext.addSparkListener(new StatsReportListener())
hiveContext = new HiveContext(sparkContext)
--
GitLab