From 704b249b6a3ea956086d6c6ef50da18e8228eeb4 Mon Sep 17 00:00:00 2001
From: Shixiong Zhu <shixiong@databricks.com>
Date: Fri, 5 May 2017 11:08:26 -0700
Subject: [PATCH] [SPARK-20603][SS][TEST] Set default number of topic
partitions to 1 to reduce the load
## What changes were proposed in this pull request?
I checked the logs of https://amplab.cs.berkeley.edu/jenkins/job/spark-branch-2.2-test-maven-hadoop-2.7/47/ and found it took several seconds to create Kafka internal topic `__consumer_offsets`. As Kafka creates this topic lazily, the topic creation happens in the first test `deserialization of initial offset with Spark 2.1.0` and causes it timeout.
This PR changes `offsets.topic.num.partitions` from the default value 50 to 1 to make creating `__consumer_offsets` (50 partitions -> 1 partition) much faster.
## How was this patch tested?
Jenkins
Author: Shixiong Zhu <shixiong@databricks.com>
Closes #17863 from zsxwing/fix-kafka-flaky-test.
(cherry picked from commit bd5788287957d8610a6d19c273b75bd4cdd2d166)
Signed-off-by: Shixiong Zhu <shixiong@databricks.com>
---
.../scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala | 1 +
1 file changed, 1 insertion(+)
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index c2cbd86260..4345f88213 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -281,6 +281,7 @@ class KafkaTestUtils(withBrokerProps: Map[String, Object] = Map.empty) extends L
props.put("log.flush.interval.messages", "1")
props.put("replica.socket.timeout.ms", "1500")
props.put("delete.topic.enable", "true")
+ props.put("offsets.topic.num.partitions", "1")
props.putAll(withBrokerProps.asJava)
props
}
--
GitLab