From 4cc704b12194cfa70717fb35d90b8b2ccda29866 Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Tue, 1 Aug 2017 17:10:46 -0700
Subject: [PATCH] [CORE][MINOR] Improve the error message of checkpoint RDD
verification
### What changes were proposed in this pull request?
The original error message is pretty confusing. It is unable to tell which number is `number of partitions` and which one is the `RDD ID`. This PR is to improve the checkpoint checking.
### How was this patch tested?
N/A
Author: gatorsmile <gatorsmile@gmail.com>
Closes #18796 from gatorsmile/improveErrMsgForCheckpoint.
---
.../scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala b/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
index 37c67cee55..979152b55f 100644
--- a/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/ReliableCheckpointRDD.scala
@@ -152,8 +152,10 @@ private[spark] object ReliableCheckpointRDD extends Logging {
sc, checkpointDirPath.toString, originalRDD.partitioner)
if (newRDD.partitions.length != originalRDD.partitions.length) {
throw new SparkException(
- s"Checkpoint RDD $newRDD(${newRDD.partitions.length}) has different " +
- s"number of partitions from original RDD $originalRDD(${originalRDD.partitions.length})")
+ "Checkpoint RDD has a different number of partitions from original RDD. Original " +
+ s"RDD [ID: ${originalRDD.id}, num of partitions: ${originalRDD.partitions.length}]; " +
+ s"Checkpoint RDD [ID: ${newRDD.id}, num of partitions: " +
+ s"${newRDD.partitions.length}].")
}
newRDD
}
--
GitLab