From 90a6a46bd11030672597f015dd443d954107123a Mon Sep 17 00:00:00 2001
From: Michael Armbrust <michael@databricks.com>
Date: Thu, 20 Nov 2014 20:34:43 -0800
Subject: [PATCH] [SPARK-4522][SQL] Parse schema with missing metadata.
This is just a quick fix for 1.2. SPARK-4523 describes a more complete solution.
Author: Michael Armbrust <michael@databricks.com>
Closes #3392 from marmbrus/parquetMetadata and squashes the following commits:
bcc6626 [Michael Armbrust] Parse schema with missing metadata.
---
.../org/apache/spark/sql/catalyst/types/dataTypes.scala | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/dataTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/dataTypes.scala
index ff1dc03069..892b7e1a97 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/dataTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/dataTypes.scala
@@ -84,6 +84,12 @@ object DataType {
("nullable", JBool(nullable)),
("type", dataType: JValue)) =>
StructField(name, parseDataType(dataType), nullable, Metadata.fromJObject(metadata))
+ // Support reading schema when 'metadata' is missing.
+ case JSortedObject(
+ ("name", JString(name)),
+ ("nullable", JBool(nullable)),
+ ("type", dataType: JValue)) =>
+ StructField(name, parseDataType(dataType), nullable)
}
@deprecated("Use DataType.fromJson instead", "1.2.0")
--
GitLab