Skip to content
Snippets Groups Projects
Commit c056484c authored by Cheng Lian's avatar Cheng Lian
Browse files

Revert "[SPARK-8928] [SQL] Makes CatalystSchemaConverter sticking to 1.4.x-...

Revert "[SPARK-8928] [SQL] Makes CatalystSchemaConverter sticking to 1.4.x- when handling Parquet LISTs in compatible mode"

This reverts commit 3dab0da4.
parent 3dab0da4
No related branches found
No related tags found
No related merge requests found
...@@ -461,8 +461,7 @@ private[parquet] class CatalystSchemaConverter( ...@@ -461,8 +461,7 @@ private[parquet] class CatalystSchemaConverter(
field.name, field.name,
Types Types
.buildGroup(REPEATED) .buildGroup(REPEATED)
// "array_element" is the name chosen by parquet-hive (1.7.0 and prior version) .addField(convertField(StructField("element", elementType, nullable)))
.addField(convertField(StructField("array_element", elementType, nullable)))
.named(CatalystConverter.ARRAY_CONTAINS_NULL_BAG_SCHEMA_NAME)) .named(CatalystConverter.ARRAY_CONTAINS_NULL_BAG_SCHEMA_NAME))
// Spark 1.4.x and prior versions convert ArrayType with non-nullable elements into a 2-level // Spark 1.4.x and prior versions convert ArrayType with non-nullable elements into a 2-level
...@@ -475,8 +474,7 @@ private[parquet] class CatalystSchemaConverter( ...@@ -475,8 +474,7 @@ private[parquet] class CatalystSchemaConverter(
ConversionPatterns.listType( ConversionPatterns.listType(
repetition, repetition,
field.name, field.name,
// "array" is the name chosen by parquet-avro (1.7.0 and prior version) convertField(StructField("element", elementType, nullable), REPEATED))
convertField(StructField("array", elementType, nullable), REPEATED))
// Spark 1.4.x and prior versions convert MapType into a 3-level group annotated by // Spark 1.4.x and prior versions convert MapType into a 3-level group annotated by
// MAP_KEY_VALUE. This is covered by `convertGroupField(field: GroupType): DataType`. // MAP_KEY_VALUE. This is covered by `convertGroupField(field: GroupType): DataType`.
......
...@@ -174,7 +174,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest { ...@@ -174,7 +174,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest {
""" """
|message root { |message root {
| optional group _1 (LIST) { | optional group _1 (LIST) {
| repeated int32 array; | repeated int32 element;
| } | }
|} |}
""".stripMargin) """.stripMargin)
...@@ -198,7 +198,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest { ...@@ -198,7 +198,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest {
|message root { |message root {
| optional group _1 (LIST) { | optional group _1 (LIST) {
| repeated group bag { | repeated group bag {
| optional int32 array_element; | optional int32 element;
| } | }
| } | }
|} |}
...@@ -267,7 +267,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest { ...@@ -267,7 +267,7 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest {
| optional binary _1 (UTF8); | optional binary _1 (UTF8);
| optional group _2 (LIST) { | optional group _2 (LIST) {
| repeated group bag { | repeated group bag {
| optional group array_element { | optional group element {
| required int32 _1; | required int32 _1;
| required double _2; | required double _2;
| } | }
...@@ -616,7 +616,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest { ...@@ -616,7 +616,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
"""message root { """message root {
| optional group f1 (LIST) { | optional group f1 (LIST) {
| repeated group bag { | repeated group bag {
| optional int32 array_element; | optional int32 element;
| } | }
| } | }
|} |}
...@@ -648,7 +648,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest { ...@@ -648,7 +648,7 @@ class ParquetSchemaSuite extends ParquetSchemaTest {
nullable = true))), nullable = true))),
"""message root { """message root {
| optional group f1 (LIST) { | optional group f1 (LIST) {
| repeated int32 array; | repeated int32 element;
| } | }
|} |}
""".stripMargin) """.stripMargin)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment