Skip to content

GH-3401: Remove redundant MAP_KEY_VALUE annotation at middle level of logical type map #3402

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HuaHuaY wants to merge 1 commit into
base: master
Choose a base branch
from
+22 −31
Open

GH-3401: Remove redundant MAP_KEY_VALUE annotation at middle level of logical type map #3402

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions parquet-avro/src/main/java/org/apache/parquet/avro/package-info.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@
* <tr>
* <td>map</td>
* <td>group (with original type MAP) containing one repeated group
* field (with original type MAP_KEY_VALUE) of (key, value)</td>
* field of (key, value)</td>
* </tr>
* <tr>
* <td>fixed</td>
Expand Down Expand Up @@ -148,7 +148,7 @@
* </tr>
* <tr>
* <td>group (with original type MAP) containing one repeated group
* field (with original type MAP_KEY_VALUE) of (key, value)</td>
* field of (key, value)</td>
* <td>map</td>
* </tr>
* </table>
Expand Down
14 changes: 7 additions & 7 deletions parquet-avro/src/test/java/org/apache/parquet/avro/TestAvroSchemaConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ public static void setupConf() {
+ " }\n"
+ " }\n"
+ " required group mymap (MAP) {\n"
+ " repeated group map (MAP_KEY_VALUE) {\n"
+ " repeated group map {\n"
+ " required binary key (UTF8);\n"
+ " required int32 value;\n"
+ " }\n"
Expand Down Expand Up @@ -212,13 +212,13 @@ public void testAllTypes() throws Exception {
+ " }\n"
+ " }\n"
+ " required group mymap (MAP) {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " required int32 value;\n"
+ " }\n"
+ " }\n"
+ " required group myemptymap (MAP) {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " required int32 value;\n"
+ " }\n"
Expand Down Expand Up @@ -259,13 +259,13 @@ public void testAllTypesOldListBehavior() throws Exception {
+ " repeated int32 array;\n"
+ " }\n"
+ " required group mymap (MAP) {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " required int32 value;\n"
+ " }\n"
+ " }\n"
+ " required group myemptymap (MAP) {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " required int32 value;\n"
+ " }\n"
Expand Down Expand Up @@ -320,7 +320,7 @@ public void testOptionalMapValue() throws Exception {
testRoundTripConversion(
schema,
"message record1 {\n" + " required group myintmap (MAP) {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " optional int32 value;\n"
+ " }\n"
Expand Down Expand Up @@ -971,7 +971,7 @@ public void testAvroFixed12AsParquetInt96Type() throws Exception {
+ " repeated int96 array;\n"
+ " }\n"
+ " required group mymap (MAP) {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required binary key (STRING);\n"
+ " required int96 value;\n"
+ " }\n"
Expand Down
13 changes: 2 additions & 11 deletions parquet-column/src/main/java/org/apache/parquet/schema/ConversionPatterns.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,7 @@ public static GroupType mapType(
repetition,
alias,
LogicalTypeAnnotation.mapType(),
new GroupType(
Repetition.REPEATED,
mapAlias,
LogicalTypeAnnotation.MapKeyValueTypeAnnotation.getInstance(),
keyType));
new GroupType(Repetition.REPEATED, mapAlias, keyType));
} else {
if (!valueType.getName().equals("value")) {
throw new RuntimeException(valueType.getName() + " should be value");
Expand All @@ -88,12 +84,7 @@ public static GroupType mapType(
repetition,
alias,
LogicalTypeAnnotation.mapType(),
new GroupType(
Repetition.REPEATED,
mapAlias,
LogicalTypeAnnotation.MapKeyValueTypeAnnotation.getInstance(),
keyType,
valueType));
new GroupType(Repetition.REPEATED, mapAlias, keyType, valueType));
}
}

Expand Down
4 changes: 2 additions & 2 deletions parquet-column/src/main/java/org/apache/parquet/schema/Types.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@
*
* <pre>
* // required group zipMap (MAP) {
* // repeated group map (MAP_KEY_VALUE) {
* // repeated group map {
* // required float key
* // optional int32 value
* // }
Expand All @@ -122,7 +122,7 @@
*
*
* // required group zipMap (MAP) {
* // repeated group map (MAP_KEY_VALUE) {
* // repeated group map {
* // required group key {
* // optional int64 first;
* // required group second {
Expand Down
12 changes: 6 additions & 6 deletions parquet-thrift/src/test/java/org/apache/parquet/thrift/TestThriftSchemaConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,15 +145,15 @@ public void testToProjectedThriftType() {
message TestStructInMap {
optional binary name(UTF8);
optional group names(MAP) {
repeated group map(MAP_KEY_VALUE) {
repeated group map {
required binary key(UTF8);
optional group value {
optional group name {
optional binary first_name(UTF8);
optional binary last_name(UTF8);
}
optional group phones(MAP) {
repeated group map(MAP_KEY_VALUE) {
repeated group map {
required binary key(ENUM);
optional binary value(UTF8);
}
Expand All @@ -171,15 +171,15 @@ public void testProjectMapThriftType() {
"name;names.key*;names.value",
"message ParquetSchema {\n" + " optional binary name (UTF8) = 1;\n"
+ " optional group names (MAP) = 2 {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " optional group value {\n"
+ " optional group name = 1 {\n"
+ " optional binary first_name (UTF8) = 1;\n"
+ " optional binary last_name (UTF8) = 2;\n"
+ " }\n"
+ " optional group phones (MAP) = 2 {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required binary key (ENUM);\n"
+ " optional binary value (UTF8);\n"
+ " }\n"
Expand All @@ -196,7 +196,7 @@ public void testProjectMapThriftType() {
"name;names.key;names.value.name",
"message ParquetSchema {\n" + " optional binary name (UTF8) = 1;\n"
+ " optional group names (MAP) = 2 {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " optional group value {\n"
+ " optional group name = 1 {\n"
Expand All @@ -217,7 +217,7 @@ public void testProjectOnlyKeyInMap() {
"name;names.key",
"message ParquetSchema {\n" + " optional binary name (UTF8) = 1;\n"
+ " optional group names (MAP) = 2 {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required binary key (UTF8);\n"
+ " optional group value {\n"
+ " optional group name = 1 {\n"
Expand Down
6 changes: 3 additions & 3 deletions ...thrift/src/test/java/org/apache/parquet/thrift/TestThriftSchemaConverterProjectUnion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ public void testMapWithUnionKey() {
"optMapWithUnionKey/key/**",
"optMapWithUnionKey.key",
"message ParquetSchema {\n" + " optional group optMapWithUnionKey (MAP) = 1 {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required group key {\n"
+ " optional group structV3 = 1 {\n"
+ " required binary name (UTF8) = 1;\n"
Expand Down Expand Up @@ -348,7 +348,7 @@ public void testMapWithUnionKey() {
"optMapWithUnionKey/key/**;optMapWithUnionKey/value/gender",
"optMapWithUnionKey.{key,value.gender}",
"message ParquetSchema {\n" + " optional group optMapWithUnionKey (MAP) = 1 {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required group key {\n"
+ " optional group structV3 = 1 {\n"
+ " required binary name (UTF8) = 1;\n"
Expand Down Expand Up @@ -384,7 +384,7 @@ public void testMapWithUnionValue() {
"optMapWithUnionValue/key/**;optMapWithUnionValue/value/structV4/addedStruct/gender",
"optMapWithUnionValue.{key,value.structV4.addedStruct.gender}",
"message ParquetSchema {\n" + " optional group optMapWithUnionValue (MAP) = 1 {\n"
+ " repeated group key_value (MAP_KEY_VALUE) {\n"
+ " repeated group key_value {\n"
+ " required group key {\n"
+ " required binary name (UTF8) = 1;\n"
+ " optional binary age (UTF8) = 2;\n"
Expand Down