From 39162e955fcebf366e458f755cdce2f542728bc9 Mon Sep 17 00:00:00 2001 From: Thomas Prelle Date: Thu, 11 Nov 2021 08:23:54 -0500 Subject: [PATCH] BACKPORT Hive25377 : Creating Iceberg table where some columns has comments but the last one does not fails --- .../apache/iceberg/hive/HiveSchemaConverter.java | 2 +- .../apache/iceberg/hive/TestHiveSchemaUtil.java | 16 ++++++++++++++++ .../TestHiveIcebergStorageHandlerNoScan.java | 5 +++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java index 2466cace5b4c..5dad471b3687 100644 --- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java +++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java @@ -64,7 +64,7 @@ List convertInternal(List names, List typeI List result = new ArrayList<>(names.size()); for (int i = 0; i < names.size(); ++i) { result.add(Types.NestedField.optional(id++, names.get(i), convertType(typeInfos.get(i)), - comments.isEmpty() ? null : comments.get(i))); + (comments.isEmpty() || i >= comments.size()) ? null : comments.get(i))); } return result; diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java index 0137adf4b6f1..fe246777a0bd 100644 --- a/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java +++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java @@ -154,6 +154,22 @@ public void testComplexTypeAndTypeInfoConvert() { } } + @Test + public void testConversionWithoutLastComment() { + Schema expected = new Schema( + optional(0, "customer_id", Types.LongType.get(), "customer comment"), + optional(1, "first_name", Types.StringType.get(), null) + ); + + Schema schema = HiveSchemaUtil.convert( + Arrays.asList("customer_id", "first_name"), + Arrays.asList(TypeInfoUtils.getTypeInfoFromTypeString(serdeConstants.BIGINT_TYPE_NAME), + TypeInfoUtils.getTypeInfoFromTypeString(serdeConstants.STRING_TYPE_NAME)), + Arrays.asList("customer comment")); + + Assert.assertEquals(expected.asStruct(), schema.asStruct()); + } + protected List getSupportedFieldSchemas() { List fields = new ArrayList<>(); fields.add(new FieldSchema("c_float", serdeConstants.FLOAT_TYPE_NAME, "float comment")); diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java index 9ba73a3d4cd2..a6c8b7f55d86 100644 --- a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java +++ b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java @@ -547,7 +547,8 @@ public void testCreateTableWithColumnComments() { TableIdentifier identifier = TableIdentifier.of("default", "comment_table"); shell.executeStatement("CREATE EXTERNAL TABLE comment_table (" + "t_int INT COMMENT 'int column', " + - "t_string STRING COMMENT 'string column') " + + "t_string STRING COMMENT 'string column', " + + "t_string_2 STRING) " + "STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler' " + testTables.locationForCreateTableSQL(identifier) + testTables.propertiesForCreateTableSQL(ImmutableMap.of())); @@ -558,7 +559,7 @@ public void testCreateTableWithColumnComments() { for (int i = 0; i < icebergTable.schema().columns().size(); i++) { Types.NestedField field = icebergTable.schema().columns().get(i); Assert.assertArrayEquals(new Object[] {field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(), - field.doc()}, rows.get(i)); + (field.doc() != null ? field.doc() : "from deserializer")}, rows.get(i)); } }