From 3a73f37e4802bc3948f634d27ce4c642e19e1a9d Mon Sep 17 00:00:00 2001 From: nk1506 Date: Sun, 17 Mar 2024 21:11:22 +0530 Subject: [PATCH 1/2] Hive: Test to do validation hive content and iceberg table with the same name. --- .../apache/iceberg/hive/HiveTableTest.java | 72 +++++++++++++++++-- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableTest.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableTest.java index 0fa6c94bf154..082ca34d0246 100644 --- a/hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableTest.java +++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableTest.java @@ -68,6 +68,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.NoSuchIcebergTableException; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.hadoop.ConfigProperties; @@ -334,7 +335,8 @@ public void testListTables() throws TException, IOException { // create a hive table String hiveTableName = "test_hive_table"; - org.apache.hadoop.hive.metastore.api.Table hiveTable = createHiveTable(hiveTableName); + org.apache.hadoop.hive.metastore.api.Table hiveTable = + createHiveTable(hiveTableName, TableType.EXTERNAL_TABLE); HIVE_METASTORE_EXTENSION.metastoreClient().createTable(hiveTable); catalog.setListAllTables(false); @@ -349,8 +351,70 @@ public void testListTables() throws TException, IOException { HIVE_METASTORE_EXTENSION.metastoreClient().dropTable(DB_NAME, hiveTableName); } - private org.apache.hadoop.hive.metastore.api.Table createHiveTable(String hiveTableName) - throws IOException { + @Test + public void testHiveTableAndIcebergTableWithSameName() throws TException, IOException { + List tableIdents = catalog.listTables(TABLE_IDENTIFIER.namespace()); + assertThat(tableIdents).as("should be 1 table in namespace .").hasSize(1); + + // create a hive table + String hiveTableName = "test_hive_table"; + org.apache.hadoop.hive.metastore.api.Table hiveTable = + createHiveTable(hiveTableName, TableType.EXTERNAL_TABLE); + HIVE_METASTORE_EXTENSION.metastoreClient().createTable(hiveTable); + + catalog.setListAllTables(true); + List tableIdents2 = catalog.listTables(TABLE_IDENTIFIER.namespace()); + assertThat(tableIdents2).as("should be 2 tables in namespace .").hasSize(2); + + TableIdentifier identifierWithHiveTableName = TableIdentifier.of(DB_NAME, hiveTableName); + + // create an iceberg table with the same name + assertThatThrownBy( + () -> + catalog.createTable( + identifierWithHiveTableName, schema, PartitionSpec.unpartitioned())) + .isInstanceOf(NoSuchIcebergTableException.class) + .hasMessageStartingWith("Not an iceberg table: hive.hivedb.test_hive_table"); + + assertThat(catalog.tableExists(identifierWithHiveTableName)).isFalse(); + + assertThat(catalog.tableExists(TABLE_IDENTIFIER)).isTrue(); + HIVE_METASTORE_EXTENSION.metastoreClient().dropTable(DB_NAME, hiveTableName); + } + + @Test + public void testHiveViewAndIcebergTableWithSameName() throws TException, IOException { + List tableIdents = catalog.listTables(TABLE_IDENTIFIER.namespace()); + assertThat(tableIdents).as("should be 1 table in namespace .").hasSize(1); + + // create a hive table + String hiveTableName = "test_hive_table"; + org.apache.hadoop.hive.metastore.api.Table hiveTable = + createHiveTable(hiveTableName, TableType.VIRTUAL_VIEW); + HIVE_METASTORE_EXTENSION.metastoreClient().createTable(hiveTable); + + catalog.setListAllTables(true); + List tableIdents2 = catalog.listTables(TABLE_IDENTIFIER.namespace()); + assertThat(tableIdents2).as("should be 2 tables in namespace .").hasSize(2); + + TableIdentifier identifierWithHiveTableName = TableIdentifier.of(DB_NAME, hiveTableName); + + // create an iceberg table with the same name + assertThatThrownBy( + () -> + catalog.createTable( + identifierWithHiveTableName, schema, PartitionSpec.unpartitioned())) + .isInstanceOf(NoSuchIcebergTableException.class) + .hasMessageStartingWith("Not an iceberg table: hive.hivedb.test_hive_table"); + + assertThat(catalog.tableExists(identifierWithHiveTableName)).isFalse(); + + assertThat(catalog.tableExists(TABLE_IDENTIFIER)).isTrue(); + HIVE_METASTORE_EXTENSION.metastoreClient().dropTable(DB_NAME, hiveTableName); + } + + private org.apache.hadoop.hive.metastore.api.Table createHiveTable( + String hiveTableName, TableType type) throws IOException { Map parameters = Maps.newHashMap(); parameters.put( serdeConstants.SERIALIZATION_CLASS, "org.apache.hadoop.hive.serde2.thrift.test.IntString"); @@ -387,7 +451,7 @@ private org.apache.hadoop.hive.metastore.api.Table createHiveTable(String hiveTa Maps.newHashMap(), "viewOriginalText", "viewExpandedText", - TableType.EXTERNAL_TABLE.name()); + type.name()); return hiveTable; } From 7e310d192d671203e0004bdda5e27142b6fefdc3 Mon Sep 17 00:00:00 2001 From: nk1506 Date: Sat, 23 Mar 2024 02:22:10 +0530 Subject: [PATCH 2/2] Addressed comments: Used parameters for testHiveTableAndIcebergTableWithSameName. --- .../apache/iceberg/hive/HiveTableTest.java | 71 ++++++------------- 1 file changed, 23 insertions(+), 48 deletions(-) diff --git a/hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableTest.java b/hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableTest.java index 082ca34d0246..6d8e9b4391c3 100644 --- a/hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableTest.java +++ b/hive-metastore/src/test/java/org/apache/iceberg/hive/HiveTableTest.java @@ -80,6 +80,8 @@ import org.apache.thrift.TException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; public class HiveTableTest extends HiveTableBaseTest { static final String NON_DEFAULT_DATABASE = "nondefault"; @@ -351,63 +353,36 @@ public void testListTables() throws TException, IOException { HIVE_METASTORE_EXTENSION.metastoreClient().dropTable(DB_NAME, hiveTableName); } - @Test - public void testHiveTableAndIcebergTableWithSameName() throws TException, IOException { - List tableIdents = catalog.listTables(TABLE_IDENTIFIER.namespace()); - assertThat(tableIdents).as("should be 1 table in namespace .").hasSize(1); - - // create a hive table - String hiveTableName = "test_hive_table"; - org.apache.hadoop.hive.metastore.api.Table hiveTable = - createHiveTable(hiveTableName, TableType.EXTERNAL_TABLE); - HIVE_METASTORE_EXTENSION.metastoreClient().createTable(hiveTable); + @ParameterizedTest + @EnumSource( + value = TableType.class, + names = {"EXTERNAL_TABLE", "VIRTUAL_VIEW", "MANAGED_TABLE"}) + public void testHiveTableAndIcebergTableWithSameName(TableType tableType) + throws TException, IOException { - catalog.setListAllTables(true); - List tableIdents2 = catalog.listTables(TABLE_IDENTIFIER.namespace()); - assertThat(tableIdents2).as("should be 2 tables in namespace .").hasSize(2); - - TableIdentifier identifierWithHiveTableName = TableIdentifier.of(DB_NAME, hiveTableName); - - // create an iceberg table with the same name - assertThatThrownBy( - () -> - catalog.createTable( - identifierWithHiveTableName, schema, PartitionSpec.unpartitioned())) - .isInstanceOf(NoSuchIcebergTableException.class) - .hasMessageStartingWith("Not an iceberg table: hive.hivedb.test_hive_table"); + assertThat(catalog.listTables(TABLE_IDENTIFIER.namespace())) + .hasSize(1) + .containsExactly(TABLE_IDENTIFIER); - assertThat(catalog.tableExists(identifierWithHiveTableName)).isFalse(); - - assertThat(catalog.tableExists(TABLE_IDENTIFIER)).isTrue(); - HIVE_METASTORE_EXTENSION.metastoreClient().dropTable(DB_NAME, hiveTableName); - } - - @Test - public void testHiveViewAndIcebergTableWithSameName() throws TException, IOException { - List tableIdents = catalog.listTables(TABLE_IDENTIFIER.namespace()); - assertThat(tableIdents).as("should be 1 table in namespace .").hasSize(1); - - // create a hive table + // create a hive table with a defined table type. String hiveTableName = "test_hive_table"; - org.apache.hadoop.hive.metastore.api.Table hiveTable = - createHiveTable(hiveTableName, TableType.VIRTUAL_VIEW); - HIVE_METASTORE_EXTENSION.metastoreClient().createTable(hiveTable); + TableIdentifier identifier = TableIdentifier.of(DB_NAME, hiveTableName); + HIVE_METASTORE_EXTENSION + .metastoreClient() + .createTable(createHiveTable(hiveTableName, tableType)); catalog.setListAllTables(true); - List tableIdents2 = catalog.listTables(TABLE_IDENTIFIER.namespace()); - assertThat(tableIdents2).as("should be 2 tables in namespace .").hasSize(2); - - TableIdentifier identifierWithHiveTableName = TableIdentifier.of(DB_NAME, hiveTableName); + assertThat(catalog.listTables(TABLE_IDENTIFIER.namespace())) + .hasSize(2) + .containsExactly(TABLE_IDENTIFIER, identifier); + catalog.setListAllTables(false); // reset to default. // create an iceberg table with the same name - assertThatThrownBy( - () -> - catalog.createTable( - identifierWithHiveTableName, schema, PartitionSpec.unpartitioned())) + assertThatThrownBy(() -> catalog.createTable(identifier, schema, PartitionSpec.unpartitioned())) .isInstanceOf(NoSuchIcebergTableException.class) - .hasMessageStartingWith("Not an iceberg table: hive.hivedb.test_hive_table"); + .hasMessageStartingWith(String.format("Not an iceberg table: hive.%s", identifier)); - assertThat(catalog.tableExists(identifierWithHiveTableName)).isFalse(); + assertThat(catalog.tableExists(identifier)).isFalse(); assertThat(catalog.tableExists(TABLE_IDENTIFIER)).isTrue(); HIVE_METASTORE_EXTENSION.metastoreClient().dropTable(DB_NAME, hiveTableName);