From fde302e9e0ba5dbba0587f69b5194e98d750ea8f Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Wed, 17 Mar 2021 17:07:00 -0400 Subject: [PATCH 1/4] fix: hbase 2x shell fixes #2866 bigtable hbase shell has been broken since hbase 2.2.0. It broke because hbase added a feature that the shell referenced. The feature is not actively used by the shell, but it is referenced. This PR defers the unsupported operation errors so that the shell can load --- .../bigtable/hbase2_x/BigtableConnection.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java index 994ca2ef1b..8de9e37f33 100644 --- a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java +++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java @@ -18,6 +18,9 @@ import com.google.api.core.InternalApi; import com.google.cloud.bigtable.hbase.adapters.SampledRowKeysAdapter; import java.io.IOException; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; @@ -28,6 +31,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.AbstractBigtableConnection; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Hbck; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.Table; @@ -140,4 +144,56 @@ public List getAllRegionInfos(TableName tableName) throws IOExcepti } return regionInfos; } + + @Override + public Hbck getHbck() throws IOException { + return createUnsupportedProxy(Hbck.class); + } + + @Override + public Hbck getHbck(ServerName masterServer) throws IOException { + return createUnsupportedProxy(Hbck.class); + } + + private T createUnsupportedProxy(Class cls) { + + @SuppressWarnings("unchecked") + T proxy = + (T) + Proxy.newProxyInstance( + cls.getClassLoader(), + new Class[] {cls}, + new UnsupportedInvocationHandler("Unsupported" + cls.getName())); + + return proxy; + } + + private static class UnsupportedInvocationHandler implements InvocationHandler { + private final String name; + + private UnsupportedInvocationHandler(String name) { + this.name = name; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + if (args.length == 1 + && method.getName().equals("equals") + && method.getParameterTypes()[0] == Object.class) { + Object arg = args[0]; + if (arg == null) { + return false; + } + return proxy == arg; + } + if (args.length == 1 && method.getName().equals("hashCode")) { + return super.hashCode(); + } + if (args.length == 0 && method.getName().equals("toString")) { + return name; + } + + throw new UnsupportedOperationException(name + "." + method.getName()); + } + } } From bb7987b91b9f44894998a6ad4df9085de3a863ff Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 18 Mar 2021 11:40:10 -0400 Subject: [PATCH 2/4] doc --- .../cloud/bigtable/hbase2_x/BigtableConnection.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java index 8de9e37f33..1eb96c1f66 100644 --- a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java +++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java @@ -155,8 +155,15 @@ public Hbck getHbck(ServerName masterServer) throws IOException { return createUnsupportedProxy(Hbck.class); } + /** + * Create a dynamic proxy for the passed in interface. + * + *

The proxy will implement all methods by throwing an {@link UnsupportedOperationException}. + * This is useful for deferring the exception being thrown. For example hbase shell will acquire a + * reference toHbck, but wont actually use it. So it would be useful to allow hbase shell to + * initialize and defer the UnsupportOperationExcetion on Hbck is actually invoked. + */ private T createUnsupportedProxy(Class cls) { - @SuppressWarnings("unchecked") T proxy = (T) From bbbe2b673af124409c6703d8fed0080319c58873 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 18 Mar 2021 13:26:35 -0400 Subject: [PATCH 3/4] add unit tests and fix a bug --- .../bigtable/hbase2_x/BigtableConnection.java | 8 ++-- .../hbase/TestBigtableConnection.java | 44 +++++++++++++++++-- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java index 1eb96c1f66..b22bdccacb 100644 --- a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java +++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java @@ -184,7 +184,9 @@ private UnsupportedInvocationHandler(String name) { @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - if (args.length == 1 + int argCount = (args == null) ? 0 : args.length; + + if (argCount == 1 && method.getName().equals("equals") && method.getParameterTypes()[0] == Object.class) { Object arg = args[0]; @@ -193,10 +195,10 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl } return proxy == arg; } - if (args.length == 1 && method.getName().equals("hashCode")) { + if (argCount == 0 && method.getName().equals("hashCode")) { return super.hashCode(); } - if (args.length == 0 && method.getName().equals("toString")) { + if (argCount == 0 && method.getName().equals("toString")) { return name; } diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java index a417fd306e..ec44ea46f8 100644 --- a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java +++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.BufferedMutator; +import org.apache.hadoop.hbase.client.Hbck; import org.apache.hadoop.hbase.client.Table; import org.junit.Assert; import org.junit.Test; @@ -65,9 +66,44 @@ public void testTable() throws IOException { Configuration conf = BigtableConfiguration.configure("projectId", "instanceId", "appProfileId"); conf.set(BigtableOptionsFactory.BIGTABLE_NULL_CREDENTIAL_ENABLE_KEY, "true"); conf.set(BigtableOptionsFactory.BIGTABLE_USE_SERVICE_ACCOUNTS_KEY, "false"); - BigtableConnection connection = new BigtableConnection(conf); - Admin admin = connection.getAdmin(); - Table table = connection.getTable(TableName.valueOf("someTable")); - BufferedMutator mutator = connection.getBufferedMutator(TableName.valueOf("someTable")); + try (BigtableConnection connection = new BigtableConnection(conf)) { + Admin admin = connection.getAdmin(); + Table table = connection.getTable(TableName.valueOf("someTable")); + BufferedMutator mutator = connection.getBufferedMutator(TableName.valueOf("someTable")); + } + } + + @Test + public void testHbckErrorDeferred() throws IOException { + Configuration conf = BigtableConfiguration.configure("projectId", "instanceId", "appProfileId"); + conf.setInt(BigtableOptionsFactory.BIGTABLE_DATA_CHANNEL_COUNT_KEY, 1); + conf.set(BigtableOptionsFactory.BIGTABLE_NULL_CREDENTIAL_ENABLE_KEY, "true"); + conf.set(BigtableOptionsFactory.BIGTABLE_USE_SERVICE_ACCOUNTS_KEY, "false"); + + try (BigtableConnection c1 = new BigtableConnection(conf); + BigtableConnection c2 = new BigtableConnection(conf)) { + + // Should not throw + Hbck hbck1 = c1.getHbck(); + Hbck hbck2 = c2.getHbck(); + + Assert.assertThrows( + UnsupportedOperationException.class, + () -> { + try { + hbck1.runHbckChore(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + Assert.assertEquals(hbck1, hbck1); + Assert.assertNotEquals(hbck1, hbck2); + + Assert.assertEquals(hbck1.hashCode(), hbck1.hashCode()); + Assert.assertNotEquals(hbck1.hashCode(), hbck2.hashCode()); + + Assert.assertTrue(hbck1.toString().contains("Unsupported")); + } } } From 6eaa54ea4209002c8ac1b0fab373e762c93f0c20 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Thu, 18 Mar 2021 14:40:35 -0400 Subject: [PATCH 4/4] tweaks --- .../google/cloud/bigtable/hbase2_x/BigtableConnection.java | 2 +- .../google/cloud/bigtable/hbase/TestBigtableConnection.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java index b22bdccacb..b9171cf7b5 100644 --- a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java +++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableConnection.java @@ -170,7 +170,7 @@ private T createUnsupportedProxy(Class cls) { Proxy.newProxyInstance( cls.getClassLoader(), new Class[] {cls}, - new UnsupportedInvocationHandler("Unsupported" + cls.getName())); + new UnsupportedInvocationHandler("Unsupported" + cls.getSimpleName())); return proxy; } diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java index ec44ea46f8..a4c8145761 100644 --- a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java +++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase/TestBigtableConnection.java @@ -100,10 +100,12 @@ public void testHbckErrorDeferred() throws IOException { Assert.assertEquals(hbck1, hbck1); Assert.assertNotEquals(hbck1, hbck2); + // Make sure that the hashCode is stable Assert.assertEquals(hbck1.hashCode(), hbck1.hashCode()); + // And differs for different instances Assert.assertNotEquals(hbck1.hashCode(), hbck2.hashCode()); - Assert.assertTrue(hbck1.toString().contains("Unsupported")); + Assert.assertEquals("UnsupportedHbck", hbck1.toString()); } } }