From 752131bdb1e1d4f8e2c416e42cf7261dfc847da3 Mon Sep 17 00:00:00 2001 From: Dmitry <58846611+dmitry-fa@users.noreply.github.com> Date: Tue, 29 Dec 2020 12:52:51 -0500 Subject: [PATCH 1/2] fix: bigtable-hbase-2.x-hadoop incompatible with hbase-shaded-client 2.x (#2773) (cherry picked from commit 1792c6080ad4c2c59b32e830353e438d83a8d01f) --- .../hbase/client/AbstractBigtableAdmin.java | 15 ---- .../bigtable/hbase1_x/BigtableAdmin.java | 72 +++++++------------ .../bigtable/hbase1_x/BigtableAdminTest.java | 9 +++ .../bigtable/hbase2_x/BigtableAdmin.java | 17 +++++ .../bigtable/hbase2_x/BigtableAdminTest.java | 55 ++++++++++++++ 5 files changed, 105 insertions(+), 63 deletions(-) create mode 100644 bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase2_x/BigtableAdminTest.java diff --git a/bigtable-client-core-parent/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/AbstractBigtableAdmin.java b/bigtable-client-core-parent/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/AbstractBigtableAdmin.java index 024235351f..43fb65fdf1 100644 --- a/bigtable-client-core-parent/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/AbstractBigtableAdmin.java +++ b/bigtable-client-core-parent/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/AbstractBigtableAdmin.java @@ -47,8 +47,6 @@ import io.grpc.Status; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -56,7 +54,6 @@ import java.util.regex.Pattern; import javax.annotation.Nullable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; @@ -698,18 +695,6 @@ public void deleteColumn(final String tableName, final String columnName) throws deleteColumn(TableName.valueOf(tableName), Bytes.toBytes(columnName)); } - /** {@inheritDoc} */ - @Override - public ClusterStatus getClusterStatus() throws IOException { - return new ClusterStatus() { - @Override - public Collection getServers() { - // TODO(sduskis): Point the server name to options.getServerName() - return Collections.emptyList(); - } - }; - } - /** {@inheritDoc} */ @Override public Configuration getConfiguration() { diff --git a/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableAdmin.java b/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableAdmin.java index 953cbb25c4..efbb9b2700 100644 --- a/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableAdmin.java +++ b/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableAdmin.java @@ -16,14 +16,13 @@ package com.google.cloud.bigtable.hbase1_x; import com.google.api.core.InternalApi; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.Futures; import java.io.IOException; -import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; +import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableExistsException; @@ -61,13 +60,7 @@ public BigtableAdmin(CommonConnection connection) throws IOException { @Override public MasterProtos.SnapshotResponse takeSnapshotAsync(HBaseProtos.SnapshotDescription snapshot) throws IOException, SnapshotCreationException { - snapshotTable(snapshot.getName(), TableName.valueOf(snapshot.getTable())); - LOG.warn( - "isSnapshotFinished() is not currently supported by BigtableAdmin.\n" - + "You may poll for existence of the snapshot with listSnapshots(snapshotName)"); - return MasterProtos.SnapshotResponse.newBuilder() - .setExpectedTimeout(TimeUnit.MINUTES.toMillis(5)) - .build(); + throw new UnsupportedOperationException("takeSnapshotAsync"); } /** {@inheritDoc} */ @@ -105,16 +98,16 @@ public AdminProtos.GetRegionInfoResponse.CompactionState getCompactionStateForRe */ @Override public void snapshot( - String snapshotId, TableName tableName, HBaseProtos.SnapshotDescription.Type type) + String snapshotName, TableName tableName, HBaseProtos.SnapshotDescription.Type type) throws IOException, SnapshotCreationException, IllegalArgumentException { - snapshot(snapshotId, tableName); + throw new UnsupportedOperationException("snapshot"); } /** {@inheritDoc} */ @Override public void snapshot(HBaseProtos.SnapshotDescription snapshot) throws IOException, SnapshotCreationException, IllegalArgumentException { - snapshot(snapshot.getName(), TableName.valueOf(snapshot.getTable())); + throw new UnsupportedOperationException("snapshot"); } /** {@inheritDoc} */ @@ -126,67 +119,40 @@ public boolean isSnapshotFinished(HBaseProtos.SnapshotDescription snapshot) @Override public void restoreSnapshot(String s, boolean b, boolean b1) - throws IOException, RestoreSnapshotException { - throw new UnsupportedOperationException("restoreSnapshot"); // TODO - } + throws IOException, RestoreSnapshotException {} @Override public void cloneSnapshot(String s, TableName tableName, boolean b) - throws IOException, TableExistsException, RestoreSnapshotException { - if (!b) { - this.cloneSnapshot(s, tableName); - } - throw new UnsupportedOperationException("cloneSnapshot"); // TODO - } + throws IOException, TableExistsException, RestoreSnapshotException {} /** {@inheritDoc} */ @Override public List listSnapshots() throws IOException { - List backups = - Futures.getChecked( - tableAdminClientWrapper.listBackupsAsync(getBackupClusterName().getClusterId()), - IOException.class); - - List response = new ArrayList<>(); - - for (String snapshot : backups) { - response.add(HBaseProtos.SnapshotDescription.newBuilder().setName(snapshot).build()); - } - return response; + throw new UnsupportedOperationException("listSnapshots"); } /** {@inheritDoc} */ @Override public List listSnapshots(String regex) throws IOException { - return listSnapshots(Pattern.compile(regex)); + throw new UnsupportedOperationException("listSnapshots"); } /** {@inheritDoc} */ @Override public List listSnapshots(Pattern pattern) throws IOException { - if (pattern == null || pattern.matcher("").matches()) { - return ImmutableList.of(); - } - - List response = new ArrayList<>(); - for (HBaseProtos.SnapshotDescription description : listSnapshots()) { - if (pattern.matcher(description.getName()).matches()) { - response.add(description); - } - } - return response; + throw new UnsupportedOperationException("listSnapshots"); } @Override public List listTableSnapshots( String tableNameRegex, String snapshotNameRegex) throws IOException { - throw new UnsupportedOperationException("Unsupported - please use listSnapshots"); + throw new UnsupportedOperationException("listTableSnapshots"); } @Override public List listTableSnapshots( Pattern tableNamePattern, Pattern snapshotNamePattern) throws IOException { - throw new UnsupportedOperationException("Unsupported - please use listSnapshots"); + throw new UnsupportedOperationException("listTableSnapshots"); } @Override @@ -295,4 +261,14 @@ public boolean[] setSplitOrMergeEnabled(boolean arg0, boolean arg1, MasterSwitch throws IOException { throw new UnsupportedOperationException("setSplitOrMergeEnabled"); // TODO } + + @Override + public ClusterStatus getClusterStatus() throws IOException { + return new ClusterStatus() { + @Override + public Collection getServers() { + return Collections.emptyList(); + } + }; + } } diff --git a/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/test/java/com/google/cloud/bigtable/hbase1_x/BigtableAdminTest.java b/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/test/java/com/google/cloud/bigtable/hbase1_x/BigtableAdminTest.java index 46b4bac629..fbced4e383 100644 --- a/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/test/java/com/google/cloud/bigtable/hbase1_x/BigtableAdminTest.java +++ b/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/test/java/com/google/cloud/bigtable/hbase1_x/BigtableAdminTest.java @@ -16,6 +16,7 @@ package com.google.cloud.bigtable.hbase1_x; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import com.google.bigtable.admin.v2.BigtableTableAdminGrpc; import com.google.bigtable.admin.v2.DropRowRangeRequest; @@ -28,6 +29,7 @@ import java.net.ServerSocket; import java.util.concurrent.ArrayBlockingQueue; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.TableName; import org.junit.After; import org.junit.Before; @@ -122,4 +124,11 @@ public void onMessage(ReqT message) { }; } } + + @Test + public void testGetClusterStatus() throws IOException { + // test to verify compatibility between 1x and 2x + ClusterStatus status = admin.getClusterStatus(); + assertNotNull(status); + } } diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableAdmin.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableAdmin.java index c0c2b67367..d1124d43bb 100644 --- a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableAdmin.java +++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableAdmin.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -35,6 +36,7 @@ import org.apache.hadoop.hbase.CacheEvictionStats; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; +import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor; @@ -863,4 +865,19 @@ public Future updateReplicationPeerConfigAsync( String s, ReplicationPeerConfig replicationPeerConfig) { throw new UnsupportedOperationException("updateReplicationPeerConfigAsync"); // TODO } + + @Override + public ClusterStatus getClusterStatus() throws IOException { + return new ClusterStatus( + "hbaseVersion", + "clusterid", + new HashMap(), + new ArrayList(), + null, + new ArrayList(), + new ArrayList(), + new String[0], + false, + -1); + } } diff --git a/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase2_x/BigtableAdminTest.java b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase2_x/BigtableAdminTest.java new file mode 100644 index 0000000000..5f5ed6b28d --- /dev/null +++ b/bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/test/java/com/google/cloud/bigtable/hbase2_x/BigtableAdminTest.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.hbase2_x; + +import static junit.framework.TestCase.assertEquals; +import static junit.framework.TestCase.assertFalse; +import static junit.framework.TestCase.assertNotNull; +import static junit.framework.TestCase.assertNull; +import static junit.framework.TestCase.assertTrue; +import static org.mockito.Mockito.mock; + +import com.google.cloud.bigtable.hbase.wrappers.BigtableApi; +import java.io.IOException; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.client.AbstractBigtableConnection; +import org.junit.Test; +import org.mockito.Mockito; + +public class BigtableAdminTest { + + @Test + public void testGetClusterStatus() throws IOException { + // test to verify compatibility between 1x and 2x + AbstractBigtableConnection connectionMock = mock(AbstractBigtableConnection.class); + BigtableApi bigtableApi = mock(BigtableApi.class); + Mockito.doReturn(bigtableApi).when(connectionMock).getBigtableApi(); + BigtableAdmin bigtableAdmin = new BigtableAdmin(connectionMock); + + ClusterStatus status = bigtableAdmin.getClusterStatus(); + assertNotNull(status); + assertEquals("hbaseVersion", status.getHBaseVersion()); + assertEquals("clusterid", status.getClusterId()); + assertTrue(status.getServersName().isEmpty()); + assertTrue(status.getDeadServerNames().isEmpty()); + assertNull(status.getMaster()); + assertTrue(status.getBackupMasterNames().isEmpty()); + assertTrue(status.getRegionStatesInTransition().isEmpty()); + assertEquals(0, status.getMasterCoprocessors().length); + assertFalse(status.getBalancerOn()); + assertEquals(-1, status.getMasterInfoPort()); + } +} From 11411439ba1773825ba25884f32c7604ba80132e Mon Sep 17 00:00:00 2001 From: Kristen O'Leary Date: Wed, 3 Mar 2021 12:02:00 -0500 Subject: [PATCH 2/2] sync with bigtable-1.x --- .../bigtable/hbase1_x/BigtableAdmin.java | 59 +++++++++++++++---- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableAdmin.java b/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableAdmin.java index efbb9b2700..042d306998 100644 --- a/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableAdmin.java +++ b/bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableAdmin.java @@ -16,11 +16,15 @@ package com.google.cloud.bigtable.hbase1_x; import com.google.api.core.InternalApi; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Futures; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.ProcedureInfo; @@ -60,7 +64,13 @@ public BigtableAdmin(CommonConnection connection) throws IOException { @Override public MasterProtos.SnapshotResponse takeSnapshotAsync(HBaseProtos.SnapshotDescription snapshot) throws IOException, SnapshotCreationException { - throw new UnsupportedOperationException("takeSnapshotAsync"); + snapshotTable(snapshot.getName(), TableName.valueOf(snapshot.getTable())); + LOG.warn( + "isSnapshotFinished() is not currently supported by BigtableAdmin.\n" + + "You may poll for existence of the snapshot with listSnapshots(snapshotName)"); + return MasterProtos.SnapshotResponse.newBuilder() + .setExpectedTimeout(TimeUnit.MINUTES.toMillis(5)) + .build(); } /** {@inheritDoc} */ @@ -98,16 +108,16 @@ public AdminProtos.GetRegionInfoResponse.CompactionState getCompactionStateForRe */ @Override public void snapshot( - String snapshotName, TableName tableName, HBaseProtos.SnapshotDescription.Type type) + String snapshotId, TableName tableName, HBaseProtos.SnapshotDescription.Type type) throws IOException, SnapshotCreationException, IllegalArgumentException { - throw new UnsupportedOperationException("snapshot"); + snapshot(snapshotId, tableName); } /** {@inheritDoc} */ @Override public void snapshot(HBaseProtos.SnapshotDescription snapshot) throws IOException, SnapshotCreationException, IllegalArgumentException { - throw new UnsupportedOperationException("snapshot"); + snapshot(snapshot.getName(), TableName.valueOf(snapshot.getTable())); } /** {@inheritDoc} */ @@ -119,40 +129,67 @@ public boolean isSnapshotFinished(HBaseProtos.SnapshotDescription snapshot) @Override public void restoreSnapshot(String s, boolean b, boolean b1) - throws IOException, RestoreSnapshotException {} + throws IOException, RestoreSnapshotException { + throw new UnsupportedOperationException("restoreSnapshot"); // TODO + } @Override public void cloneSnapshot(String s, TableName tableName, boolean b) - throws IOException, TableExistsException, RestoreSnapshotException {} + throws IOException, TableExistsException, RestoreSnapshotException { + if (!b) { + this.cloneSnapshot(s, tableName); + } + throw new UnsupportedOperationException("cloneSnapshot"); // TODO + } /** {@inheritDoc} */ @Override public List listSnapshots() throws IOException { - throw new UnsupportedOperationException("listSnapshots"); + List backups = + Futures.getChecked( + tableAdminClientWrapper.listBackupsAsync(getBackupClusterName().getClusterId()), + IOException.class); + + List response = new ArrayList<>(); + + for (String snapshot : backups) { + response.add(HBaseProtos.SnapshotDescription.newBuilder().setName(snapshot).build()); + } + return response; } /** {@inheritDoc} */ @Override public List listSnapshots(String regex) throws IOException { - throw new UnsupportedOperationException("listSnapshots"); + return listSnapshots(Pattern.compile(regex)); } /** {@inheritDoc} */ @Override public List listSnapshots(Pattern pattern) throws IOException { - throw new UnsupportedOperationException("listSnapshots"); + if (pattern == null || pattern.matcher("").matches()) { + return ImmutableList.of(); + } + + List response = new ArrayList<>(); + for (HBaseProtos.SnapshotDescription description : listSnapshots()) { + if (pattern.matcher(description.getName()).matches()) { + response.add(description); + } + } + return response; } @Override public List listTableSnapshots( String tableNameRegex, String snapshotNameRegex) throws IOException { - throw new UnsupportedOperationException("listTableSnapshots"); + throw new UnsupportedOperationException("Unsupported - please use listSnapshots"); } @Override public List listTableSnapshots( Pattern tableNamePattern, Pattern snapshotNamePattern) throws IOException { - throw new UnsupportedOperationException("listTableSnapshots"); + throw new UnsupportedOperationException("Unsupported - please use listSnapshots"); } @Override