diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java index e8a9df70d3..cbd2456e1b 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java @@ -86,25 +86,25 @@ public Map> reportTablesMissingRegions(final List n return missingChecker.reportTablesRegions(namespacesOrTables, this::findMissingRegionsInMETA); } - public Map> + public Map> reportTablesExtraRegions(final List namespacesOrTables) throws IOException { - InternalMetaChecker extraChecker = new InternalMetaChecker<>(); + InternalMetaChecker extraChecker = new InternalMetaChecker<>(); return extraChecker.reportTablesRegions(namespacesOrTables, this::findExtraRegionsInMETA); } List findMissingRegionsInMETA(String table) throws IOException { InternalMetaChecker missingChecker = new InternalMetaChecker<>(); return missingChecker.checkRegionsInMETA(table, (regions, dirs) -> { - ListUtils utils = new ListUtils<>(); - return utils.complement(dirs, regions, d -> d.getName(), r -> r.getEncodedName()); + ListUtils utils = new ListUtils<>(); + return utils.complement(dirs, regions, d -> d.getName(), r -> r.getEncodedRegionName()); }); } - List findExtraRegionsInMETA(String table) throws IOException { - InternalMetaChecker extraChecker = new InternalMetaChecker<>(); + List findExtraRegionsInMETA(String table) throws IOException { + InternalMetaChecker extraChecker = new InternalMetaChecker<>(); return extraChecker.checkRegionsInMETA(table, (regions,dirs) -> { - ListUtils utils = new ListUtils<>(); - return utils.complement(regions, dirs, r -> r.getEncodedName(), d -> d.getName()); + ListUtils utils = new ListUtils<>(); + return utils.complement(regions, dirs, r -> r.getEncodedRegionName(), d -> d.getName()); }); } @@ -132,18 +132,18 @@ public List>> addMissingRegionsInMetaForTables( public List>> removeExtraRegionsFromMetaForTables( List nameSpaceOrTable) throws IOException { if(nameSpaceOrTable.size()>0) { - InternalMetaChecker extraChecker = new InternalMetaChecker<>(); + InternalMetaChecker extraChecker = new InternalMetaChecker<>(); return extraChecker.processRegionsMetaCleanup(this::reportTablesExtraRegions, this::deleteAllRegions, nameSpaceOrTable); } return null; } - private List deleteAllRegions(List regions) throws IOException { + private List deleteAllRegions(List regions) throws IOException { List resulting = new ArrayList<>(); - for(RegionInfo r : regions){ - HBCKMetaTableAccessor.deleteRegionInfo(conn, r); - resulting.add(r.getEncodedName()); + for(HBCKMetaEntry r : regions){ + HBCKMetaTableAccessor.deleteRegion(conn, r); + resulting.add(r.getEncodedRegionName()); } return resulting; } @@ -156,11 +156,11 @@ public void close() throws IOException { private class InternalMetaChecker { List checkRegionsInMETA(String table, - CheckingFunction, List, T> checkingFunction) throws IOException { + CheckingFunction, List, T> checkingFunction) throws IOException { final List regionsDirs = getTableRegionsDirs(table); TableName tableName = TableName.valueOf(table); - List regions = HBCKMetaTableAccessor. - getTableRegions(FsRegionsMetaRecoverer.this.conn, tableName); + List regions = HBCKMetaTableAccessor. + getTableRegionsAsMetaEntries(FsRegionsMetaRecoverer.this.conn, tableName); return checkingFunction.check(regions, regionsDirs); } diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index a4e180bb9d..ca1597d1eb 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -344,12 +344,12 @@ Map> extraRegionsInMeta(String[] args) new FsRegionsMetaRecoverer(this.conf)) { List namespacesTables = getFromArgsOrFiles(commandLine.getArgList(), inputFileFlag); - Map> reportMap = + Map> reportMap = fsRegionsMetaRecoverer.reportTablesExtraRegions(namespacesTables); final List toFix = new ArrayList<>(); reportMap.entrySet().forEach(e -> { result.put(e.getKey(), - e.getValue().stream().map(r->r.getEncodedName()).collect(Collectors.toList())); + e.getValue().stream().map(r->r.getEncodedRegionName()).collect(Collectors.toList())); if(fix && e.getValue().size()>0){ toFix.add(e.getKey().getNameWithNamespaceInclAsString()); } diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKActions.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKActions.java index a1126b6272..01dc209618 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKActions.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKActions.java @@ -87,7 +87,7 @@ private void deleteRegionFromMeta(String tname) throws IOException, InterruptedE System.out.println(String.format("Current Regions of the table " + tn.getNameAsString() + " in Meta before deletion of the region are: " + ris)); RegionInfo ri = ris.get(ris.size() / 2); - System.out.println("Deleting Region " + ri.getRegionNameAsString()); + System.out.println("Deleting Region " + ri); byte[] key = HBCKMetaTableAccessor.getMetaKeyForRegion(ri); Delete delete = new Delete(key); diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaEntry.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaEntry.java new file mode 100644 index 0000000000..2c6013982b --- /dev/null +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaEntry.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hbase; + +/** + * A conveninent representation of a row in meta, composing the encoded name and the related + * rowkey (region name). + */ +public class HBCKMetaEntry { + + private String encodedRegionName; + private byte[] regionName; + + public HBCKMetaEntry(byte[] regionName, String encodedRegionName) { + this.regionName = regionName; + this.encodedRegionName = encodedRegionName; + } + + public byte[] getRegionName() { + return regionName; + } + + public String getEncodedRegionName() { + return encodedRegionName; + } +} diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java index 8d50bb5ada..ed7e507cf8 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCKMetaTableAccessor.java @@ -128,13 +128,44 @@ public static List getMergeRegions(Cell[] cells) { * @throws IOException if it's not able to delete the regionInfo */ public static void deleteRegionInfo(Connection connection, RegionInfo regionInfo) - throws IOException { + throws IOException { Delete delete = new Delete(regionInfo.getRegionName()); delete.addFamily(HConstants.CATALOG_FAMILY, HConstants.LATEST_TIMESTAMP); deleteFromMetaTable(connection, delete); LOG.info("Deleted {}", regionInfo.getRegionNameAsString()); } + /** + * Delete the passed RegionInfo from the hbase:meta table. + * + * @param connection connection we're using + * @param regionInfo the regionInfo to delete from the meta table + * @throws IOException if it's not able to delete the regionInfo + */ + public static void deleteRegionInfoColumn(Connection connection, RegionInfo regionInfo) + throws IOException { + Delete delete = new Delete(regionInfo.getRegionName()); + delete.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, + HConstants.LATEST_TIMESTAMP); + deleteFromMetaTable(connection, delete); + LOG.info("Deleted regioninfo for {}", regionInfo.getRegionNameAsString()); + } + + /** + * Delete the passed HBCKMetaEntry from the hbase:meta table. + * + * @param connection connection we're using + * @param region the region to be deleted from the meta table + * @throws IOException if it's not able to delete the regionInfo + */ + public static void deleteRegion(Connection connection, HBCKMetaEntry region) + throws IOException { + Delete delete = new Delete(region.getRegionName()); + delete.addFamily(HConstants.CATALOG_FAMILY, HConstants.LATEST_TIMESTAMP); + deleteFromMetaTable(connection, delete); + LOG.info("Deleted {}", region.getEncodedRegionName()); + } + // Private helper methods // COPIED from MetaTableAccessor.isMergeQualifierPrefix() @@ -187,11 +218,49 @@ private static void deleteFromMetaTable(final Connection connection, final Delet * Returns all regions in meta for the given table. * @param conn a valid, open connection. * @param table the table to list regions in meta. - * @return a list of RegionInfo for all table regions present in meta. + * @return a list of HBCKMetaEntry with encoded region names, and the meta row key + * for all table regions present in meta. + * @throws IOException on any issues related with scanning meta table + * */ + public static List getTableRegionsAsMetaEntries(final Connection conn, + final TableName table) throws IOException { + final MetaScanner scanner = new MetaScanner<>(); + final String startRow = Bytes.toString(table.getName()) + ",,"; + final String stopRow = Bytes.toString(table.getName()) + " ,,"; + return scanner.scanMeta(conn, + scan -> { + scan.withStartRow(Bytes.toBytes(startRow)); + scan.withStopRow(Bytes.toBytes(stopRow)); + }, + r -> { + if (r.getRow() != null) { + boolean encodedNameOffset = false; + StringBuilder encodedNameBuilder = new StringBuilder(); + for(int i=0; iString of encoded region names, + * for all table regions present in meta. * @throws IOException on any issues related with scanning meta table */ public static List getTableRegions(final Connection conn, final TableName table) - throws IOException { + throws IOException { final MetaScanner scanner = new MetaScanner<>(); final String startRow = Bytes.toString(table.getName()) + ",,"; final String stopRow = Bytes.toString(table.getName()) + " ,,"; diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java index 20939966b4..68101a61d5 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java @@ -210,10 +210,10 @@ private RegionInfo createRegionInMeta(ResultScanner mockedRS) throws Exception { @Test public void testFindExtraRegionsInMETAOneExtra() throws Exception { RegionInfo info = createRegionInMeta(Mockito.mock(ResultScanner.class)); - List missingRegions = fixer.findExtraRegionsInMETA("test-tbl"); + List missingRegions = fixer.findExtraRegionsInMETA("test-tbl"); assertEquals("Should had returned 1 extra region", 1, missingRegions.size()); - assertEquals(info.getEncodedName(),missingRegions.get(0).getEncodedName()); + assertEquals(info.getEncodedName(),missingRegions.get(0).getEncodedRegionName()); } @Test @@ -243,7 +243,7 @@ public void testReportTablesExtraRegionsOneExtra() throws Exception { cells.add(createCellForTableState(TableName.valueOf("test-tbl"))); Result result = Result.create(cells); Mockito.when(mockedRS.next()).thenReturn(result,(Result)null); - Map> report = fixer.reportTablesExtraRegions(null); + Map> report = fixer.reportTablesExtraRegions(null); assertEquals("Should had returned 1 extra region.", 1,report.size()); } diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java index 09f9559374..e795650bec 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java @@ -696,6 +696,20 @@ public void testFunctionNotSupported() throws IOException { } } + @Test + public void testRemoveExtraRegionWithoutInfo() throws Exception { + TableName tableName = createTestTable(5); + HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration()); + List regions = HBCKMetaTableAccessor + .getTableRegions(TEST_UTIL.getConnection(), tableName); + deleteRegionDir(tableName, regions.get(0).getEncodedName()); + HBCKMetaTableAccessor.deleteRegionInfoColumn(TEST_UTIL.getConnection(), regions.get(0)); + assertEquals(1, hbck.extraRegionsInMeta(new String[]{"-f", + "default:" + tableName.getNameAsString()}).get(tableName).size()); + assertEquals("Table regions should had been removed from META.", 4, + HBCKMetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName).size()); + } + private String testFormatExtraRegionsInMetaReport() throws IOException { return testRunWithArgs(new String[]{EXTRA_REGIONS_IN_META}); } @@ -735,7 +749,7 @@ private void testRemoveExtraRegionsInMetaForTables(int extraRegions, int totalRe assertEquals(extraRegions, hbck.extraRegionsInMeta(new String[]{"-f", "default:" + tableName.getNameAsString()}).get(tableName).size()); assertEquals("Table regions should had been removed from META.", remaining, - HBCKMetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName)); + HBCKMetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tableName).size()); } private void testReportExtraRegionsInMeta(int extraRegionsInTestTbl,