From 88f96b3df61d86cf0d98a559759d96569efa1cf5 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Fri, 7 Jun 2019 17:35:54 +0100 Subject: [PATCH 01/10] HBASE-22567 - HBCK2 addMissingRegionsToMeta --- hbase-hbck2/README.md | 53 +++- hbase-hbck2/pom.xml | 6 + .../src/main/java/org/apache/hbase/HBCK2.java | 234 ++++++++++++++++++ .../apache/hbase/hbck2/meta/MetaFixer.java | 130 ++++++++++ .../test/java/org/apache/hbase/TestHBCK2.java | 135 ++++++++++ .../hbase/hbck2/meta/TestMetaFixer.java | 208 ++++++++++++++++ 6 files changed, 764 insertions(+), 2 deletions(-) create mode 100644 hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java create mode 100644 hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java diff --git a/hbase-hbck2/README.md b/hbase-hbck2/README.md index 9e6e6f61c4..6617f26e09 100644 --- a/hbase-hbck2/README.md +++ b/hbase-hbck2/README.md @@ -375,9 +375,55 @@ To schedule an assign for the hbase:namespace table noted in the above log line, ```HBASE_CLASSPATH_PREFIX=./hbase-hbck2-1.0.0-SNAPSHOT.jar hbase org.apache.hbase.HBCK2 -skip assigns 9559cf72b8e81e1291c626a8e781a6ae``` ... passing the encoded name for the namespace region (the encoded name will differ per deploy). -### hbase:meta region/table restore/rebuild +### Missing Regions in META - hbase:meta region/table restore/rebuild -Should a cluster suffer a catastrophic loss of the `hbase:meta` region, a rough rebuild is possible following the below recipe. In outline: stop the cluster; run the _OfflineMetaRepair_ tool which reads directories and metadata dropped into the filesystem making a best effort at reconstructing a viable _hbase:meta_ table; restart your cluster; inject an assign to bring the system namespace table online; and then finally, re-assign userspace tables you'd like enabled (the rebuilt _hbase:meta_ creates a table with all tables offline and no regions assigned). +There's been some extra-ordinary cases where table regions are removed from META table. +Some triage on such cases revealed those were operator-induced, after execution +attempts of the obsolete *hbck1* _OfflineMetaRepair_ tool. _OfflineMetaRepair_ is a well known tool +for fixing META table related issues on HBase 1.x versions. The original version is not compatible +with HBase 2.x or higher versions, and it has undergone some adjustments to be now run within hbck2. + +In most of these cases, regions may be missing in meta at random, but hbase may still be +operational. In such situations, problem can be addressed with master online, using _addMissingRegionsInMeta_ command. +This command is less disruptive to hbase than the full meta rebuild covered later, and it can be used even for +recovering _namespace_ table region. + +#### Online meta rebuild recipe + +If meta corruption is not too critical, hbase would still be able to bring it online. Even if namespace region +is among the missing ones in meta, it will still be possible to scan meta in the initialization period, +where master will be waiting for namespace to be assigned. To verify on this, a meta scan command can be executed +as below. If it does not time out or show any errors, _meta_ is online: + +``` +echo "scan 'hbase:meta', {COLUMN=>'info:regioninfo'}" | hbase shell +``` + +HBCK2 _addMissingRegionsInMeta_ can be used if the above does not show any errors. It reads region +metadata info available on the FS region dirs, in order to re-create regions in META. +It can check for specific tables/namespaces, or all tables +from all namespaces. An example adding missing regions for tables 'tbl_1' on default namespace, +'tbl_2' on namespace 'n1' and for all tables from namespace 'n2': + +``` +$ HBCK2 addMissingRegionsInMeta default:tbl_1 n1:tbl_2 n2 +``` + +As it operates independently from Master, once it finishes successfully, additional steps are +required to actually have the re-added regions assigned. These are listed below: + +1. _addMissingRegionsInMeta_ outputs an _assigns_ command with all regions that got re-added. This +command needs to be executed later, so copy and save it for convenience. + +2. For HBase versions prior to 2.3.0, after _addMissingRegionsInMeta_ finished successfully and output has been saved, restart all +running HBase Masters. + +3. Once Master's are restarted and META is already online (check if Web UI is accessible), run +_assigns_ command from _addMissingRegionsInMeta_ output saved per instructions from #1. + + +Should a cluster suffer a catastrophic loss of the `hbase:meta` region, a rough rebuild is possible following the below recipe. +In outline: stop the cluster; run the _OfflineMetaRepair_ tool which reads directories and metadata dropped into the filesystem making a best effort at reconstructing a viable _hbase:meta_ table; restart your cluster; inject an assign to bring the system namespace table online; and then finally, re-assign userspace tables you'd like enabled (the rebuilt _hbase:meta_ creates a table with all tables offline and no regions assigned). #### Detailed rebuild recipe Stop the cluster. @@ -410,3 +456,6 @@ The rebuild meta will likely be missing edits and may need subsequent repair and ### Dropped reference files, missing hbase.version file, and corrupted hfiles HBCK2 can check for hanging references and corrupt hfiles. You can ask it to sideline bad files which may be needed to get over humps where regions won't online or reads are failing. See the _filesystem_ command in the HBCK2 listing. Pass one or more tablename (or 'none' to check all tables). It will report bad files. Pass the _--fix_ option to effect repairs. + + + diff --git a/hbase-hbck2/pom.xml b/hbase-hbck2/pom.xml index 5de2b1157b..1f61653145 100644 --- a/hbase-hbck2/pom.xml +++ b/hbase-hbck2/pom.xml @@ -214,6 +214,12 @@ org.apache.commons commons-lang3 + + org.mockito + mockito-core + 2.1.0 + test + diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index cbc6184dcd..be09633fa4 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -25,11 +25,18 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.stream.Collectors; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.CompareOperator; import org.apache.hadoop.hbase.HBaseConfiguration; @@ -48,6 +55,9 @@ import org.apache.hadoop.hbase.filter.RowFilter; import org.apache.hadoop.hbase.filter.SubstringComparator; import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.util.Pair; + +import org.apache.hbase.hbck2.meta.MetaFixer; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -65,6 +75,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; + + /** * HBase fixup tool version 2, for hbase-2.0.0+ clusters. * Supercedes hbck1. @@ -88,6 +100,10 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool { private static final String VERSION = "version"; private static final String SET_REGION_STATE = "setRegionState"; private static final String SCHEDULE_RECOVERIES = "scheduleRecoveries"; + private static final String ADD_MISSING_REGIONS_IN_META_FOR_TABLES = + "addMissingRegionsInMetaForTables"; + private static final String ADD_MISSING_REGIONS_IN_META = "addMissingRegionsInMeta"; + private static final String REPORT_MISSING_REGIONS_IN_META = "reportMissingRegionsInMeta"; private Configuration conf; static String [] MINIMUM_HBCK2_VERSION = {"2.0.3", "2.1.1", "2.2.0", "3.0.0"}; private boolean skipCheck = false; @@ -161,6 +177,106 @@ int setRegionState(ClusterConnection connection, String region, return EXIT_FAILURE; } + Map> reportTablesWithMissingRegionsInMeta(String... nameSpaceOrTable) + throws IOException { + Map> report; + try(final MetaFixer metaFixer = new MetaFixer(this.conf)){ + List names = nameSpaceOrTable != null ? Arrays.asList(nameSpaceOrTable) : null; + report = metaFixer.reportTablesMissingRegions(names); + } catch (IOException e) { + LOG.error("Error reporting missing regions: ", e); + throw e; + } + if(LOG.isDebugEnabled()) { + LOG.debug(formatMissingRegionsInMetaReport(report)); + } + return report; + } + + List addMissingRegionsInMeta(List regionsPath) throws IOException { + List reAddedRegionsEncodedNames = new ArrayList<>(); + try(final MetaFixer metaFixer = new MetaFixer(this.conf)){ + for(Path regionPath : regionsPath){ + metaFixer.putRegionInfoFromHdfsInMeta(regionPath); + reAddedRegionsEncodedNames.add(regionPath.getName()); + } + } + return reAddedRegionsEncodedNames; + } + + Pair, List> addMissingRegionsInMetaForTables(String... + nameSpaceOrTable) { + ExecutorService executorService = Executors.newFixedThreadPool( + (nameSpaceOrTable == null || + nameSpaceOrTable.length > Runtime.getRuntime().availableProcessors()) ? + Runtime.getRuntime().availableProcessors() : + nameSpaceOrTable.length); + List>> futures = new ArrayList<>( nameSpaceOrTable == null ? 1 : + nameSpaceOrTable.length); + final List readdedRegionNames = new ArrayList<>(); + List executionErrors = new ArrayList<>(); + try { + //reducing number of retries in case disable fails due to namespace table region also missing + this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); + try(ClusterConnection conn = connect(); + final Admin admin = conn.getAdmin()) { + Map> report = reportTablesWithMissingRegionsInMeta(nameSpaceOrTable); + for (TableName tableName : report.keySet()) { + if(admin.tableExists(tableName)) { + futures.add(executorService.submit(new Callable>() { + @Override + public List call() throws Exception { + LOG.debug("running thread for {}", tableName.getNameWithNamespaceInclAsString()); + boolean didDisable = false; + try { + admin.disableTable(tableName); + didDisable = true; + } catch (IOException e) { + LOG.debug("Failed to disable table {}, " + + "is namespace table also missing regions? Continue anyway...", + tableName.getNameWithNamespaceInclAsString(), e); + } + List reAddedRegions = addMissingRegionsInMeta(report.get(tableName)); + if(didDisable) { + try { + admin.enableTable(tableName); + } catch (IOException e) { + LOG.debug("Failed enabling table {}. It might be that namespace table " + + "region is also missing.\n" + + "After this command finishes, please make sure on this table state.", + tableName.getNameWithNamespaceInclAsString(), e); + } + } + return reAddedRegions; + } + })); + } else { + LOG.warn("Table {} does not exist! Skipping...", + tableName.getNameWithNamespaceInclAsString()); + } + } + for(Future> f : futures){ + try { + readdedRegionNames.addAll(f.get()); + } catch (ExecutionException e){ + //we want to allow potential running threads to finish, so we collect execution + //errors and show those later + LOG.debug("Caught execution error: ", e); + executionErrors.add(e); + } + } + } + } catch (IOException | InterruptedException e) { + LOG.error("ERROR executing thread: ", e); + } finally { + executorService.shutdown(); + } + Pair, List> result = new Pair<>(); + result.setFirst(readdedRegionNames); + result.setSecond(executionErrors); + return result; + } + List assigns(Hbck hbck, String [] args) throws IOException { Options options = new Options(); Option override = Option.builder("o").longOpt("override").build(); @@ -265,6 +381,31 @@ private static String getCommandUsage() { StringWriter sw = new StringWriter(); PrintWriter writer = new PrintWriter(sw); writer.println("Command:"); + writer.println(" " + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + " ..."); + writer.println(" To be used in scenarios where some regions may be missing in META,"); + writer.println(" but there's still a valid 'regioninfo metadata file on HDFS. "); + writer.println(" This is a lighter version of 'OfflineMetaRepair tool commonly used for "); + writer.println(" similar issues on 1.x release line. "); + writer.println(" This command needs META to be online. For each table name passed as"); + writer.println(" parameter, it performs a diff between regions available in META, "); + writer.println(" against existing regions dirs on HDFS. Then, for region dirs with "); + writer.println(" no matches in META, it reads regioninfo metadata file and "); + writer.println(" re-creates given region in META. Regions are re-created in 'CLOSED' "); + writer.println(" state at META table only, but not in Masters' cache, and are not "); + writer.println(" assigned either. A rolling Masters restart, followed by a "); + writer.println(" hbck2 'assigns' command with all re-inserted regions is required. "); + writer.println(" This hbck2 'assigns' command is printed out after this command "); + writer.println(" completes for users subsequent convenience."); + writer.println(" WARNING: To avoid potential region overlapping problems due to ongoing "); + writer.println(" splits, this command attempts to disable given tables "); + writer.println(" (if it fails, disable is skipped) while re-inserting regions. "); + writer.println(" An example adding missing regions for tables 'tbl_1' on default "); + writer.println(" namespace, 'tbl_2' on namespace 'n1' and for all tables from "); + writer.println(" namespace 'n2': "); + writer.println(" $ HBCK2 addMissingRegionsInMeta default:tbl_1 n1:tbl_2 n2 "); + writer.println(" Returns hbck2 'assigns' command with all re-inserted regions."); + writer.println(); writer.println(" " + ASSIGNS + " [OPTIONS] ..."); writer.println(" Options:"); writer.println(" -o,--override override ownership by another procedure"); @@ -313,6 +454,33 @@ private static String getCommandUsage() { writer.println(" '--fix' option. Pass a table name to check for replication barrier and"); writer.println(" purge if '--fix'."); writer.println(); + writer.println(" " + REPORT_MISSING_REGIONS_IN_META + " ..."); + writer.println(" To be used in scenarios where some regions may be missing in META,"); + writer.println(" but there's still a valid 'regioninfo metadata file on HDFS. "); + writer.println(" This is a checking only method, designed for reporting purposes and"); + writer.println(" doesn't perform any fixes, providing a view of which regions (if any) "); + writer.println(" would get re-added to meta, grouped by respective table/namespace. "); + writer.println(" To effectively re-add regions in meta, " + + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + " should be executed. "); + writer.println(" This command needs META to be online. For each namespace/table passed"); + writer.println(" as parameter, it performs a diff between regions available in META, "); + writer.println(" against existing regions dirs on HDFS. Region dirs with no matches"); + writer.println(" are printed grouped under its related table name. Tables with no"); + writer.println(" missing regions will show a 'no missing regions' message. If no"); + writer.println(" namespace or table is specified, it will verify all existing regions."); + writer.println(" It accepts a combination of multiple namespace and tables. Table names"); + writer.println(" should include the namespace portion, even for tables in the default"); + writer.println(" namespace, otherwise it will assume as a namespace value."); + writer.println(" An example triggering missing regions report for tables 'table_1'"); + writer.println(" and 'table_2', under default namespace:"); + writer.println(" $ HBCK2 reportMissingRegionsInMeta default:table_1 default:table_2"); + writer.println(" An example triggering missing regions report for table 'table_1'"); + writer.println(" under default namespace, and for all tables from namespace 'ns1':"); + writer.println(" $ HBCK2 reportMissingRegionsInMeta default:table_1 ns1"); + writer.println(" Returns list of missing regions for each table passed as parameter, or "); + writer.println(" for each table on namespaces specified as parameter."); + writer.println(); writer.println(" " + SET_REGION_STATE + " "); writer.println(" Possible region states:"); writer.println(" OFFLINE, OPENING, OPEN, CLOSING, CLOSED, SPLITTING, SPLIT,"); @@ -364,6 +532,7 @@ private static String getCommandUsage() { writer.println(); writer.println(" SEE ALSO, org.apache.hbase.hbck1.OfflineMetaRepair, the offline"); writer.println(" hbase:meta tool. See the HBCK2 README for how to use."); + writer.println(); writer.close(); return sw.toString(); } @@ -586,6 +755,26 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc } break; + case ADD_MISSING_REGIONS_IN_META_FOR_TABLES: + if(commands.length < 2){ + showErrorMessage(command + " takes one or more table names."); + return EXIT_FAILURE; + } + Pair, List> result = + addMissingRegionsInMetaForTables(purgeFirst(commands)); + System.out.println(formatReAddedRegionsMessage(result.getFirst(),result.getSecond())); + break; + + case REPORT_MISSING_REGIONS_IN_META: + try { + Map> report = + reportTablesWithMissingRegionsInMeta(purgeFirst(commands)); + System.out.println(formatMissingRegionsInMetaReport(report)); + } catch (Exception e) { + return EXIT_FAILURE; + } + break; + default: showErrorMessage("Unsupported command: " + command); return EXIT_FAILURE; @@ -597,6 +786,51 @@ private static String toString(List things) { return things.stream().map(Object::toString).collect(Collectors.joining(", ")); } + private String formatMissingRegionsInMetaReport(Map> report) { + final StringBuilder builder = new StringBuilder(); + builder.append("Missing Regions for each table:\n\t"); + report.keySet().stream().forEach(table -> { + builder.append(table); + if (!report.get(table).isEmpty()){ + builder.append("->\n\t\t"); + report.get(table).stream().forEach(region -> builder.append(region.getName()) + .append(" ")); + } else { + builder.append(" -> No missing regions"); + } + builder.append("\n\t"); + }); + return builder.toString(); + } + + private String formatReAddedRegionsMessage(List readdedRegionNames, + List executionErrors) { + final StringBuilder finalText = new StringBuilder(); + finalText.append("Regions re-added into Meta: ").append(readdedRegionNames.size()); + if(!readdedRegionNames.isEmpty()){ + finalText.append("\n") + .append("WARNING: \n\t") + .append(readdedRegionNames.size()).append(" regions were added ") + .append("to META, but these are not yet on Masters cache. \n") + .append("You need to restart Masters, then run hbck2 'assigns' command below:\n\t\t") + .append(buildHbck2AssignsCommand(readdedRegionNames)); + } + if(!executionErrors.isEmpty()){ + finalText.append("\n") + .append("ERROR: \n\t") + .append("There were following errors on at least one table thread:\n"); + executionErrors.stream().forEach(e -> finalText.append(e.getMessage()).append("\n")); + } + return finalText.toString(); + } + + private String buildHbck2AssignsCommand(List regions) { + final StringBuilder builder = new StringBuilder(); + builder.append("assigns "); + regions.forEach(region -> builder.append(region).append(" ")); + return builder.toString(); + } + /** * @return A new array with first element dropped. */ diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java b/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java new file mode 100644 index 0000000000..2413310e6e --- /dev/null +++ b/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hbase.hbck2.meta; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * This class implements the inner works required for check and recover regions that wrongly + * went missing in META. It assumes HDFS state as the source of truth, in other words, + * methods provided here consider meta information found on HDFS region dirs as the valid ones. + */ +public class MetaFixer implements Closeable { + private static final String TABLE_DESC_FILE = ".tabledesc"; + private static final Logger LOG = LogManager.getLogger(MetaFixer.class); + private final FileSystem fs; + private final Connection conn; + private final Configuration config; + + public MetaFixer(Configuration configuration) throws IOException { + this.config = configuration; + this.fs = FileSystem.get(configuration); + this.conn = ConnectionFactory.createConnection(configuration); + } + + /*Initially defined for test only purposes */ + MetaFixer(Configuration configuration, Connection connection, FileSystem fileSystem){ + this.config = configuration; + this.conn = connection; + this.fs = fileSystem; + } + + private FileStatus[] getTableRegionsDirs(String table) throws Exception { + String hbaseRoot = this.config.get("hbase.rootdir"); + String tableRootDir = hbaseRoot + "/" + HConstants.BASE_NAMESPACE_DIR + "/" + + TableName.valueOf(table).getNameWithNamespaceInclAsString() + .replaceAll(":", "/"); + return fs.listStatus(new Path(tableRootDir)); + } + + public Map> reportTablesMissingRegions(final List namespacesOrTables) + throws IOException { + final Map> result = new HashMap<>(); + List tableNames = MetaTableAccessor.getTableStates(this.conn).keySet().stream() + .filter(tableName -> { + if(namespacesOrTables==null || namespacesOrTables.isEmpty()){ + return true; + } else { + Optional findings = namespacesOrTables.stream().filter( + name -> (name.indexOf(":") > 0) ? + tableName.getNameWithNamespaceInclAsString().equals(name) : + tableName.getNamespaceAsString().equals(name)).findFirst(); + return findings.isPresent(); + } + }).collect(Collectors.toList()); + tableNames.stream().forEach(tableName -> { + try { + result.put(tableName, + findMissingRegionsInMETA(tableName.getNameWithNamespaceInclAsString())); + } catch (Exception e) { + LOG.warn(e); + } + }); + return result; + } + + List findMissingRegionsInMETA(String table) throws Exception { + final List missingRegions = new ArrayList<>(); + final FileStatus[] regionsDirs = getTableRegionsDirs(table); + TableName tableName = TableName.valueOf(table); + List regionInfos = MetaTableAccessor. + getTableRegions(this.conn, tableName, false); + for(final FileStatus regionDir : regionsDirs){ + if(!regionDir.getPath().getName().equals(TABLE_DESC_FILE) && + !regionDir.getPath().getName().equals(HConstants.HBASE_TEMP_DIRECTORY)) { + boolean foundInMeta = regionInfos.stream() + .anyMatch(info -> info.getEncodedName().equals(regionDir.getPath().getName())); + if (!foundInMeta) { + LOG.debug(regionDir + "is not in META."); + missingRegions.add(regionDir.getPath()); + } + } + } + return missingRegions; + } + + public void putRegionInfoFromHdfsInMeta(Path region) throws IOException { + RegionInfo info = HRegionFileSystem.loadRegionInfoFileContent(fs, region); + MetaTableAccessor.addRegionToMeta(conn, info); + } + + @Override public void close() throws IOException { + this.conn.close(); + } +} diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java index 1516503564..4b96bddd3e 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java @@ -25,8 +25,10 @@ import java.util.stream.Collectors; import junit.framework.TestCase; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ClusterConnection; @@ -43,7 +45,15 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestName; + +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.Map; + +import static org.junit.Assert.assertTrue; /** * Tests commands. For command-line parsing, see adjacent test. @@ -56,6 +66,9 @@ public class TestHBCK2 { private static final TableName REGION_STATES_TABLE_NAME = TableName. valueOf(TestHBCK2.class.getSimpleName() + "-REGIONS_STATES"); + @Rule + public TestName testName = new TestName(); + /** * A 'connected' hbck2 instance. */ @@ -175,6 +188,128 @@ public void testSetRegionStateInvalidState() throws IOException { } } + @Test + public void testAddMissingRegionsInMetaAllRegionsMissing() throws Exception { + this.testAddMissingRegionsInMetaForTables(5,5); + } + + @Test + public void testAddMissingRegionsInMetaTwoMissingOnly() throws Exception { + this.testAddMissingRegionsInMetaForTables(2,5); + } + + @Test + public void testReportMissingRegionsInMetaAllNsTbls() throws Exception { + this.testReportMissingRegionsInMeta(5, 5,null); + } + + @Test + public void testReportMissingRegionsInMetaSpecificTbl() throws Exception { + this.testReportMissingRegionsInMeta(5, 5, + TABLE_NAME.getNameWithNamespaceInclAsString()); + } + + @Test + public void testReportMissingRegionsInMetaSpecificTblAndNsTbl() throws Exception { + this.testReportMissingRegionsInMeta(5, 5, + TABLE_NAME.getNameWithNamespaceInclAsString(), "hbase:namespace"); + } + + @Test + public void testReportMissingRegionsInMetaSpecificTblAndNsTblAlsoMissing() throws Exception { + List regions = MetaTableAccessor + .getTableRegions(TEST_UTIL.getConnection(), TableName.valueOf("hbase:namespace")); + MetaTableAccessor.deleteRegions(TEST_UTIL.getConnection(), + regions.subList(0,1)); + this.testReportMissingRegionsInMeta(5, 6, + TABLE_NAME.getNameWithNamespaceInclAsString(), "hbase:namespace"); + } + + @Test + public void testFormatReportMissingRegionsInMetaNoMissing() throws IOException { + final String expectedResult = "Missing Regions for each table:\n" + + "\tTestHBCK2 -> No missing regions\n\thbase:namespace -> No missing regions\n\t\n"; + String result = testFormatMissingRegionsInMetaReport(); + assertTrue(result.contains(expectedResult)); + } + + @Test + public void testFormatReportMissingInMetaOneMissing() throws IOException { + TableName tableName = createTestTable(5); + List regions = MetaTableAccessor + .getTableRegions(TEST_UTIL.getConnection(), tableName); + MetaTableAccessor.deleteRegions(TEST_UTIL.getConnection(), regions.subList(0,1)); + String expectedResult = "Missing Regions for each table:\n"; + String result = testFormatMissingRegionsInMetaReport(); + //validates initial report message + assertTrue(result.contains(expectedResult)); + //validates our test table region is reported missing + expectedResult = "\t" + tableName.getNameAsString() + "->\n\t\t" + + regions.get(0).getEncodedName(); + assertTrue(result.contains(expectedResult)); + //validates namespace region is not reported missing + expectedResult = "\n\thbase:namespace -> No missing regions\n\t"; + assertTrue(result.contains(expectedResult)); + } + + private String testFormatMissingRegionsInMetaReport() + throws IOException { + HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration()); + final StringBuilder builder = new StringBuilder(); + PrintStream originalOS = System.out; + OutputStream testOS = new OutputStream() { + @Override public void write(int b) throws IOException { + builder.append((char)b); + } + }; + System.setOut(new PrintStream(testOS)); + + hbck.run(new String[]{"reportMissingRegionsInMeta"}); + System.setOut(originalOS); + return builder.toString(); + } + + private TableName createTestTable(int totalRegions) throws IOException { + TableName tableName = TableName.valueOf(testName.getMethodName()); + TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("family1"), totalRegions); + return tableName; + } + + private void testAddMissingRegionsInMetaForTables(int missingRegions, int totalRegions) + throws Exception { + TableName tableName = createTestTable(totalRegions); + HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration()); + List regions = MetaTableAccessor + .getTableRegions(TEST_UTIL.getConnection(), tableName); + MetaTableAccessor.deleteRegions(TEST_UTIL.getConnection(), regions.subList(0,missingRegions)); + int remaining = totalRegions - missingRegions; + assertEquals("Table should have " + remaining + " regions in META.", remaining, + MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName)); + assertEquals(missingRegions,hbck.addMissingRegionsInMetaForTables("default:" + + tableName.getNameAsString()).getFirst().size()); + assertEquals("Table regions should had been re-added in META.", totalRegions, + MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName)); + //compare the added regions to make sure those are the same + List newRegions = MetaTableAccessor + .getTableRegions(TEST_UTIL.getConnection(), tableName); + assertEquals("All re-added regions should be the same", regions, newRegions); + } + + private void testReportMissingRegionsInMeta(int missingRegionsInTestTbl, + int expectedTotalMissingRegions, String... namespaceOrTable) throws Exception { + List regions = MetaTableAccessor + .getTableRegions(TEST_UTIL.getConnection(), TABLE_NAME); + MetaTableAccessor.deleteRegions(TEST_UTIL.getConnection(), + regions.subList(0,missingRegionsInTestTbl)); + HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration()); + final Map> report = + hbck.reportTablesWithMissingRegionsInMeta(namespaceOrTable); + long resultingMissingRegions = report.keySet().stream().mapToLong( nsTbl -> + report.get(nsTbl).size()).sum(); + assertEquals(expectedTotalMissingRegions, resultingMissingRegions); + hbck.addMissingRegionsInMetaForTables(null); + } + @Test (expected = IllegalArgumentException.class) public void testSetRegionStateInvalidRegionAndInvalidState() throws IOException { try (ClusterConnection connection = this.hbck2.connect()) { diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java b/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java new file mode 100644 index 0000000000..345c6b0f9e --- /dev/null +++ b/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java @@ -0,0 +1,208 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hbase.hbck2.meta; + +import static org.junit.Assert.assertEquals; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellBuilderFactory; +import org.apache.hadoop.hbase.CellBuilderType; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class TestMetaFixer { + + private Connection mockedConnection; + private FileSystem mockedFileSystem; + private Table mockedTable; + private MetaFixer fixer; + private String testTblDir; + + @Before + public void setup() throws Exception { + this.mockedConnection = Mockito.mock(Connection.class); + this.mockedFileSystem = Mockito.mock(FileSystem.class); + this.mockedTable = Mockito.mock(Table.class); + Configuration config = HBaseConfiguration.create(); + Mockito.when(this.mockedConnection.getConfiguration()).thenReturn(config); + Mockito.when(this.mockedConnection.getTable(TableName.META_TABLE_NAME)).thenReturn(mockedTable); + this.testTblDir = config.get("hbase.rootdir") + "/data/default/test-tbl"; + this.fixer = new MetaFixer(config, mockedConnection, mockedFileSystem); + } + + private RegionInfo createRegionInfo(String table){ + long regionTS = System.currentTimeMillis(); + RegionInfo info = RegionInfoBuilder.newBuilder(TableName.valueOf(table)) + .setRegionId(regionTS) + .build(); + return info; + } + + private Cell createCellForRegionInfo(RegionInfo info){ + byte[] regionInfoValue = ProtobufUtil.prependPBMagic(ProtobufUtil.toRegionInfo(info) + .toByteArray()); + Cell cell = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) + .setRow(info.getRegionName()) + .setFamily(Bytes.toBytes("info")) + .setQualifier(Bytes.toBytes("regioninfo")) + .setType(Cell.Type.Put) + .setValue(regionInfoValue) + .build(); + return cell; + } + + private Cell createCellForTableState(TableName tableName){ + Cell cell = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) + .setRow(tableName.getName()) + .setFamily(Bytes.toBytes("table")) + .setQualifier(Bytes.toBytes("state")) + .setType(Cell.Type.Put) + .setValue(HBaseProtos.TableState.newBuilder() + .setState(TableState.State.ENABLED.convert()).build().toByteArray()) + .build(); + return cell; + } + + @Test + public void testFindMissingRegionsInMETANoMissing() throws Exception { + ResultScanner mockedRS = Mockito.mock(ResultScanner.class); + Mockito.when(this.mockedTable.getScanner(Mockito.any(Scan.class))).thenReturn(mockedRS); + RegionInfo info = createRegionInfo("test-tbl"); + List cells = new ArrayList(); + cells.add(createCellForRegionInfo(info)); + Result result = Result.create(cells); + Mockito.when(mockedRS.next()).thenReturn(result,null); + FileStatus status = new FileStatus(); + status.setPath(new Path(this.testTblDir + "/" + info.getEncodedName())); + Mockito.when(mockedFileSystem.listStatus(new Path(this.testTblDir))) + .thenReturn(new FileStatus[]{status}); + assertEquals("Should had returned 0 missing regions", + 0, fixer.findMissingRegionsInMETA("test-tbl").size()); + } + + @Test + public void testFindMissingRegionsInMETAOneMissing() throws Exception { + ResultScanner mockedRS = Mockito.mock(ResultScanner.class); + Mockito.when(this.mockedTable.getScanner(Mockito.any(Scan.class))).thenReturn(mockedRS); + List cells = new ArrayList(); + Result result = Result.create(cells); + Mockito.when(mockedRS.next()).thenReturn(result,null); + FileStatus status = new FileStatus(); + Path p = new Path(this.testTblDir+ "/182182182121"); + status.setPath(p); + Mockito.when(mockedFileSystem.listStatus(new Path(this.testTblDir))) + .thenReturn(new FileStatus[]{status}); + List missingRegions = fixer.findMissingRegionsInMETA("test-tbl"); + assertEquals("Should had returned 1 missing region", + 1, missingRegions.size()); + assertEquals(p,missingRegions.get(0)); + } + + @Test + public void testPutRegionInfoFromHdfsInMeta() throws Exception { + RegionInfo info = this.createRegionInfo("test-tbl"); + Path regionPath = new Path("/hbase/data/default/test-tbl/" + info.getEncodedName()); + FSDataInputStream fis = new FSDataInputStream(new TestInputStreamSeekable(info)); + Mockito.when(this.mockedFileSystem.open(new Path(regionPath, ".regioninfo"))) + .thenReturn(fis); + fixer.putRegionInfoFromHdfsInMeta(regionPath); + Mockito.verify(this.mockedConnection).getTable(TableName.META_TABLE_NAME); + Mockito.verify(this.mockedTable).put(Mockito.any(Put.class)); + } + + @Test + public void testReportTablesMissingRegionsOneMissing() throws Exception { + ResultScanner mockedRS = Mockito.mock(ResultScanner.class); + Mockito.when(this.mockedTable.getScanner(Mockito.any(Scan.class))).thenReturn(mockedRS); + List cells = new ArrayList(); + cells.add(createCellForTableState(TableName.valueOf("test-tbl"))); + Result result = Result.create(cells); + Mockito.when(mockedRS.next()).thenReturn(result,null); + FileStatus status = new FileStatus(); + Path p = new Path(this.testTblDir+ "/182182182121"); + status.setPath(p); + Mockito.when(mockedFileSystem.listStatus(new Path(this.testTblDir))) + .thenReturn(new FileStatus[]{status}); + Mockito.when(this.mockedConnection.getTable(TableName.META_TABLE_NAME)) + .thenReturn(this.mockedTable); + Map> report = fixer.reportTablesMissingRegions(null); + assertEquals("Should had returned 1 missing region", + 1,report.size()); + } + + private class TestInputStreamSeekable extends FSInputStream { + + private ByteArrayInputStream in; + private long length; + + private TestInputStreamSeekable(RegionInfo info) throws Exception { + byte[] bytes = RegionInfo.toDelimitedByteArray(info); + this.length = bytes.length; + this.in = new ByteArrayInputStream(bytes); + } + + @Override + public void seek(long l) throws IOException { + this.in.skip(l); + } + + @Override + public long getPos() throws IOException { + return this.length - in.available(); + } + + @Override + public boolean seekToNewSource(long l) throws IOException { + this.in.skip(l); + return true; + } + + @Override + public int read() throws IOException { + return in.read(); + } + } + +} + From 2e3fab7f6d9aa55e7f1a2767d5c3522d5245d162 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Mon, 2 Sep 2019 15:41:32 +0100 Subject: [PATCH 02/10] HBASE-22567 - HBCK2 addMissingRegionsToMeta Addressing following PR suggestions: 1) should we be throwing an exception to fail the tool run here? 2) could we add an option to fail the command if the disable fails? for folks who don't want to risk the overlap? --- .../src/main/java/org/apache/hbase/HBCK2.java | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index be09633fa4..ed76834b70 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -205,7 +205,21 @@ List addMissingRegionsInMeta(List regionsPath) throws IOException } Pair, List> addMissingRegionsInMetaForTables(String... - nameSpaceOrTable) { + nameSpaceOrTable) throws IOException { + //TODO next block logic for parsing optional parameters can be moved to util method for reuse + Options options = new Options(); + Option disable = Option.builder("d").longOpt("force_disable").build(); + options.addOption(disable); + // Parse command-line. + CommandLineParser parser = new DefaultParser(); + CommandLine commandLine; + try { + commandLine = parser.parse(options, nameSpaceOrTable, false); + } catch (ParseException e) { + showErrorMessage(e.getMessage()); + return null; + } + boolean enforceDisable = commandLine.hasOption(disable.getOpt()); ExecutorService executorService = Executors.newFixedThreadPool( (nameSpaceOrTable == null || nameSpaceOrTable.length > Runtime.getRuntime().availableProcessors()) ? @@ -233,8 +247,17 @@ public List call() throws Exception { didDisable = true; } catch (IOException e) { LOG.debug("Failed to disable table {}, " - + "is namespace table also missing regions? Continue anyway...", + + "is namespace table also missing regions?", tableName.getNameWithNamespaceInclAsString(), e); + if (enforceDisable) { + final StringBuilder errorMsgBuilder = + new StringBuilder("Failed re-adding following regions: \n\t"); + report.get(tableName).forEach( r -> + errorMsgBuilder.append(r.getName()).append("\t")); + throw new IOException(errorMsgBuilder.toString()); + } else { + LOG.debug("Continuing anyway, as no force_disable."); + } } List reAddedRegions = addMissingRegionsInMeta(report.get(tableName)); if(didDisable) { @@ -268,6 +291,7 @@ public List call() throws Exception { } } catch (IOException | InterruptedException e) { LOG.error("ERROR executing thread: ", e); + throw new IOException(e); } finally { executorService.shutdown(); } @@ -381,8 +405,10 @@ private static String getCommandUsage() { StringWriter sw = new StringWriter(); PrintWriter writer = new PrintWriter(sw); writer.println("Command:"); - writer.println(" " + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + " ..."); + writer.println(" Options:"); + writer.println(" -d,--force_disable aborts fix for table if disable fails."); writer.println(" To be used in scenarios where some regions may be missing in META,"); writer.println(" but there's still a valid 'regioninfo metadata file on HDFS. "); writer.println(" This is a lighter version of 'OfflineMetaRepair tool commonly used for "); From 581e7ab4b012e4f0e049340c93fbba228d30f592 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Tue, 3 Sep 2019 12:12:22 +0100 Subject: [PATCH 03/10] Addressing PR suggestions from ankitsinghal made on 2019-09-02 --- hbase-hbck2/README.md | 8 +++-- .../src/main/java/org/apache/hbase/HBCK2.java | 11 ++++--- .../apache/hbase/hbck2/meta/MetaFixer.java | 32 ++++++++----------- .../hbase/hbck2/meta/TestMetaFixer.java | 3 +- 4 files changed, 26 insertions(+), 28 deletions(-) diff --git a/hbase-hbck2/README.md b/hbase-hbck2/README.md index 6617f26e09..088e1d3acb 100644 --- a/hbase-hbck2/README.md +++ b/hbase-hbck2/README.md @@ -400,7 +400,9 @@ echo "scan 'hbase:meta', {COLUMN=>'info:regioninfo'}" | hbase shell ``` HBCK2 _addMissingRegionsInMeta_ can be used if the above does not show any errors. It reads region -metadata info available on the FS region dirs, in order to re-create regions in META. +metadata info available on the FS region dirs, in order to re-create regions in META. Since it can +run with hbase partially operational, it attempts to disable online tables that are affected by the +reported problem and is gonna have regions re-added to _meta_. It can check for specific tables/namespaces, or all tables from all namespaces. An example adding missing regions for tables 'tbl_1' on default namespace, 'tbl_2' on namespace 'n1' and for all tables from namespace 'n2': @@ -415,8 +417,8 @@ required to actually have the re-added regions assigned. These are listed below: 1. _addMissingRegionsInMeta_ outputs an _assigns_ command with all regions that got re-added. This command needs to be executed later, so copy and save it for convenience. -2. For HBase versions prior to 2.3.0, after _addMissingRegionsInMeta_ finished successfully and output has been saved, restart all -running HBase Masters. +2. For HBase versions prior to 2.3.0, after _addMissingRegionsInMeta_ finished successfully and output has been saved, +restart all running HBase Masters. 3. Once Master's are restarted and META is already online (check if Web UI is accessible), run _assigns_ command from _addMissingRegionsInMeta_ output saved per instructions from #1. diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index ed76834b70..ebb2b0695d 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -419,13 +419,16 @@ private static String getCommandUsage() { writer.println(" no matches in META, it reads regioninfo metadata file and "); writer.println(" re-creates given region in META. Regions are re-created in 'CLOSED' "); writer.println(" state at META table only, but not in Masters' cache, and are not "); - writer.println(" assigned either. A rolling Masters restart, followed by a "); - writer.println(" hbck2 'assigns' command with all re-inserted regions is required. "); - writer.println(" This hbck2 'assigns' command is printed out after this command "); - writer.println(" completes for users subsequent convenience."); + writer.println(" assigned either. To get these regions online, run hbck2 'assigns'command "); + writer.println(" printed at the end of this command results for convenience."); + writer.println(); + writer.println(" NOTE: If using hbase releases older than 2.3.0, a rolling restart of "); + writer.println(" HMasters is needed prior to executing the provided 'assigns' command. "); + writer.println(); writer.println(" WARNING: To avoid potential region overlapping problems due to ongoing "); writer.println(" splits, this command attempts to disable given tables "); writer.println(" (if it fails, disable is skipped) while re-inserting regions. "); + writer.println(); writer.println(" An example adding missing regions for tables 'tbl_1' on default "); writer.println(" namespace, 'tbl_2' on namespace 'n1' and for all tables from "); writer.println(" namespace 'n2': "); diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java b/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java index 2413310e6e..c7f68de5b7 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java @@ -18,16 +18,15 @@ package org.apache.hbase.hbck2.meta; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.util.FSUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -35,6 +34,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -46,7 +46,6 @@ * methods provided here consider meta information found on HDFS region dirs as the valid ones. */ public class MetaFixer implements Closeable { - private static final String TABLE_DESC_FILE = ".tabledesc"; private static final Logger LOG = LogManager.getLogger(MetaFixer.class); private final FileSystem fs; private final Connection conn; @@ -65,12 +64,10 @@ public MetaFixer(Configuration configuration) throws IOException { this.fs = fileSystem; } - private FileStatus[] getTableRegionsDirs(String table) throws Exception { + private List getTableRegionsDirs(String table) throws Exception { String hbaseRoot = this.config.get("hbase.rootdir"); - String tableRootDir = hbaseRoot + "/" + HConstants.BASE_NAMESPACE_DIR + "/" + - TableName.valueOf(table).getNameWithNamespaceInclAsString() - .replaceAll(":", "/"); - return fs.listStatus(new Path(tableRootDir)); + Path tableDir = FSUtils.getTableDir(new Path(hbaseRoot), TableName.valueOf(table)); + return FSUtils.getRegionDirs(fs, tableDir); } public Map> reportTablesMissingRegions(final List namespacesOrTables) @@ -83,7 +80,7 @@ public Map> reportTablesMissingRegions(final List n } else { Optional findings = namespacesOrTables.stream().filter( name -> (name.indexOf(":") > 0) ? - tableName.getNameWithNamespaceInclAsString().equals(name) : + tableName.equals(TableName.valueOf(name)) : tableName.getNamespaceAsString().equals(name)).findFirst(); return findings.isPresent(); } @@ -101,19 +98,16 @@ public Map> reportTablesMissingRegions(final List n List findMissingRegionsInMETA(String table) throws Exception { final List missingRegions = new ArrayList<>(); - final FileStatus[] regionsDirs = getTableRegionsDirs(table); + final List regionsDirs = getTableRegionsDirs(table); TableName tableName = TableName.valueOf(table); List regionInfos = MetaTableAccessor. getTableRegions(this.conn, tableName, false); - for(final FileStatus regionDir : regionsDirs){ - if(!regionDir.getPath().getName().equals(TABLE_DESC_FILE) && - !regionDir.getPath().getName().equals(HConstants.HBASE_TEMP_DIRECTORY)) { - boolean foundInMeta = regionInfos.stream() - .anyMatch(info -> info.getEncodedName().equals(regionDir.getPath().getName())); - if (!foundInMeta) { - LOG.debug(regionDir + "is not in META."); - missingRegions.add(regionDir.getPath()); - } + HashSet regionsInMeta = regionInfos.stream().map(info -> + info.getEncodedName()).collect(Collectors.toCollection(HashSet::new)); + for(final Path regionDir : regionsDirs){ + if (!regionsInMeta.contains(regionDir.getName())) { + LOG.debug(regionDir + "is not in META."); + missingRegions.add(regionDir); } } return missingRegions; diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java b/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java index 345c6b0f9e..807c24241a 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java @@ -128,9 +128,8 @@ public void testFindMissingRegionsInMETAOneMissing() throws Exception { List cells = new ArrayList(); Result result = Result.create(cells); Mockito.when(mockedRS.next()).thenReturn(result,null); - FileStatus status = new FileStatus(); Path p = new Path(this.testTblDir+ "/182182182121"); - status.setPath(p); + FileStatus status = new FileStatus(0, true, 0, 0,0, p); Mockito.when(mockedFileSystem.listStatus(new Path(this.testTblDir))) .thenReturn(new FileStatus[]{status}); List missingRegions = fixer.findMissingRegionsInMETA("test-tbl"); From 25aa329563bde7b19d47f4cd6d946a2dbebb70b6 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Tue, 3 Sep 2019 16:20:26 +0100 Subject: [PATCH 04/10] Addressing additional comments from ankitsinghal on PR #18 --- .../src/main/java/org/apache/hbase/HBCK2.java | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index ebb2b0695d..96c8c3eb96 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -242,32 +242,34 @@ Pair, List> addMissingRegionsInMetaForTables(St public List call() throws Exception { LOG.debug("running thread for {}", tableName.getNameWithNamespaceInclAsString()); boolean didDisable = false; - try { - admin.disableTable(tableName); - didDisable = true; - } catch (IOException e) { - LOG.debug("Failed to disable table {}, " - + "is namespace table also missing regions?", - tableName.getNameWithNamespaceInclAsString(), e); - if (enforceDisable) { - final StringBuilder errorMsgBuilder = - new StringBuilder("Failed re-adding following regions: \n\t"); - report.get(tableName).forEach( r -> - errorMsgBuilder.append(r.getName()).append("\t")); - throw new IOException(errorMsgBuilder.toString()); - } else { - LOG.debug("Continuing anyway, as no force_disable."); - } - } - List reAddedRegions = addMissingRegionsInMeta(report.get(tableName)); - if(didDisable) { + if(admin.isTableEnabled(tableName)){ try { - admin.enableTable(tableName); + admin.disableTable(tableName); + didDisable = true; } catch (IOException e) { - LOG.debug("Failed enabling table {}. It might be that namespace table " - + "region is also missing.\n" - + "After this command finishes, please make sure on this table state.", + LOG.debug("Failed to disable table {}, " + "is namespace table also missing regions?", tableName.getNameWithNamespaceInclAsString(), e); + if (enforceDisable) { + final StringBuilder errorMsgBuilder = new StringBuilder("Failed re-adding following regions: \n\t"); + report.get(tableName).forEach(r -> errorMsgBuilder.append(r.getName()).append("\t")); + throw new IOException(errorMsgBuilder.toString()); + } else { + LOG.debug("Continuing anyway, as no force_disable."); + } + } + } + List reAddedRegions; + try { + reAddedRegions = addMissingRegionsInMeta(report.get(tableName)); + } finally { + if (didDisable) { + try { + admin.enableTable(tableName); + } catch (IOException e) { + LOG.debug("Failed enabling table {}. It might be that namespace table " + "region is also missing.\n" + + "After this command finishes, please make sure on this table state.", + tableName.getNameWithNamespaceInclAsString(), e); + } } } return reAddedRegions; From e15eb702eeb173a6124fd2296c7d220373e4349e Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 4 Sep 2019 16:49:06 +0100 Subject: [PATCH 05/10] Addressing latest PR comments from 2019/09/04 --- .../src/main/java/org/apache/hbase/HBCK2.java | 18 ++++++++++-------- ...aFixer.java => FsRegionsMetaRecoverer.java} | 18 +++++++++++------- ...er.java => TestFsRegionsMetaRecoverer.java} | 9 +++++---- 3 files changed, 26 insertions(+), 19 deletions(-) rename hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/{MetaFixer.java => FsRegionsMetaRecoverer.java} (84%) rename hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/{TestMetaFixer.java => TestFsRegionsMetaRecoverer.java} (96%) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 96c8c3eb96..129ee24dd5 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -57,7 +57,7 @@ import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.util.Pair; -import org.apache.hbase.hbck2.meta.MetaFixer; +import org.apache.hbase.hbck2.meta.FsRegionsMetaRecoverer; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -101,7 +101,7 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool { private static final String SET_REGION_STATE = "setRegionState"; private static final String SCHEDULE_RECOVERIES = "scheduleRecoveries"; private static final String ADD_MISSING_REGIONS_IN_META_FOR_TABLES = - "addMissingRegionsInMetaForTables"; + "addFsRegionsMissingInMeta"; private static final String ADD_MISSING_REGIONS_IN_META = "addMissingRegionsInMeta"; private static final String REPORT_MISSING_REGIONS_IN_META = "reportMissingRegionsInMeta"; private Configuration conf; @@ -180,9 +180,9 @@ int setRegionState(ClusterConnection connection, String region, Map> reportTablesWithMissingRegionsInMeta(String... nameSpaceOrTable) throws IOException { Map> report; - try(final MetaFixer metaFixer = new MetaFixer(this.conf)){ + try(final FsRegionsMetaRecoverer fsRegionsMetaRecoverer = new FsRegionsMetaRecoverer(this.conf)){ List names = nameSpaceOrTable != null ? Arrays.asList(nameSpaceOrTable) : null; - report = metaFixer.reportTablesMissingRegions(names); + report = fsRegionsMetaRecoverer.reportTablesMissingRegions(names); } catch (IOException e) { LOG.error("Error reporting missing regions: ", e); throw e; @@ -195,9 +195,9 @@ Map> reportTablesWithMissingRegionsInMeta(String... nameSpa List addMissingRegionsInMeta(List regionsPath) throws IOException { List reAddedRegionsEncodedNames = new ArrayList<>(); - try(final MetaFixer metaFixer = new MetaFixer(this.conf)){ + try(final FsRegionsMetaRecoverer fsRegionsMetaRecoverer = new FsRegionsMetaRecoverer(this.conf)){ for(Path regionPath : regionsPath){ - metaFixer.putRegionInfoFromHdfsInMeta(regionPath); + fsRegionsMetaRecoverer.putRegionInfoFromHdfsInMeta(regionPath); reAddedRegionsEncodedNames.add(regionPath.getName()); } } @@ -412,7 +412,7 @@ private static String getCommandUsage() { writer.println(" Options:"); writer.println(" -d,--force_disable aborts fix for table if disable fails."); writer.println(" To be used in scenarios where some regions may be missing in META,"); - writer.println(" but there's still a valid 'regioninfo metadata file on HDFS. "); + writer.println(" but there's still a valid 'regioninfo' metadata file on HDFS. "); writer.println(" This is a lighter version of 'OfflineMetaRepair tool commonly used for "); writer.println(" similar issues on 1.x release line. "); writer.println(" This command needs META to be online. For each table name passed as"); @@ -434,8 +434,10 @@ private static String getCommandUsage() { writer.println(" An example adding missing regions for tables 'tbl_1' on default "); writer.println(" namespace, 'tbl_2' on namespace 'n1' and for all tables from "); writer.println(" namespace 'n2': "); - writer.println(" $ HBCK2 addMissingRegionsInMeta default:tbl_1 n1:tbl_2 n2 "); + writer.println(" $ HBCK2 " + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + + " default:tbl_1 n1:tbl_2 n2 "); writer.println(" Returns hbck2 'assigns' command with all re-inserted regions."); + writer.println(" SEE ALSO: " + REPORT_MISSING_REGIONS_IN_META); writer.println(); writer.println(" " + ASSIGNS + " [OPTIONS] ..."); writer.println(" Options:"); diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java b/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/FsRegionsMetaRecoverer.java similarity index 84% rename from hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java rename to hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/FsRegionsMetaRecoverer.java index c7f68de5b7..6c63a54168 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/FsRegionsMetaRecoverer.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; @@ -42,30 +43,33 @@ /** * This class implements the inner works required for check and recover regions that wrongly - * went missing in META. It assumes HDFS state as the source of truth, in other words, - * methods provided here consider meta information found on HDFS region dirs as the valid ones. + * went missing in META. + * Normally HBCK2 fix options rely on Master self-contained information to recover/fix + * inconsistencies, but this an exception case where META table is in a broken state. + * So, it assumes HDFS state as the source of truth, in other words, methods provided here consider + * meta information found on HDFS region dirs as the valid ones. */ -public class MetaFixer implements Closeable { - private static final Logger LOG = LogManager.getLogger(MetaFixer.class); +public class FsRegionsMetaRecoverer implements Closeable { + private static final Logger LOG = LogManager.getLogger(FsRegionsMetaRecoverer.class); private final FileSystem fs; private final Connection conn; private final Configuration config; - public MetaFixer(Configuration configuration) throws IOException { + public FsRegionsMetaRecoverer(Configuration configuration) throws IOException { this.config = configuration; this.fs = FileSystem.get(configuration); this.conn = ConnectionFactory.createConnection(configuration); } /*Initially defined for test only purposes */ - MetaFixer(Configuration configuration, Connection connection, FileSystem fileSystem){ + FsRegionsMetaRecoverer(Configuration configuration, Connection connection, FileSystem fileSystem){ this.config = configuration; this.conn = connection; this.fs = fileSystem; } private List getTableRegionsDirs(String table) throws Exception { - String hbaseRoot = this.config.get("hbase.rootdir"); + String hbaseRoot = this.config.get(HConstants.HBASE_DIR); Path tableDir = FSUtils.getTableDir(new Path(hbaseRoot), TableName.valueOf(table)); return FSUtils.getRegionDirs(fs, tableDir); } diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java b/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestFsRegionsMetaRecoverer.java similarity index 96% rename from hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java rename to hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestFsRegionsMetaRecoverer.java index 807c24241a..90498b825c 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestFsRegionsMetaRecoverer.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Put; @@ -51,12 +52,12 @@ import java.util.List; import java.util.Map; -public class TestMetaFixer { +public class TestFsRegionsMetaRecoverer { private Connection mockedConnection; private FileSystem mockedFileSystem; private Table mockedTable; - private MetaFixer fixer; + private FsRegionsMetaRecoverer fixer; private String testTblDir; @Before @@ -67,8 +68,8 @@ public void setup() throws Exception { Configuration config = HBaseConfiguration.create(); Mockito.when(this.mockedConnection.getConfiguration()).thenReturn(config); Mockito.when(this.mockedConnection.getTable(TableName.META_TABLE_NAME)).thenReturn(mockedTable); - this.testTblDir = config.get("hbase.rootdir") + "/data/default/test-tbl"; - this.fixer = new MetaFixer(config, mockedConnection, mockedFileSystem); + this.testTblDir = config.get(HConstants.HBASE_DIR) + "/data/default/test-tbl"; + this.fixer = new FsRegionsMetaRecoverer(config, mockedConnection, mockedFileSystem); } private RegionInfo createRegionInfo(String table){ From a74cb9c2c0c1b71e1b0051c11a8b5849ef4f25bb Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 4 Sep 2019 17:10:55 +0100 Subject: [PATCH 06/10] updated readme to reflect the method name changes, and also to mention -skip flag if namespace region is among the missing ones. --- hbase-hbck2/README.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/hbase-hbck2/README.md b/hbase-hbck2/README.md index 088e1d3acb..baa785f038 100644 --- a/hbase-hbck2/README.md +++ b/hbase-hbck2/README.md @@ -384,7 +384,7 @@ for fixing META table related issues on HBase 1.x versions. The original version with HBase 2.x or higher versions, and it has undergone some adjustments to be now run within hbck2. In most of these cases, regions may be missing in meta at random, but hbase may still be -operational. In such situations, problem can be addressed with master online, using _addMissingRegionsInMeta_ command. +operational. In such situations, problem can be addressed with master online, using _addFsRegionsMissingInMeta_ command. This command is less disruptive to hbase than the full meta rebuild covered later, and it can be used even for recovering _namespace_ table region. @@ -399,7 +399,7 @@ as below. If it does not time out or show any errors, _meta_ is online: echo "scan 'hbase:meta', {COLUMN=>'info:regioninfo'}" | hbase shell ``` -HBCK2 _addMissingRegionsInMeta_ can be used if the above does not show any errors. It reads region +HBCK2 _addFsRegionsMissingInMeta_ can be used if the above does not show any errors. It reads region metadata info available on the FS region dirs, in order to re-create regions in META. Since it can run with hbase partially operational, it attempts to disable online tables that are affected by the reported problem and is gonna have regions re-added to _meta_. @@ -408,20 +408,23 @@ from all namespaces. An example adding missing regions for tables 'tbl_1' on def 'tbl_2' on namespace 'n1' and for all tables from namespace 'n2': ``` -$ HBCK2 addMissingRegionsInMeta default:tbl_1 n1:tbl_2 n2 +$ HBCK2 addFsRegionsMissingInMeta default:tbl_1 n1:tbl_2 n2 ``` As it operates independently from Master, once it finishes successfully, additional steps are required to actually have the re-added regions assigned. These are listed below: -1. _addMissingRegionsInMeta_ outputs an _assigns_ command with all regions that got re-added. This +1. _addFsRegionsMissingInMeta_ outputs an _assigns_ command with all regions that got re-added. This command needs to be executed later, so copy and save it for convenience. -2. For HBase versions prior to 2.3.0, after _addMissingRegionsInMeta_ finished successfully and output has been saved, +2. For HBase versions prior to 2.3.0, after _addFsRegionsMissingInMeta_ finished successfully and output has been saved, restart all running HBase Masters. 3. Once Master's are restarted and META is already online (check if Web UI is accessible), run -_assigns_ command from _addMissingRegionsInMeta_ output saved per instructions from #1. +_assigns_ command from _addFsRegionsMissingInMeta_ output saved per instructions from #1. + +NOTE: If _namespace_ region is among the missing ones, you will need to add _--skip_ flag at the +beginning of _assigns_ command returned. Should a cluster suffer a catastrophic loss of the `hbase:meta` region, a rough rebuild is possible following the below recipe. From f85865dfa18bfcdeff5c0ef3d757d7b09e1e9dab Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 4 Sep 2019 17:17:05 +0100 Subject: [PATCH 07/10] Moving former MetaFixer class (now FsRegionsMetaRecoverer) from 'hbck2.meta' package to project's top level package. --- .../apache/hbase/{hbck2/meta => }/FsRegionsMetaRecoverer.java | 2 +- hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java | 2 -- .../hbase/{hbck2/meta => }/TestFsRegionsMetaRecoverer.java | 3 ++- 3 files changed, 3 insertions(+), 4 deletions(-) rename hbase-hbck2/src/main/java/org/apache/hbase/{hbck2/meta => }/FsRegionsMetaRecoverer.java (99%) rename hbase-hbck2/src/test/java/org/apache/hbase/{hbck2/meta => }/TestFsRegionsMetaRecoverer.java (99%) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/FsRegionsMetaRecoverer.java b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java similarity index 99% rename from hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/FsRegionsMetaRecoverer.java rename to hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java index 6c63a54168..09899c69b3 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/FsRegionsMetaRecoverer.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hbase.hbck2.meta; +package org.apache.hbase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 129ee24dd5..97c2924876 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -57,8 +57,6 @@ import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.util.Pair; -import org.apache.hbase.hbck2.meta.FsRegionsMetaRecoverer; - import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestFsRegionsMetaRecoverer.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java similarity index 99% rename from hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestFsRegionsMetaRecoverer.java rename to hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java index 90498b825c..c06e92dd8a 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestFsRegionsMetaRecoverer.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestFsRegionsMetaRecoverer.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hbase.hbck2.meta; +package org.apache.hbase; import static org.junit.Assert.assertEquals; import org.apache.hadoop.conf.Configuration; @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.FsRegionsMetaRecoverer; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; From d845fcf1dc82223aaf009801500d8979e62ca704 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 4 Sep 2019 17:48:30 +0100 Subject: [PATCH 08/10] replacing mentions to 'hbck2' to 'HBCK2' --- .../src/main/java/org/apache/hbase/HBCK2.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 97c2924876..ef85b0f747 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -245,11 +245,14 @@ public List call() throws Exception { admin.disableTable(tableName); didDisable = true; } catch (IOException e) { - LOG.debug("Failed to disable table {}, " + "is namespace table also missing regions?", + LOG.debug("Failed to disable table {}, " + + "is namespace table also missing regions?", tableName.getNameWithNamespaceInclAsString(), e); if (enforceDisable) { - final StringBuilder errorMsgBuilder = new StringBuilder("Failed re-adding following regions: \n\t"); - report.get(tableName).forEach(r -> errorMsgBuilder.append(r.getName()).append("\t")); + final StringBuilder errorMsgBuilder = + new StringBuilder("Failed re-adding following regions: \n\t"); + report.get(tableName).forEach(r -> + errorMsgBuilder.append(r.getName()).append("\t")); throw new IOException(errorMsgBuilder.toString()); } else { LOG.debug("Continuing anyway, as no force_disable."); @@ -264,7 +267,8 @@ public List call() throws Exception { try { admin.enableTable(tableName); } catch (IOException e) { - LOG.debug("Failed enabling table {}. It might be that namespace table " + "region is also missing.\n" + LOG.debug("Failed enabling table {}. It might be that namespace table " + + "region is also missing.\n" + "After this command finishes, please make sure on this table state.", tableName.getNameWithNamespaceInclAsString(), e); } @@ -419,7 +423,7 @@ private static String getCommandUsage() { writer.println(" no matches in META, it reads regioninfo metadata file and "); writer.println(" re-creates given region in META. Regions are re-created in 'CLOSED' "); writer.println(" state at META table only, but not in Masters' cache, and are not "); - writer.println(" assigned either. To get these regions online, run hbck2 'assigns'command "); + writer.println(" assigned either. To get these regions online, run HBCK2 'assigns'command "); writer.println(" printed at the end of this command results for convenience."); writer.println(); writer.println(" NOTE: If using hbase releases older than 2.3.0, a rolling restart of "); @@ -434,7 +438,7 @@ private static String getCommandUsage() { writer.println(" namespace 'n2': "); writer.println(" $ HBCK2 " + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + " default:tbl_1 n1:tbl_2 n2 "); - writer.println(" Returns hbck2 'assigns' command with all re-inserted regions."); + writer.println(" Returns HBCK2 'assigns' command with all re-inserted regions."); writer.println(" SEE ALSO: " + REPORT_MISSING_REGIONS_IN_META); writer.println(); writer.println(" " + ASSIGNS + " [OPTIONS] ..."); From 1bd877b197f4136c19a09bb4bb1a760237770385 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 4 Sep 2019 18:05:04 +0100 Subject: [PATCH 09/10] Addressing suggestion from ankitsinghal for replacing FileSystem.get(configuration) by CommonFSUtils.getRootDirFileSystem(configuration) --- .../src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java index 09899c69b3..977ca7739e 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/FsRegionsMetaRecoverer.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -57,7 +58,7 @@ public class FsRegionsMetaRecoverer implements Closeable { public FsRegionsMetaRecoverer(Configuration configuration) throws IOException { this.config = configuration; - this.fs = FileSystem.get(configuration); + this.fs = CommonFSUtils.getRootDirFileSystem(configuration); this.conn = ConnectionFactory.createConnection(configuration); } From dd534e05de015243c13befe50d0345149c356f1d Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Fri, 6 Sep 2019 09:28:48 +0100 Subject: [PATCH 10/10] removed the attempt to disable/enable target table prior to regions readding. --- .../src/main/java/org/apache/hbase/HBCK2.java | 55 +------------------ 1 file changed, 1 insertion(+), 54 deletions(-) diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index ef85b0f747..dd88fc2e79 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -204,20 +204,6 @@ List addMissingRegionsInMeta(List regionsPath) throws IOException Pair, List> addMissingRegionsInMetaForTables(String... nameSpaceOrTable) throws IOException { - //TODO next block logic for parsing optional parameters can be moved to util method for reuse - Options options = new Options(); - Option disable = Option.builder("d").longOpt("force_disable").build(); - options.addOption(disable); - // Parse command-line. - CommandLineParser parser = new DefaultParser(); - CommandLine commandLine; - try { - commandLine = parser.parse(options, nameSpaceOrTable, false); - } catch (ParseException e) { - showErrorMessage(e.getMessage()); - return null; - } - boolean enforceDisable = commandLine.hasOption(disable.getOpt()); ExecutorService executorService = Executors.newFixedThreadPool( (nameSpaceOrTable == null || nameSpaceOrTable.length > Runtime.getRuntime().availableProcessors()) ? @@ -239,42 +225,7 @@ Pair, List> addMissingRegionsInMetaForTables(St @Override public List call() throws Exception { LOG.debug("running thread for {}", tableName.getNameWithNamespaceInclAsString()); - boolean didDisable = false; - if(admin.isTableEnabled(tableName)){ - try { - admin.disableTable(tableName); - didDisable = true; - } catch (IOException e) { - LOG.debug("Failed to disable table {}, " - + "is namespace table also missing regions?", - tableName.getNameWithNamespaceInclAsString(), e); - if (enforceDisable) { - final StringBuilder errorMsgBuilder = - new StringBuilder("Failed re-adding following regions: \n\t"); - report.get(tableName).forEach(r -> - errorMsgBuilder.append(r.getName()).append("\t")); - throw new IOException(errorMsgBuilder.toString()); - } else { - LOG.debug("Continuing anyway, as no force_disable."); - } - } - } - List reAddedRegions; - try { - reAddedRegions = addMissingRegionsInMeta(report.get(tableName)); - } finally { - if (didDisable) { - try { - admin.enableTable(tableName); - } catch (IOException e) { - LOG.debug("Failed enabling table {}. It might be that namespace table " - + "region is also missing.\n" - + "After this command finishes, please make sure on this table state.", - tableName.getNameWithNamespaceInclAsString(), e); - } - } - } - return reAddedRegions; + return addMissingRegionsInMeta(report.get(tableName)); } })); } else { @@ -429,10 +380,6 @@ private static String getCommandUsage() { writer.println(" NOTE: If using hbase releases older than 2.3.0, a rolling restart of "); writer.println(" HMasters is needed prior to executing the provided 'assigns' command. "); writer.println(); - writer.println(" WARNING: To avoid potential region overlapping problems due to ongoing "); - writer.println(" splits, this command attempts to disable given tables "); - writer.println(" (if it fails, disable is skipped) while re-inserting regions. "); - writer.println(); writer.println(" An example adding missing regions for tables 'tbl_1' on default "); writer.println(" namespace, 'tbl_2' on namespace 'n1' and for all tables from "); writer.println(" namespace 'n2': ");