diff --git a/hbase-hbck2/README.md b/hbase-hbck2/README.md
index fd74014ece..44b3174891 100644
--- a/hbase-hbck2/README.md
+++ b/hbase-hbck2/README.md
@@ -345,3 +345,47 @@ HBASE_CLASSPATH_PREFIX=./hbase-hbck2-1.0.0-SNAPSHOT.jar hbase org.apache.hbase.H
The same may happen to the _hbase:namespace_ system table. Look for the
encoded Region name of the _hbase:namespace_ Region and do similar to
what we did for _hbase:meta_.
+
+### Missing Regions in META
+
+There's been some extra-ordinary cases where table regions are removed from META table.
+Some triage on such cases revealed those were operator-induced, after execution
+attempts of the obsolete _OfflineMetaRepair_ tool. _OfflineMetaRepair_ is a well known tool
+for fixing META table related issues on HBase 1.x versions. This tool is not compatible with
+HBase 2.x or higher versions, its usage can create the type of damage described here. That
+ultimately motivated its complete removal on HBASE-22690, and it's no longer available from
+HBase 2.3.0 onwards.
+
+For recovering from missing regions in META scenarios, HBCK2 provides _addMissingRegionsInMeta_
+command. On such cases, _namespace_ table region usually will also be among those missing in
+META. This will cause master (hbase) initialization to fail after some initial period. A META
+scan can help confirm if this condition is being reached:
+
+```
+echo "scan 'hbase:meta', {COLUMN=>'info:regioninfo'}" | hbase shell | grep namespace
+```
+
+If above doesn't return anything, it means _namespace_ region info is missing in META. The same
+command can be adjusted for checking other tables regions as well.
+
+HBCK2 _addMissingRegionsInMeta_ reads region metadata info available on the FS region dirs, in
+order to re-create regions in META. It can check for specific tables/namespaces, or all tables
+from all namespaces. An example adding missing regions for tables 'tbl_1' on default namespace,
+'tbl_2' on namespace 'n1' and for all tables from namespace 'n2':
+
+```
+$ HBCK2 addMissingRegionsInMeta default:tbl_1 n1:tbl_2 n2
+```
+
+As it operates independently from Master, once it finishes successfully, additional steps are
+required to actually have the re-added regions assigned. These are listed below:
+
+1. _addMissingRegionsInMeta_ outputs an _assigns_ command with all regions that got re-added. This
+command needs to be executed later, so copy and save it for convenience.
+
+2. After _addMissingRegionsInMeta_ finished successfully and output has been saved, restart all
+running HBase Masters.
+
+3. Once Master's are restarted and META is already online (check if Web UI is accessible), run
+_assigns_ command from _addMissingRegionsInMeta_ output saved per instructions from #1.
+
diff --git a/hbase-hbck2/pom.xml b/hbase-hbck2/pom.xml
index d6f2b7d382..9a74d247ce 100644
--- a/hbase-hbck2/pom.xml
+++ b/hbase-hbck2/pom.xml
@@ -181,6 +181,12 @@
${hbase.version}
test
+
+ org.mockito
+ mockito-core
+ 2.1.0
+ test
+
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
index 14323125e9..6beaf82d12 100644
--- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java
@@ -26,6 +26,7 @@
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ClusterMetrics;
import org.apache.hadoop.hbase.CompareOperator;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -45,9 +46,11 @@
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hbase.hbck2.meta.MetaFixer;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -59,9 +62,16 @@
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.stream.Collectors;
/**
@@ -87,6 +97,10 @@ public class HBCK2 extends Configured implements Tool {
private static final String BYPASS = "bypass";
private static final String VERSION = "version";
private static final String SET_REGION_STATE = "setRegionState";
+ private static final String ADD_MISSING_REGIONS_IN_META_FOR_TABLES =
+ "addMissingRegionsInMetaForTables";
+ private static final String ADD_MISSING_REGIONS_IN_META = "addMissingRegionsInMeta";
+ private static final String REPORT_MISSING_REGIONS_IN_META = "reportMissingRegionsInMeta";
private Configuration conf;
private static final String TWO_POINT_ONE = "2.1.0";
private static final String MININUM_VERSION = "2.0.3";
@@ -164,6 +178,102 @@ int setRegionState(String region, RegionState.State newState)
return EXIT_FAILURE;
}
+ Map> reportTablesWithMissingRegionsInMeta(String... nameSpaceOrTable)
+ throws Exception {
+ Map> report;
+ try(final MetaFixer metaFixer = new MetaFixer(this.conf)){
+ List names = nameSpaceOrTable != null ? Arrays.asList(nameSpaceOrTable) : null;
+ report = metaFixer.reportTablesMissingRegions(names);
+ } catch (Exception e) {
+ LOG.error("Error reporting missing regions: ", e);
+ throw e;
+ }
+ if(LOG.isDebugEnabled()) {
+ LOG.debug(formatMissingRegionsInMetaReport(report));
+ }
+ return report;
+ }
+
+ List addMissingRegionsInMeta(List regionsPath) throws IOException {
+ List reAddedRegionsEncodedNames = new ArrayList<>();
+ try(final MetaFixer metaFixer = new MetaFixer(this.conf)){
+ for(Path regionPath : regionsPath){
+ metaFixer.putRegionInfoFromHdfsInMeta(regionPath);
+ reAddedRegionsEncodedNames.add(regionPath.getName());
+ }
+ }
+ return reAddedRegionsEncodedNames;
+ }
+
+ Pair, List> addMissingRegionsInMetaForTables(String...
+ nameSpaceOrTable) {
+ ExecutorService executorService = Executors.newFixedThreadPool(
+ (nameSpaceOrTable == null ||
+ nameSpaceOrTable.length > Runtime.getRuntime().availableProcessors()) ?
+ Runtime.getRuntime().availableProcessors() :
+ nameSpaceOrTable.length);
+ List>> futures = new ArrayList<>( nameSpaceOrTable == null ? 1 :
+ nameSpaceOrTable.length);
+ final List readdedRegionNames = new ArrayList<>();
+ List executionErrors = new ArrayList<>();
+ try {
+ //reducing number of retries in case disable fails due to namespace table region also missing
+ this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
+ try(Connection conn = ConnectionFactory.createConnection(this.conf);
+ final Admin admin = conn.getAdmin()) {
+ Map> report = reportTablesWithMissingRegionsInMeta(nameSpaceOrTable);
+ for (TableName tableName : report.keySet()) {
+ if(admin.tableExists(tableName)) {
+ futures.add(executorService.submit(new Callable>() {
+ @Override
+ public List call() throws Exception {
+ LOG.debug("running thread for {}", tableName.getNameWithNamespaceInclAsString());
+ try {
+ admin.disableTable(tableName);
+ } catch (IOException e) {
+ LOG.debug("Failed to disable table {}, "
+ + "is namespace table also missing regions? Continue anyway...",
+ tableName.getNameWithNamespaceInclAsString(), e);
+ }
+ List reAddedRegions = addMissingRegionsInMeta(report.get(tableName));
+ try {
+ admin.enableTable(tableName);
+ } catch (IOException e) {
+ LOG.debug("Failed enabling table {}. It might be that namespace table "
+ + "region is also missing.\n"
+ + "After this command finishes, please make sure on this table state.",
+ tableName.getNameWithNamespaceInclAsString(), e);
+ }
+ return reAddedRegions;
+ }
+ }));
+ } else {
+ LOG.warn("Table {} does not exist! Skipping...",
+ tableName.getNameWithNamespaceInclAsString());
+ }
+ }
+ for(Future> f : futures){
+ try {
+ readdedRegionNames.addAll(f.get());
+ } catch (ExecutionException e){
+ //we want to allow potential running threads to finish, so we collect execution
+ //errors and show those later
+ LOG.debug("Caught execution error: ", e);
+ executionErrors.add(e);
+ }
+ }
+ }
+ } catch (Exception ie){
+ LOG.error("ERROR executing thread: ", ie);
+ } finally {
+ executorService.shutdown();
+ }
+ Pair, List> result = new Pair<>();
+ result.setFirst(readdedRegionNames);
+ result.setSecond(executionErrors);
+ return result;
+ }
+
List assigns(String [] args) throws IOException {
Options options = new Options();
Option override = Option.builder("o").longOpt("override").build();
@@ -334,6 +444,58 @@ private static final String getCommandUsage() {
writer.println(" Returns \"0\" SUCCESS code if it informed region state is changed, "
+ "\"1\" FAIL code otherwise.");
writer.println();
+ writer.println(" " + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + " ...");
+ writer.println(" To be used in scenarios where some regions may be missing in META,");
+ writer.println(" but there's still a valid 'regioninfo metadata file on HDFS. ");
+ writer.println(" This is a lighter version of 'OfflineMetaRepair tool commonly used for ");
+ writer.println(" similar issues on 1.x release line. ");
+ writer.println(" This command needs META to be online. For each table name passed as");
+ writer.println(" parameter, it performs a diff between regions available in META, ");
+ writer.println(" against existing regions dirs on HDFS. Then, for region dirs with ");
+ writer.println(" no matches in META, it reads regioninfo metadata file and ");
+ writer.println(" re-creates given region in META. Regions are re-created in 'CLOSED' ");
+ writer.println(" state at META table only, but not in Masters' cache, and are not ");
+ writer.println(" assigned either. A rolling Masters restart, followed by a ");
+ writer.println(" hbck2 'assigns' command with all re-inserted regions is required. ");
+ writer.println(" This hbck2 'assigns' command is printed out after this command ");
+ writer.println(" completes for users subsequent convenience.");
+ writer.println(" WARNING: To avoid potential region overlapping problems due to ongoing ");
+ writer.println(" splits, this command attempts to disable given tables ");
+ writer.println(" (if it fails, disable is skipped) while re-inserting regions. ");
+ writer.println(" An example adding missing regions for tables 'tbl_1' on default ");
+ writer.println(" namespace, 'tbl_2' on namespace 'n1' and for all tables from ");
+ writer.println(" namespace 'n2': ");
+ writer.println(" $ HBCK2 addMissingRegionsInMeta default:tbl_1 n1:tbl_2 n2 ");
+ writer.println(" Returns hbck2 'assigns' command with all re-inserted regions.");
+ writer.println();
+ writer.println(" " + REPORT_MISSING_REGIONS_IN_META + " ...");
+ writer.println(" To be used in scenarios where some regions may be missing in META,");
+ writer.println(" but there's still a valid 'regioninfo metadata file on HDFS. ");
+ writer.println(" This is a checking only method, designed for reporting purposes and");
+ writer.println(" doesn't perform any fixes, providing a view of which regions (if any) ");
+ writer.println(" would get re-added to meta, grouped by respective table/namespace. ");
+ writer.println(" To effectively re-add regions in meta, "
+ + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + " should be executed. ");
+ writer.println(" This command needs META to be online. For each namespace/table passed");
+ writer.println(" as parameter, it performs a diff between regions available in META, ");
+ writer.println(" against existing regions dirs on HDFS. Region dirs with no matches");
+ writer.println(" are printed grouped under its related table name. Tables with no");
+ writer.println(" missing regions will show a 'no missing regions' message. If no");
+ writer.println(" namespace or table is specified, it will verify all existing regions.");
+ writer.println(" It accepts a combination of multiple namespace and tables. Table names");
+ writer.println(" should include the namespace portion, even for tables in the default");
+ writer.println(" namespace, otherwise it will assume as a namespace value.");
+ writer.println(" An example triggering missing regions report for tables 'table_1'");
+ writer.println(" and 'table_2', under default namespace:");
+ writer.println(" $ HBCK2 reportMissingRegionsInMeta default:table_1 default:table_2");
+ writer.println(" An example triggering missing regions report for table 'table_1'");
+ writer.println(" under default namespace, and for all tables from namespace 'ns1':");
+ writer.println(" $ HBCK2 reportMissingRegionsInMeta default:table_1 ns1");
+ writer.println(" Returns list of missing regions for each table passed as parameter, or ");
+ writer.println(" for each table on namespaces specified as parameter.");
+ writer.println();
writer.close();
return sw.toString();
@@ -440,51 +602,68 @@ public int run(String[] args) throws IOException {
String[] commands = commandLine.getArgs();
String command = commands[0];
switch (command) {
- case SET_TABLE_STATE:
- if (commands.length < 3) {
- usage(options, command + " takes tablename and state arguments: e.g. user ENABLED");
- return EXIT_FAILURE;
- }
- System.out.println(setTableState(TableName.valueOf(commands[1]),
- TableState.State.valueOf(commands[2])));
- break;
-
- case ASSIGNS:
- if (commands.length < 2) {
- usage(options, command + " takes one or more encoded region names");
- return EXIT_FAILURE;
- }
- System.out.println(assigns(purgeFirst(commands)));
- break;
+ case SET_TABLE_STATE:
+ if (commands.length < 3) {
+ usage(options, command + " takes tablename and state arguments: e.g. user ENABLED");
+ return EXIT_FAILURE;
+ }
+ System.out.println(setTableState(TableName.valueOf(commands[1]), TableState.State.valueOf(commands[2])));
+ break;
- case BYPASS:
- if (commands.length < 2) {
- usage(options, command + " takes one or more pids");
- return EXIT_FAILURE;
- }
- List bs = bypass(purgeFirst(commands));
- if (bs == null) {
- // Something went wrong w/ the parse and command didn't run.
- return EXIT_FAILURE;
- }
- System.out.println(toString(bs));
- break;
+ case ASSIGNS:
+ if (commands.length < 2) {
+ usage(options, command + " takes one or more encoded region names");
+ return EXIT_FAILURE;
+ }
+ System.out.println(assigns(purgeFirst(commands)));
+ break;
- case UNASSIGNS:
- if (commands.length < 2) {
- usage(options, command + " takes one or more encoded region names");
- return EXIT_FAILURE;
- }
- System.out.println(toString(unassigns(purgeFirst(commands))));
- break;
-
- case SET_REGION_STATE:
- if(commands.length < 3){
- usage(options, command + " takes region encoded name and state arguments: e.g. "
- + "35f30b0ce922c34bf5c284eff33ba8b3 CLOSING");
- return EXIT_FAILURE;
- }
- return setRegionState(commands[1], RegionState.State.valueOf(commands[2]));
+ case BYPASS:
+ if (commands.length < 2) {
+ usage(options, command + " takes one or more pids");
+ return EXIT_FAILURE;
+ }
+ List bs = bypass(purgeFirst(commands));
+ if (bs == null) {
+ // Something went wrong w/ the parse and command didn't run.
+ return EXIT_FAILURE;
+ }
+ System.out.println(toString(bs));
+ break;
+
+ case UNASSIGNS:
+ if (commands.length < 2) {
+ usage(options, command + " takes one or more encoded region names");
+ return EXIT_FAILURE;
+ }
+ System.out.println(toString(unassigns(purgeFirst(commands))));
+ break;
+
+ case SET_REGION_STATE:
+ if (commands.length < 3) {
+ usage(options, command + " takes region encoded name and state arguments: e.g. "
+ + "35f30b0ce922c34bf5c284eff33ba8b3 CLOSING");
+ return EXIT_FAILURE;
+ }
+ return setRegionState(commands[1], RegionState.State.valueOf(commands[2]));
+ case ADD_MISSING_REGIONS_IN_META_FOR_TABLES:
+ if(commands.length < 2){
+ usage(options, command + " takes one or more table names.");
+ return EXIT_FAILURE;
+ }
+ Pair, List> result =
+ addMissingRegionsInMetaForTables(purgeFirst(commands));
+ System.out.println(formatReAddedRegionsMessage(result.getFirst(),result.getSecond()));
+ break;
+ case REPORT_MISSING_REGIONS_IN_META:
+ try {
+ Map> report =
+ reportTablesWithMissingRegionsInMeta(purgeFirst(commands));
+ System.out.println(formatMissingRegionsInMetaReport(report));
+ } catch (Exception e) {
+ return EXIT_FAILURE;
+ }
+ break;
default:
usage(options, "Unsupported command: " + command);
return EXIT_FAILURE;
@@ -496,6 +675,51 @@ private static String toString(List things) {
return things.stream().map(i -> i.toString()).collect(Collectors.joining(", "));
}
+ private String formatMissingRegionsInMetaReport(Map> report) {
+ final StringBuilder builder = new StringBuilder();
+ builder.append("Missing Regions for each table:\n\t");
+ report.keySet().stream().forEach(table -> {
+ builder.append(table);
+ if (!report.get(table).isEmpty()){
+ builder.append("->\n\t\t");
+ report.get(table).stream().forEach(region -> builder.append(region.getName())
+ .append(" "));
+ } else {
+ builder.append(" -> No missing regions");
+ }
+ builder.append("\n\t");
+ });
+ return builder.toString();
+ }
+
+ private String formatReAddedRegionsMessage(List readdedRegionNames,
+ List executionErrors) {
+ final StringBuilder finalText = new StringBuilder();
+ finalText.append("Regions re-added into Meta: ").append(readdedRegionNames.size());
+ if(!readdedRegionNames.isEmpty()){
+ finalText.append("\n")
+ .append("WARNING: \n\t")
+ .append(readdedRegionNames.size()).append(" regions were added ")
+ .append("to META, but these are not yet on Masters cache. \n")
+ .append("You need to restart Masters, then run hbck2 'assigns' command below:\n\t\t")
+ .append(buildHbck2AssignsCommand(readdedRegionNames));
+ }
+ if(!executionErrors.isEmpty()){
+ finalText.append("\n")
+ .append("ERROR: \n\t")
+ .append("There were following errors on at least one table thread:\n");
+ executionErrors.stream().forEach(e -> finalText.append(e.getMessage()).append("\n"));
+ }
+ return finalText.toString();
+ }
+
+ private String buildHbck2AssignsCommand(List regions) {
+ final StringBuilder builder = new StringBuilder();
+ builder.append("assigns ");
+ regions.forEach(region -> builder.append(region).append(" "));
+ return builder.toString();
+ }
+
/**
* @return A new array with first element dropped.
*/
diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java b/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java
new file mode 100644
index 0000000000..9d30291117
--- /dev/null
+++ b/hbase-hbck2/src/main/java/org/apache/hbase/hbck2/meta/MetaFixer.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hbase.hbck2.meta;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MetaTableAccessor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * This class implements the inner works required for check and recover regions that wrongly
+ * went missing in META. It assumes HDFS state as the source of truth, in other words,
+ * methods provided here consider meta information found on HDFS region dirs as the valid ones.
+ */
+public class MetaFixer implements Closeable {
+ private static final String TABLE_DESC_FILE = ".tabledesc";
+ private static final Logger LOG = LogManager.getLogger(MetaFixer.class);
+ private final FileSystem fs;
+ private final Connection conn;
+ private final Configuration config;
+
+ public MetaFixer(Configuration configuration) throws IOException {
+ this.config = configuration;
+ this.fs = FileSystem.get(configuration);
+ this.conn = ConnectionFactory.createConnection(configuration);
+ }
+
+ /*Initially defined for test only purposes */
+ MetaFixer(Configuration configuration, Connection connection, FileSystem fileSystem){
+ this.config = configuration;
+ this.conn = connection;
+ this.fs = fileSystem;
+ }
+
+ private FileStatus[] getTableRegionsDirs(String table) throws Exception {
+ String hbaseRoot = this.config.get("hbase.rootdir");
+ String tableRootDir = hbaseRoot + "/" + HConstants.BASE_NAMESPACE_DIR + "/" +
+ TableName.valueOf(table).getNameWithNamespaceInclAsString()
+ .replaceAll(":", "/");
+ return fs.listStatus(new Path(tableRootDir));
+ }
+
+ public Map> reportTablesMissingRegions(final List namespacesOrTables)
+ throws IOException {
+ final Map> result = new HashMap<>();
+ List tableNames = MetaTableAccessor.getTableStates(this.conn).keySet().stream()
+ .filter(tableName -> {
+ if(namespacesOrTables==null || namespacesOrTables.isEmpty()){
+ return true;
+ } else {
+ Optional findings = namespacesOrTables.stream().filter(
+ name -> (name.indexOf(":") > 0) ?
+ tableName.getNameWithNamespaceInclAsString().equals(name) :
+ tableName.getNamespaceAsString().equals(name)).findFirst();
+ return findings.isPresent();
+ }
+ }).collect(Collectors.toList());
+ tableNames.stream().forEach(tableName -> {
+ try {
+ result.put(tableName,
+ findMissingRegionsInMETA(tableName.getNameWithNamespaceInclAsString()));
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ });
+ return result;
+ }
+
+ List findMissingRegionsInMETA(String table) throws Exception {
+ final List missingRegions = new ArrayList<>();
+ final FileStatus[] regionsDirs = getTableRegionsDirs(table);
+ TableName tableName = TableName.valueOf(table);
+ List regionInfos = MetaTableAccessor.
+ getTableRegions(this.conn, tableName, false);
+ for(final FileStatus regionDir : regionsDirs){
+ if(!regionDir.getPath().getName().equals(TABLE_DESC_FILE) &&
+ !regionDir.getPath().getName().equals(HConstants.HBASE_TEMP_DIRECTORY)) {
+ boolean foundInMeta = regionInfos.stream()
+ .anyMatch(info -> info.getEncodedName().equals(regionDir.getPath().getName()));
+ if (!foundInMeta) {
+ LOG.debug(regionDir + "is not in META.");
+ missingRegions.add(regionDir.getPath());
+ }
+ }
+ }
+ return missingRegions;
+ }
+
+ public void putRegionInfoFromHdfsInMeta(Path region) throws IOException {
+ RegionInfo info = HRegionFileSystem.loadRegionInfoFileContent(fs, region);
+ MetaTableAccessor.addRegionToMeta(conn, info);
+ }
+
+ @Override public void close() throws IOException {
+ this.conn.close();
+ }
+}
diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
index 39c9c5bbd1..4d8cf548b7 100644
--- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
+++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java
@@ -18,11 +18,12 @@
package org.apache.hbase;
import junit.framework.TestCase;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Result;
@@ -37,12 +38,15 @@
import org.junit.Test;
import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
/**
* Tests commands. For command-line parsing, see adjacent test.
@@ -177,6 +181,114 @@ public void testSetRegionStateInvalidState() throws IOException {
}
}
+ @Test
+ public void testAddMissingRegionsInMetaAllRegionsMissing() throws Exception {
+ this.testAddMissingRegionsInMetaForTables(5,5);
+ }
+
+ @Test
+ public void testAddMissingRegionsInMetaTwoMissingOnly() throws Exception {
+ this.testAddMissingRegionsInMetaForTables(2,5);
+ }
+
+ @Test
+ public void testReportMissingRegionsInMetaAllNsTbls() throws Exception {
+ this.testReportMissingRegionsInMeta(5, 5,null);
+ }
+
+ @Test
+ public void testReportMissingRegionsInMetaSpecificTbl() throws Exception {
+ this.testReportMissingRegionsInMeta(5, 5,
+ TABLE_NAME.getNameWithNamespaceInclAsString());
+ }
+
+ @Test
+ public void testReportMissingRegionsInMetaSpecificTblAndNsTbl() throws Exception {
+ this.testReportMissingRegionsInMeta(5, 5,
+ TABLE_NAME.getNameWithNamespaceInclAsString(), "hbase:namespace");
+ }
+
+ @Test
+ public void testReportMissingRegionsInMetaSpecificTblAndNsTblAlsoMissing() throws Exception {
+ List regions = MetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), TableName.valueOf("hbase:namespace"));
+ MetaTableAccessor.deleteRegions(TEST_UTIL.getConnection(),
+ regions.subList(0,1));
+ this.testReportMissingRegionsInMeta(5, 6,
+ TABLE_NAME.getNameWithNamespaceInclAsString(), "hbase:namespace");
+ }
+
+ @Test
+ public void testFormatReportMissingRegionsInMetaNoMissing() throws IOException {
+ final String expectedResult = "Missing Regions for each table:\n"
+ + "\tTestHBCK2 -> No missing regions\n\thbase:namespace -> No missing regions\n\t\n";
+ testFormatMissingRegionsInMetaReport(expectedResult);
+ }
+
+ @Test
+ public void testFormatReportMissingInMetaOneMissing() throws IOException {
+ List regions = MetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), TABLE_NAME);
+ MetaTableAccessor.deleteRegions(TEST_UTIL.getConnection(), regions.subList(0,1));
+ final String expectedResult = "Missing Regions for each table:\n"
+ + "\tTestHBCK2->\n\t\t" + regions.get(0).getEncodedName()
+ + " \n\thbase:namespace -> No missing regions\n\t\n";
+ testFormatMissingRegionsInMetaReport(expectedResult);
+ HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
+ hbck.addMissingRegionsInMetaForTables(null);
+ }
+
+ private void testFormatMissingRegionsInMetaReport(String expectedResultMessage)
+ throws IOException {
+ HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
+ final StringBuilder builder = new StringBuilder();
+ PrintStream originalOS = System.out;
+ OutputStream testOS = new OutputStream() {
+ @Override public void write(int b) throws IOException {
+ builder.append((char)b);
+ }
+ };
+ System.setOut(new PrintStream(testOS));
+
+ hbck.run(new String[]{"reportMissingRegionsInMeta"});
+ assertTrue(builder.toString().contains(expectedResultMessage));
+ System.setOut(originalOS);
+ }
+
+ private void testAddMissingRegionsInMetaForTables(int missingRegions, int totalRegions)
+ throws Exception {
+ HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
+ List regions = MetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), TABLE_NAME);
+ MetaTableAccessor.deleteRegions(TEST_UTIL.getConnection(), regions.subList(0,missingRegions));
+ int remaining = totalRegions - missingRegions;
+ assertEquals("Table should had " + remaining + " regions in META.", remaining,
+ MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLE_NAME));
+ assertEquals(missingRegions,hbck.addMissingRegionsInMetaForTables("default:"
+ + TABLE_NAME.getNameAsString()).getFirst().size());
+ assertEquals("Table regions should had been re-added in META.", totalRegions,
+ MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), TABLE_NAME));
+ //compare the added regions to make sure those are the same
+ List newRegions = MetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), TABLE_NAME);
+ assertEquals("All re-added regions should be the same", regions, newRegions);
+ }
+
+ private void testReportMissingRegionsInMeta(int missingRegionsInTestTbl,
+ int expectedTotalMissingRegions, String... namespaceOrTable) throws Exception {
+ List regions = MetaTableAccessor
+ .getTableRegions(TEST_UTIL.getConnection(), TABLE_NAME);
+ MetaTableAccessor.deleteRegions(TEST_UTIL.getConnection(),
+ regions.subList(0,missingRegionsInTestTbl));
+ HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
+ final Map> report =
+ hbck.reportTablesWithMissingRegionsInMeta(namespaceOrTable);
+ long resultingMissingRegions = report.keySet().stream().mapToLong( nsTbl ->
+ report.get(nsTbl).size()).sum();
+ assertEquals(expectedTotalMissingRegions, resultingMissingRegions);
+ hbck.addMissingRegionsInMetaForTables(null);
+ }
+
@Test (expected = IllegalArgumentException.class)
public void testSetRegionStateInvalidRegionAndInvalidState() throws IOException {
HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration());
diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java b/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java
new file mode 100644
index 0000000000..345c6b0f9e
--- /dev/null
+++ b/hbase-hbck2/src/test/java/org/apache/hbase/hbck2/meta/TestMetaFixer.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hbase.hbck2.meta;
+
+import static org.junit.Assert.assertEquals;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableState;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class TestMetaFixer {
+
+ private Connection mockedConnection;
+ private FileSystem mockedFileSystem;
+ private Table mockedTable;
+ private MetaFixer fixer;
+ private String testTblDir;
+
+ @Before
+ public void setup() throws Exception {
+ this.mockedConnection = Mockito.mock(Connection.class);
+ this.mockedFileSystem = Mockito.mock(FileSystem.class);
+ this.mockedTable = Mockito.mock(Table.class);
+ Configuration config = HBaseConfiguration.create();
+ Mockito.when(this.mockedConnection.getConfiguration()).thenReturn(config);
+ Mockito.when(this.mockedConnection.getTable(TableName.META_TABLE_NAME)).thenReturn(mockedTable);
+ this.testTblDir = config.get("hbase.rootdir") + "/data/default/test-tbl";
+ this.fixer = new MetaFixer(config, mockedConnection, mockedFileSystem);
+ }
+
+ private RegionInfo createRegionInfo(String table){
+ long regionTS = System.currentTimeMillis();
+ RegionInfo info = RegionInfoBuilder.newBuilder(TableName.valueOf(table))
+ .setRegionId(regionTS)
+ .build();
+ return info;
+ }
+
+ private Cell createCellForRegionInfo(RegionInfo info){
+ byte[] regionInfoValue = ProtobufUtil.prependPBMagic(ProtobufUtil.toRegionInfo(info)
+ .toByteArray());
+ Cell cell = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
+ .setRow(info.getRegionName())
+ .setFamily(Bytes.toBytes("info"))
+ .setQualifier(Bytes.toBytes("regioninfo"))
+ .setType(Cell.Type.Put)
+ .setValue(regionInfoValue)
+ .build();
+ return cell;
+ }
+
+ private Cell createCellForTableState(TableName tableName){
+ Cell cell = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
+ .setRow(tableName.getName())
+ .setFamily(Bytes.toBytes("table"))
+ .setQualifier(Bytes.toBytes("state"))
+ .setType(Cell.Type.Put)
+ .setValue(HBaseProtos.TableState.newBuilder()
+ .setState(TableState.State.ENABLED.convert()).build().toByteArray())
+ .build();
+ return cell;
+ }
+
+ @Test
+ public void testFindMissingRegionsInMETANoMissing() throws Exception {
+ ResultScanner mockedRS = Mockito.mock(ResultScanner.class);
+ Mockito.when(this.mockedTable.getScanner(Mockito.any(Scan.class))).thenReturn(mockedRS);
+ RegionInfo info = createRegionInfo("test-tbl");
+ List cells = new ArrayList();
+ cells.add(createCellForRegionInfo(info));
+ Result result = Result.create(cells);
+ Mockito.when(mockedRS.next()).thenReturn(result,null);
+ FileStatus status = new FileStatus();
+ status.setPath(new Path(this.testTblDir + "/" + info.getEncodedName()));
+ Mockito.when(mockedFileSystem.listStatus(new Path(this.testTblDir)))
+ .thenReturn(new FileStatus[]{status});
+ assertEquals("Should had returned 0 missing regions",
+ 0, fixer.findMissingRegionsInMETA("test-tbl").size());
+ }
+
+ @Test
+ public void testFindMissingRegionsInMETAOneMissing() throws Exception {
+ ResultScanner mockedRS = Mockito.mock(ResultScanner.class);
+ Mockito.when(this.mockedTable.getScanner(Mockito.any(Scan.class))).thenReturn(mockedRS);
+ List cells = new ArrayList();
+ Result result = Result.create(cells);
+ Mockito.when(mockedRS.next()).thenReturn(result,null);
+ FileStatus status = new FileStatus();
+ Path p = new Path(this.testTblDir+ "/182182182121");
+ status.setPath(p);
+ Mockito.when(mockedFileSystem.listStatus(new Path(this.testTblDir)))
+ .thenReturn(new FileStatus[]{status});
+ List missingRegions = fixer.findMissingRegionsInMETA("test-tbl");
+ assertEquals("Should had returned 1 missing region",
+ 1, missingRegions.size());
+ assertEquals(p,missingRegions.get(0));
+ }
+
+ @Test
+ public void testPutRegionInfoFromHdfsInMeta() throws Exception {
+ RegionInfo info = this.createRegionInfo("test-tbl");
+ Path regionPath = new Path("/hbase/data/default/test-tbl/" + info.getEncodedName());
+ FSDataInputStream fis = new FSDataInputStream(new TestInputStreamSeekable(info));
+ Mockito.when(this.mockedFileSystem.open(new Path(regionPath, ".regioninfo")))
+ .thenReturn(fis);
+ fixer.putRegionInfoFromHdfsInMeta(regionPath);
+ Mockito.verify(this.mockedConnection).getTable(TableName.META_TABLE_NAME);
+ Mockito.verify(this.mockedTable).put(Mockito.any(Put.class));
+ }
+
+ @Test
+ public void testReportTablesMissingRegionsOneMissing() throws Exception {
+ ResultScanner mockedRS = Mockito.mock(ResultScanner.class);
+ Mockito.when(this.mockedTable.getScanner(Mockito.any(Scan.class))).thenReturn(mockedRS);
+ List| cells = new ArrayList();
+ cells.add(createCellForTableState(TableName.valueOf("test-tbl")));
+ Result result = Result.create(cells);
+ Mockito.when(mockedRS.next()).thenReturn(result,null);
+ FileStatus status = new FileStatus();
+ Path p = new Path(this.testTblDir+ "/182182182121");
+ status.setPath(p);
+ Mockito.when(mockedFileSystem.listStatus(new Path(this.testTblDir)))
+ .thenReturn(new FileStatus[]{status});
+ Mockito.when(this.mockedConnection.getTable(TableName.META_TABLE_NAME))
+ .thenReturn(this.mockedTable);
+ Map> report = fixer.reportTablesMissingRegions(null);
+ assertEquals("Should had returned 1 missing region",
+ 1,report.size());
+ }
+
+ private class TestInputStreamSeekable extends FSInputStream {
+
+ private ByteArrayInputStream in;
+ private long length;
+
+ private TestInputStreamSeekable(RegionInfo info) throws Exception {
+ byte[] bytes = RegionInfo.toDelimitedByteArray(info);
+ this.length = bytes.length;
+ this.in = new ByteArrayInputStream(bytes);
+ }
+
+ @Override
+ public void seek(long l) throws IOException {
+ this.in.skip(l);
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return this.length - in.available();
+ }
+
+ @Override
+ public boolean seekToNewSource(long l) throws IOException {
+ this.in.skip(l);
+ return true;
+ }
+
+ @Override
+ public int read() throws IOException {
+ return in.read();
+ }
+ }
+
+}
+
| | |