diff --git a/example/simple-hbase-demo/src/main/java/com/oceanbase/example/SimpleHBaseClientDemo.java b/example/simple-hbase-demo/src/main/java/com/oceanbase/example/SimpleHBaseClientDemo.java index 0ab3b588..e0ff821c 100644 --- a/example/simple-hbase-demo/src/main/java/com/oceanbase/example/SimpleHBaseClientDemo.java +++ b/example/simple-hbase-demo/src/main/java/com/oceanbase/example/SimpleHBaseClientDemo.java @@ -19,6 +19,8 @@ import com.alipay.oceanbase.hbase.OHTableClient; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -49,7 +51,7 @@ public static void simpleTest() throws Exception { byte[] rowKey = toBytes("rowKey1"); byte[] column = toBytes("column1"); Put put = new Put(rowKey); - put.add(family, column, System.currentTimeMillis(), toBytes("value1")); + put.addColumn(family, column, System.currentTimeMillis(), toBytes("value1")); hTable.put(put); // 3. get data like hbase @@ -58,7 +60,7 @@ public static void simpleTest() throws Exception { Result r = hTable.get(get); if (!r.isEmpty()) { Cell cell = r.rawCells()[0]; - System.out.printf("column1: " + CellUtil.cloneQualifier(r)); + System.out.printf("column1: " + CellUtil.cloneQualifier(cell)); } // 4. close diff --git a/pom.xml b/pom.xml index 3b8fbf94..3e2e7667 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ ${project.encoding} UTF-8 1.7.21 - 2.0.0 + 2.0.1-SNAPSHOT @@ -167,6 +167,10 @@ jersey-json com.sun.jersey + + jackson-databind + com.fasterxml.jackson.core + diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index f66dcd8c..106058eb 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -220,8 +220,7 @@ public OHTable(Configuration configuration, String tableName) throws IOException DEFAULT_HBASE_HTABLE_THREAD_KEEP_ALIVE_TIME); this.executePool = createDefaultThreadPoolExecutor(1, maxThreads, keepAliveTime); OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); - int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + int numRetries = ohConnectionConf.getNumRetries(); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); @@ -273,8 +272,7 @@ public OHTable(Configuration configuration, final byte[] tableName, this.executePool = executePool; this.cleanupPoolOnClose = false; OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); - int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + int numRetries = ohConnectionConf.getNumRetries(); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); @@ -345,8 +343,7 @@ public OHTable(TableName tableName, Connection connection, DEFAULT_HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK); this.writeBufferSize = connectionConfig.getWriteBufferSize(); this.tableName = tableName.getName(); - int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + int numRetries = connectionConfig.getNumRetries(); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, connectionConfig)); this.obTableClient.setRpcExecuteTimeout(rpcTimeout); @@ -389,8 +386,7 @@ public OHTable(Connection connection, ObTableBuilderBase builder, this.putWriteBufferCheck = this.configuration.getInt(HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK, DEFAULT_HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK); this.writeBufferSize = connectionConfig.getWriteBufferSize(); - int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + int numRetries = connectionConfig.getNumRetries(); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, connectionConfig)); this.obTableClient.setRpcExecuteTimeout(rpcTimeout); @@ -462,7 +458,7 @@ private void finishSetUp() { WRITE_BUFFER_SIZE_DEFAULT); } - private OHConnectionConfiguration setUserDefinedNamespace(String tableNameString, + public static OHConnectionConfiguration setUserDefinedNamespace(String tableNameString, OHConnectionConfiguration ohConnectionConf) throws IllegalArgumentException { if (tableNameString.indexOf(':') != -1) { @@ -503,13 +499,15 @@ public Configuration getConfiguration() { } @Override - public HTableDescriptor getTableDescriptor() { - throw new FeatureNotSupportedException("not supported yet."); + public HTableDescriptor getTableDescriptor() throws IOException { + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableNameString, obTableClient); + return executor.getTableDescriptor(); } @Override public TableDescriptor getDescriptor() throws IOException { - throw new FeatureNotSupportedException("not supported yet."); + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableNameString, obTableClient); + return executor.getTableDescriptor(); } /** @@ -1950,6 +1948,8 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE)); obTableQuery.setObKVParams(buildOBKVParams(scan)); obTableQuery.setScanRangeColumns("K", "Q", "T"); + byte[] hotOnly = scan.getAttribute(HBASE_HTABLE_QUERY_HOT_ONLY); + obTableQuery.setHotOnly(hotOnly != null && Arrays.equals(hotOnly, "true".getBytes())); return obTableQuery; } @@ -1967,6 +1967,8 @@ private ObTableQuery buildObTableQuery(final Get get, Collection columnQ } obTableQuery.setObKVParams(buildOBKVParams(get)); obTableQuery.setScanRangeColumns("K", "Q", "T"); + byte[] hotOnly = get.getAttribute(HBASE_HTABLE_QUERY_HOT_ONLY); + obTableQuery.setHotOnly(hotOnly != null && Arrays.equals(hotOnly, "true".getBytes())); return obTableQuery; } diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java b/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java index 770d523c..8a2c2f89 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java @@ -223,7 +223,8 @@ public HTableDescriptor getTableDescriptor() throws IOException { @Override public TableDescriptor getDescriptor() throws IOException { - return null; + checkStatus(); + return ohTable.getDescriptor(); } @Override diff --git a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java index 19311744..deb94f75 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java +++ b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java @@ -23,17 +23,17 @@ public final class OHConstants { /** - * ocenbase hbase root server http url + * oceanbase hbase root server http url */ public static final String HBASE_OCEANBASE_PARAM_URL = "hbase.oceanbase.paramURL"; /** - * ocenbase hbase connect server username + * oceanbase hbase connect server username */ public static final String HBASE_OCEANBASE_FULL_USER_NAME = "hbase.oceanbase.fullUserName"; /** - * ocenbase hbase connect server password + * oceanbase hbase connect server password */ public static final String HBASE_OCEANBASE_PASSWORD = "hbase.oceanbase.password"; @@ -48,39 +48,39 @@ public final class OHConstants { public static final String HBASE_OCEANBASE_SYS_PASSWORD = "hbase.oceanbase.sysPassword"; /** - * ocenbase hbase connect server password + * oceanbase hbase connect server password */ public static final String HBASE_OCEANBASE_BATCH_EXECUTOR = "hbase.oceanbase.batch.executor"; /** - * ocenbase hbase connect server ODP address + * oceanbase hbase connect server ODP address */ public static final String HBASE_OCEANBASE_ODP_ADDR = "hbase.oceanbase.odpAddr"; /** - * ocenbase hbase connect server ODP port + * oceanbase hbase connect server ODP port */ public static final String HBASE_OCEANBASE_ODP_PORT = "hbase.oceanbase.odpPort"; /** - * ocenbase hbase connect server ODP mode + * oceanbase hbase connect server ODP mode */ public static final String HBASE_OCEANBASE_ODP_MODE = "hbase.oceanbase.odpMode"; /** - * ocenbase hbase connect server database + * oceanbase hbase connect server database */ public static final String HBASE_OCEANBASE_DATABASE = "hbase.oceanbase.database"; /** - * ocenbase hbase model rowkey column is consist of following column + * oceanbase hbase model rowkey column is consist of following column * K, Q, T hbase value */ public static final String[] ROW_KEY_COLUMNS = new String[] { "K", "Q", "T" }; /** - * ocenbase hbase model value column is consist of following column + * oceanbase hbase model value column is consist of following column * V hbase value */ public static final String[] V_COLUMNS = new String[] { "V" }; @@ -124,6 +124,11 @@ public final class OHConstants { */ public static final String DEFAULT_HBASE_HTABLE_TEST_LOAD_SUFFIX = "_t"; + /** + * use to specify whether to query only the data in hot storage when performing a query. + */ + public static final String HBASE_HTABLE_QUERY_HOT_ONLY = "hbase.htable.query.hot_only"; + /*-------------------------------------------------------------------------------------------------------------*/ /** diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java new file mode 100644 index 00000000..05b48b4a --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java @@ -0,0 +1,49 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.execute; + +import com.alipay.oceanbase.hbase.util.OHBaseExceptionUtil; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.table.ObTable; + +import java.io.IOException; + +public abstract class AbstractObTableMetaExecutor implements ObTableMetaExecutor { + + @Override + public T execute(ObTableClient client, ObTableMetaRequest request) throws IOException { + if (request.getMetaType() != getMetaType()) { + throw new IOException("Invalid meta type, expected " + getMetaType()); + } + ObTable table = client.getRandomTable(); + ObTableMetaResponse response; + try { + response = (ObTableMetaResponse) client.executeWithRetry( + table, + request, + null /*tableName*/ + ); + } catch (Exception e) { + throw OHBaseExceptionUtil.convertTableException(e); + } + return parse(response); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java new file mode 100644 index 00000000..d55ff275 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java @@ -0,0 +1,49 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.execute; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; + +import java.io.IOException; + +public interface ObTableMetaExecutor { + /** + * 执行元数据请求 + * @param request 元数据请求 + * @return 解析后的元数据对象 + * @throws IOException 如果执行失败或解析失败 + */ + T execute(ObTableClient client, ObTableMetaRequest request) throws IOException; + + /** + * 解析元数据响应, 用户需要重写 + * @param response 元数据响应 + * @return 解析后的元数据对象 + * @throws IOException 如果解析失败 + */ + T parse(ObTableMetaResponse response) throws IOException; + + /** + * 获取元信息类型, 用户需要重写 + * @return 元信息类型 + */ + ObTableRpcMetaType getMetaType() throws IOException; +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java new file mode 100644 index 00000000..5cb7d92c --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -0,0 +1,1148 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.bolt.transport.TransportCodes; +import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.exception.ObTableTransportException; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.replication.TableCFs; +import org.apache.hadoop.hbase.client.security.SecurityCapability; +import org.apache.hadoop.hbase.exceptions.TimeoutIOException; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; +import org.apache.hadoop.hbase.quotas.QuotaFilter; +import org.apache.hadoop.hbase.quotas.QuotaRetriever; +import org.apache.hadoop.hbase.quotas.QuotaSettings; +import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; +import org.apache.hadoop.hbase.replication.ReplicationException; +import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; +import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException; +import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; +import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; +import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException; +import org.apache.hadoop.hbase.util.Pair; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.Future; +import java.util.regex.Pattern; + +public class OHAdmin implements Admin { + private boolean aborted = false; + private final OHConnectionImpl connection; + private final Configuration conf; + OHAdmin(OHConnectionImpl connection) { + this.connection = connection; + this.conf = connection.getConfiguration(); + } + + @Override + public int getOperationTimeout() { + return connection.getOHConnectionConfiguration().getOperationTimeout(); + } + + @Override + public void abort(String msg, Throwable t) { + // do nothing, just throw the message and exception + this.aborted = true; + throw new RuntimeException(msg, t); + } + + @Override + public boolean isAborted() { + return this.aborted; + } + + @Override + public Connection getConnection() { + return this.connection; + } + + @Override + public boolean tableExists(TableName tableName) throws IOException { + try { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableExistsExecutor executor = new OHTableExistsExecutor(tableClient); + return executor.tableExists(tableName.getNameAsString()); + } catch (Exception e) { + // try to get the original cause + Throwable cause = e.getCause(); + while(cause != null && cause.getCause() != null) { + cause = cause.getCause(); + } + if (cause instanceof ObTableException) { + int errCode = ((ObTableException) cause).getErrorCode(); + // if the original cause is database_not_exist, means namespace in tableName does not exist + // for HBase, namespace not exist will not throw exceptions but will return false + if (errCode == ResultCodes.OB_ERR_BAD_DATABASE.errorCode) { + return false; + } + } + throw e; + } + } + + @Override + public HTableDescriptor[] listTables() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listTableDescriptors() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor[] listTables(Pattern pattern) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listTableDescriptors(Pattern pattern) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor[] listTables(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor[] listTables(Pattern pattern, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listTableDescriptors(Pattern pattern, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor[] listTables(String s, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public TableName[] listTableNames() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public TableName[] listTableNames(Pattern pattern) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public TableName[] listTableNames(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public TableName[] listTableNames(Pattern pattern, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public TableName[] listTableNames(String s, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor getTableDescriptor(TableName tableName) throws TableNotFoundException, IOException { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableName.getNameAsString(), tableClient); + try { + return executor.getTableDescriptor(); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else if (e.getCause().getMessage().contains("OB_TABLEGROUP_NOT_EXIST")) { + throw new TableNotFoundException(tableName); + } else { + throw e; + } + } + } + + @Override + public TableDescriptor getDescriptor(TableName tableName) throws IOException { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableName.getNameAsString(), tableClient); + try { + return executor.getTableDescriptor(); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else if (e.getCause().getMessage().contains("OB_TABLEGROUP_NOT_EXIST")) { + throw new TableNotFoundException(tableName); + } else { + throw e; + } + } + } + + @Override + public void createTable(TableDescriptor tableDescriptor) throws IOException { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableDescriptor.getTableName(), connectionConf); + OHCreateTableExecutor executor = new OHCreateTableExecutor(tableClient); + try { + executor.createTable(tableDescriptor, null); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } + } + + @Override + public void createTable(TableDescriptor tableDescriptor, byte[] bytes, byte[] bytes1, int i) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void createTable(TableDescriptor tableDescriptor, byte[][] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future createTableAsync(TableDescriptor tableDescriptor, byte[][] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void deleteTable(TableName tableName) throws IOException { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHDeleteTableExecutor executor = new OHDeleteTableExecutor(tableClient); + try { + executor.deleteTable(tableName.getNameAsString()); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } + } + + @Override + public Future deleteTableAsync(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor[] deleteTables(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor[] deleteTables(Pattern pattern) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void truncateTable(TableName tableName, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future truncateTableAsync(TableName tableName, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void enableTable(TableName tableName) throws IOException { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_ENABLE_TABLE); + try { + executor.enableTable(tableName.getNameAsString()); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } + } + + @Override + public Future enableTableAsync(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor[] enableTables(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor[] enableTables(Pattern pattern) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future disableTableAsync(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void disableTable(TableName tableName) throws IOException { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_DISABLE_TABLE); + try { + executor.disableTable(tableName.getNameAsString()); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } + } + + @Override + public HTableDescriptor[] disableTables(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor[] disableTables(Pattern pattern) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean isTableEnabled(TableName tableName) throws IOException { + return isDisabled(tableName) == false; + } + + @Override + public boolean isTableDisabled(TableName tableName) throws IOException { + return isDisabled(tableName) == true; + } + + private boolean isDisabled(TableName tableName) throws IOException { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableDescriptorExecutor tableDescriptor = new OHTableDescriptorExecutor(tableName.getNameAsString(), tableClient); + return tableDescriptor.isDisable(); + } + + @Override + public boolean isTableAvailable(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean isTableAvailable(TableName tableName, byte[][] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Pair getAlterStatus(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Pair getAlterStatus(byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void addColumnFamily(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future addColumnFamilyAsync(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void deleteColumn(TableName tableName, byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void deleteColumnFamily(TableName tableName, byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future deleteColumnFamilyAsync(TableName tableName, byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void modifyColumnFamily(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future modifyColumnFamilyAsync(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void closeRegion(String s, String s1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void closeRegion(byte[] bytes, String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean closeRegionWithEncodedRegionName(String s, String s1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void closeRegion(ServerName serverName, HRegionInfo hRegionInfo) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List getOnlineRegions(ServerName serverName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List getRegions(ServerName serverName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void flush(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void flushRegion(byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void flushRegionServer(ServerName serverName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void compact(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void compactRegion(byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void compact(TableName tableName, byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void compactRegion(byte[] bytes, byte[] bytes1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void compact(TableName tableName, CompactType compactType) throws IOException, InterruptedException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void compact(TableName tableName, byte[] bytes, CompactType compactType) throws IOException, InterruptedException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void majorCompact(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void majorCompactRegion(byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void majorCompact(TableName tableName, byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void majorCompactRegion(byte[] bytes, byte[] bytes1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void majorCompact(TableName tableName, CompactType compactType) throws IOException, InterruptedException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void majorCompact(TableName tableName, byte[] bytes, CompactType compactType) throws IOException, InterruptedException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void compactRegionServer(ServerName serverName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void majorCompactRegionServer(ServerName serverName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void move(byte[] bytes, byte[] bytes1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void assign(byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void unassign(byte[] bytes, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void offline(byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean balancerSwitch(boolean b, boolean b1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean balance() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean balance(boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean isBalancerEnabled() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public CacheEvictionStats clearBlockCache(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean normalize() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean isNormalizerEnabled() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean normalizerSwitch(boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean catalogJanitorSwitch(boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public int runCatalogJanitor() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean isCatalogJanitorEnabled() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean cleanerChoreSwitch(boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean runCleanerChore() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean isCleanerChoreEnabled() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void mergeRegions(byte[] bytes, byte[] bytes1, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future mergeRegionsAsync(byte[] bytes, byte[] bytes1, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future mergeRegionsAsync(byte[][] bytes, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void split(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void splitRegion(byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void split(TableName tableName, byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void splitRegion(byte[] bytes, byte[] bytes1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future splitRegionAsync(byte[] bytes, byte[] bytes1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void modifyTable(TableName tableName, TableDescriptor tableDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void modifyTable(TableDescriptor tableDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future modifyTableAsync(TableName tableName, TableDescriptor tableDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future modifyTableAsync(TableDescriptor tableDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void shutdown() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void stopMaster() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean isMasterInMaintenanceMode() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void stopRegionServer(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public ClusterMetrics getClusterMetrics(EnumSet enumSet) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List getRegionMetrics(ServerName serverName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List getRegionMetrics(ServerName serverName, TableName tableName) throws IOException { + if (tableName == null) { + throw new FeatureNotSupportedException("does not support tableName is null"); + } + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHRegionMetricsExecutor executor = new OHRegionMetricsExecutor(tableClient); + return executor.getRegionMetrics(tableName.getNameAsString()); + } + + @Override + public Configuration getConfiguration() { + return connection.getConfiguration(); + } + + @Override + public void createNamespace(NamespaceDescriptor namespaceDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future createNamespaceAsync(NamespaceDescriptor namespaceDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void modifyNamespace(NamespaceDescriptor namespaceDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future modifyNamespaceAsync(NamespaceDescriptor namespaceDescriptor) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void deleteNamespace(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future deleteNamespaceAsync(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public NamespaceDescriptor getNamespaceDescriptor(String s) throws NamespaceNotFoundException, IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public NamespaceDescriptor[] listNamespaceDescriptors() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor[] listTableDescriptorsByNamespace(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listTableDescriptorsByNamespace(byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public TableName[] listTableNamesByNamespace(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List getTableRegions(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List getRegions(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public synchronized void close() throws IOException { + } + + @Override + public HTableDescriptor[] getTableDescriptorsByTableName(List list) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listTableDescriptors(List list) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public HTableDescriptor[] getTableDescriptors(List list) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean abortProcedure(long l, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future abortProcedureAsync(long l, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public String getProcedures() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public String getLocks() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void rollWALWriter(ServerName serverName) throws IOException, FailedLogCloseException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public CompactionState getCompactionState(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public CompactionState getCompactionState(TableName tableName, CompactType compactType) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public CompactionState getCompactionStateForRegion(byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public long getLastMajorCompactionTimestamp(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public long getLastMajorCompactionTimestampForRegion(byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void snapshot(String s, TableName tableName) throws IOException, SnapshotCreationException, IllegalArgumentException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void snapshot(byte[] bytes, TableName tableName) throws IOException, SnapshotCreationException, IllegalArgumentException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void snapshot(String s, TableName tableName, SnapshotType snapshotType) throws IOException, SnapshotCreationException, IllegalArgumentException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void snapshot(SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException, IllegalArgumentException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void snapshotAsync(SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean isSnapshotFinished(SnapshotDescription snapshotDescription) throws IOException, HBaseSnapshotException, UnknownSnapshotException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void restoreSnapshot(byte[] bytes) throws IOException, RestoreSnapshotException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void restoreSnapshot(String s) throws IOException, RestoreSnapshotException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future restoreSnapshotAsync(String s) throws IOException, RestoreSnapshotException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void restoreSnapshot(byte[] bytes, boolean b) throws IOException, RestoreSnapshotException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void restoreSnapshot(String s, boolean b) throws IOException, RestoreSnapshotException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void restoreSnapshot(String s, boolean b, boolean b1) throws IOException, RestoreSnapshotException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void cloneSnapshot(byte[] bytes, TableName tableName) throws IOException, TableExistsException, RestoreSnapshotException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void cloneSnapshot(String s, TableName tableName, boolean b) throws IOException, TableExistsException, RestoreSnapshotException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void cloneSnapshot(String s, TableName tableName) throws IOException, TableExistsException, RestoreSnapshotException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Future cloneSnapshotAsync(String s, TableName tableName) throws IOException, TableExistsException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void execProcedure(String s, String s1, Map map) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public byte[] execProcedureWithReturn(String s, String s1, Map map) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean isProcedureFinished(String s, String s1, Map map) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listSnapshots() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listSnapshots(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listSnapshots(Pattern pattern) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listTableSnapshots(String s, String s1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listTableSnapshots(Pattern pattern, Pattern pattern1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void deleteSnapshot(byte[] bytes) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void deleteSnapshot(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void deleteSnapshots(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void deleteSnapshots(Pattern pattern) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void deleteTableSnapshots(String s, String s1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void deleteTableSnapshots(Pattern pattern, Pattern pattern1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void setQuota(QuotaSettings quotaSettings) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public QuotaRetriever getQuotaRetriever(QuotaFilter quotaFilter) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List getQuota(QuotaFilter quotaFilter) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public CoprocessorRpcChannel coprocessorService() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public CoprocessorRpcChannel coprocessorService(ServerName serverName) { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void updateConfiguration(ServerName serverName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void updateConfiguration() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List getSecurityCapabilities() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean splitSwitch(boolean b, boolean b1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean mergeSwitch(boolean b, boolean b1) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean isSplitEnabled() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public boolean isMergeEnabled() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void addReplicationPeer(String s, ReplicationPeerConfig replicationPeerConfig, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void removeReplicationPeer(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void enableReplicationPeer(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void disableReplicationPeer(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public ReplicationPeerConfig getReplicationPeerConfig(String s) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void updateReplicationPeerConfig(String s, ReplicationPeerConfig replicationPeerConfig) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void appendReplicationPeerTableCFs(String s, Map> map) throws ReplicationException, IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void removeReplicationPeerTableCFs(String s, Map> map) throws ReplicationException, IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listReplicationPeers() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listReplicationPeers(Pattern pattern) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void decommissionRegionServers(List list, boolean b) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listDecommissionedRegionServers() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void recommissionRegionServer(ServerName serverName, List list) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List listReplicatedTableCFs() throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void enableTableReplication(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void disableTableReplication(TableName tableName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public void clearCompactionQueues(ServerName serverName, Set set) throws IOException, InterruptedException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List clearDeadServers(List list) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBaseExceptionUtil.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBaseExceptionUtil.java new file mode 100644 index 00000000..7223ca90 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBaseExceptionUtil.java @@ -0,0 +1,27 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import org.apache.hadoop.hbase.*; + +import java.io.IOException; + +public class OHBaseExceptionUtil { + public static IOException convertTableException(Exception e) { + if (e instanceof ObTableException) { + final int errCode = ((ObTableException) e).getErrorCode(); + if (errCode == ResultCodes.OB_KV_HBASE_TABLE_NOT_EXISTS.errorCode) { + return (TableNotFoundException) new TableNotFoundException(e.getMessage()).initCause(e); + } else if (errCode == ResultCodes.OB_KV_HBASE_TABLE_EXISTS.errorCode) { + return (TableExistsException) new TableExistsException(e.getMessage()).initCause(e); + } else if (errCode == ResultCodes.OB_KV_HBASE_NAMESPACE_NOT_FOUND.errorCode) { + return (NamespaceNotFoundException) new NamespaceNotFoundException(e.getMessage()).initCause(e); + } else if (errCode == ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode) { + return (TableNotEnabledException) new TableNotEnabledException(e.getMessage()).initCause(e); + } else if (errCode == ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode) { + return (TableNotDisabledException) new TableNotDisabledException(e.getMessage()).initCause(e); + } + } + return new IOException("Failed to execute request", e); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index d6e63d40..76ddf9f7 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -275,15 +275,11 @@ private void execute(boolean flushAll) throws IOException { // if commit all successfully, clean execBuffer execBuffer.clear(); } catch (Exception ex) { - LOGGER.error(LCD.convert("01-00026"), ex); - if (ex.getCause() instanceof RetriesExhaustedWithDetailsException) { - LOGGER.error(tableName + ": One or more of the operations have failed after retries."); - RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex.getCause(); - // recollect failed mutations - execBuffer.clear(); - for (int i = 0; i < retryException.getNumExceptions(); ++i) { - execBuffer.add((Mutation) retryException.getRow(i)); - } + // do not recollect error operations, notify outside + LOGGER.error("error happens, table name: {}", tableName.getNameAsString(), ex); + if (ex instanceof RetriesExhaustedWithDetailsException) { + LOGGER.error("TableName: {}, One or more of the operations have failed after retries.", tableName.getNameAsString(), ex); + RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex; if (listener != null) { listener.onException(retryException, this); } else { @@ -293,13 +289,6 @@ private void execute(boolean flushAll) throws IOException { LOGGER.error("Errors unrelated to operations occur during mutation operation", ex); throw ex; } - } finally { - for (Mutation mutation : execBuffer) { - long size = mutation.heapSize(); - currentAsyncBufferSize.addAndGet(size); - asyncWriteBuffer.add(mutation); - undealtMutationCount.incrementAndGet(); - } } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java index 9b98dd0b..3c966a18 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java @@ -57,6 +57,7 @@ public class OHConnectionConfiguration { private final int rpcConnectTimeout; private final long writeBufferPeriodicFlushTimeoutMs; private final long writeBufferPeriodicFlushTimerTickMs; + private final int numRetries; public OHConnectionConfiguration(Configuration conf) { this.paramUrl = conf.get(HBASE_OCEANBASE_PARAM_URL); @@ -100,6 +101,8 @@ public OHConnectionConfiguration(Configuration conf) { } } this.rpcConnectTimeout = rpcConnectTimeout; + this.numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.scannerCaching = conf.getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, Integer.MAX_VALUE); this.scannerMaxResultSize = conf.getLong( @@ -209,4 +212,8 @@ public long getWriteBufferPeriodicFlushTimeoutMs() { public long getWriteBufferPeriodicFlushTimerTickMs() { return this.writeBufferPeriodicFlushTimerTickMs; } + + public int getNumRetries() { + return this.numRetries; + } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index 41a0690a..fd666159 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -19,6 +19,7 @@ import com.alipay.oceanbase.hbase.OHTable; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.rpc.ObTableClient; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; @@ -140,12 +141,17 @@ public BufferedMutator getBufferedMutator(BufferedMutatorParams params) throws I @Override public RegionLocator getRegionLocator(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("not supported yet'"); + // need to use new connection configuration + // to avoid change the database in original param url by namespace in tableName + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), obTableClient); + return executor.getRegionLocator(String.valueOf(tableName)); } @Override public Admin getAdmin() throws IOException { - throw new FeatureNotSupportedException("not supported yet'"); + return new OHAdmin(this); } private void shutdownBatchPool(ExecutorService pool) { diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java new file mode 100644 index 00000000..4bab8e3b --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java @@ -0,0 +1,70 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; + +import java.io.IOException; +import java.util.*; + +public class OHCreateTableExecutor extends AbstractObTableMetaExecutor { + private final ObTableClient client; + + OHCreateTableExecutor(ObTableClient client) { + this.client = client; + } + + @Override + public ObTableRpcMetaType getMetaType() { + return ObTableRpcMetaType.HTABLE_CREATE_TABLE; + } + + @Override + public Void parse(ObTableMetaResponse response) throws IOException { + // success, do nothing + return null; + } + + public void createTable(TableDescriptor tableDescriptor, byte[][] splitKeys) throws IOException { + final ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestData = new HashMap<>(); + requestData.put("htable_name", tableDescriptor.getTableName().getNameAsString()); + Map> columnFamilies = new HashMap<>(); + for (ColumnFamilyDescriptor columnDescriptor : tableDescriptor.getColumnFamilies()) { + Map columnFamily = new HashMap<>(); + columnFamily.put("ttl", columnDescriptor.getTimeToLive()); + columnFamily.put("max_version", columnDescriptor.getMaxVersions()); + columnFamilies.put(columnDescriptor.getNameAsString(), columnFamily); + } + requestData.put("column_families", columnFamilies); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); + request.setData(jsonData); + execute(client, request); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java new file mode 100644 index 00000000..1f689596 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java @@ -0,0 +1,59 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class OHDeleteTableExecutor extends AbstractObTableMetaExecutor { + private final ObTableClient tableClient; + + OHDeleteTableExecutor(ObTableClient tableClient) { + this.tableClient = tableClient; + } + + @Override + public ObTableRpcMetaType getMetaType() { + return ObTableRpcMetaType.HTABLE_DELETE_TABLE; + } + + @Override + public Void parse(ObTableMetaResponse response) throws IOException { + // do nothing, error will be thrown from table + return null; + } + + public Void deleteTable(String tableName) throws IOException { + ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestDataMap = new HashMap<>(); + requestDataMap.put("table_name", tableName); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestDataMap); + request.setData(jsonData); + return execute(tableClient, request); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java new file mode 100644 index 00000000..42ea4179 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java @@ -0,0 +1,142 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.bolt.transport.TransportCodes; +import com.alipay.oceanbase.rpc.exception.ObTableTransportException; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.exceptions.TimeoutIOException; +import org.apache.hadoop.hbase.util.Pair; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +public class OHRegionLocator implements RegionLocator { + private byte[][] startKeys; + private byte[][] endKeys; + private final ObTableClient tableClient; + private final TableName tableName; + + private List> regionLocations; + + public OHRegionLocator(byte[][] startKeys, byte[][] endKeys, + List> regionLocations, TableName tableName, + ObTableClient tableClient) { + this.startKeys = startKeys; + this.endKeys = endKeys; + this.regionLocations = regionLocations; + this.tableName = tableName; + this.tableClient = tableClient; + } + + @Override + public HRegionLocation getRegionLocation(byte[] bytes) throws IOException { + // check if bytes is in the range of startKeys and endKeys + for (Pair pair : regionLocations) { + if (pair.getSecond() && pair.getFirst().getRegionInfo().containsRow(bytes)) { + return pair.getFirst(); + } + } + return null; + } + + public List> getRegionLocationPair() { + return regionLocations; + } + + @Override + public HRegionLocation getRegionLocation(byte[] bytes, boolean b) throws IOException { + if (b || regionLocations.isEmpty()) { + OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), + tableClient); + try { + OHRegionLocator location = executor.getRegionLocator(tableName.toString()); + this.startKeys = location.getStartKeys(); + this.endKeys = location.getEndKeys(); + this.regionLocations = location.getRegionLocationPair(); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } + } + return getRegionLocation(bytes); + } + + @Override + public List getAllRegionLocations() throws IOException { + return regionLocations.stream().map(Pair::getFirst).collect(Collectors.toList()); + } + + /** + * Gets the starting row key for every region in the currently open table. + *

+ * This is mainly useful for the MapReduce integration. + * + * @return Array of region starting row keys + * @throws IOException if a remote or network exception occurs + */ + @Override + public byte[][] getStartKeys() throws IOException { + return startKeys; + } + + /** + * Gets the ending row key for every region in the currently open table. + *

+ * This is mainly useful for the MapReduce integration. + * + * @return Array of region ending row keys + * @throws IOException if a remote or network exception occurs + */ + @Override + public byte[][] getEndKeys() throws IOException { + return endKeys; + } + + /** + * Gets the starting and ending row keys for every region in the currently + * open table. + *

+ * This is mainly useful for the MapReduce integration. + * + * @return Pair of arrays of region starting and ending row keys + * @throws IOException if a remote or network exception occurs + */ + @Override + public Pair getStartEndKeys() throws IOException { + return Pair.newPair(startKeys, endKeys); + } + + @Override + public TableName getName() { + return tableName; + } + + @Override + public void close() throws IOException { + return; + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java new file mode 100644 index 00000000..63978d6c --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java @@ -0,0 +1,258 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.constant.Constants; +import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Pair; + +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class OHRegionLocatorExecutor extends AbstractObTableMetaExecutor { + private final String tableName; + private final ObTableClient client; + + OHRegionLocatorExecutor(String tableName, ObTableClient client) { + this.tableName = tableName; + this.client = client; + } + + @Override + public ObTableRpcMetaType getMetaType() { + return ObTableRpcMetaType.HTABLE_REGION_LOCATOR; + } + + /** + * Parses the response and creates a region locator + * @param response response from the server + * @return OHRegionLocator + * @throws IOException if failed to parse the response + */ + @Override + public OHRegionLocator parse(ObTableMetaResponse response) throws IOException { + try { + final String jsonData = response.getData(); + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(jsonData)) + .orElseThrow(() -> new IOException("jsonMap is null")); + /* + { + "table_id_dict": [1001, 1002], + "replica_dict": [ + ["127.0.0.1", 2881], + ["127.0.0.2", 2882], + ["127.0.0.3", 2883] + ], + "partitions": [ + // 表1001的3个分区，每个分区3副本 + [0, 50001, "rowkey_1", 0, 1], // leader + [0, 50001, "rowkey_1", 1, 2], // follower + [0, 50001, "rowkey_1", 2, 2], // follower + [0, 50002, "rowkey_2", 0, 1], + [0, 50002, "rowkey_2", 1, 2], + [0, 50002, "rowkey_2", 2, 2], + [0, 50003, "rowkey_3", 0, 1], + [0, 50003, "rowkey_3", 1, 2], + [0, 50003, "rowkey_3", 2, 2], + + // 表1002的3个分区，每个分区3副本 + [1, 50004, "rowkey_1", 0, 1], + [1, 50004, "rowkey_1", 1, 2], + [1, 50004, "rowkey_1", 2, 2], + [1, 50005, "rowkey_2", 0, 1], + [1, 50005, "rowkey_2", 1, 2], + [1, 50005, "rowkey_2", 2, 2], + [1, 50006, "rowkey_3", 0, 1], + [1, 50006, "rowkey_3", 1, 2], + [1, 50006, "rowkey_3", 2, 2] + ] + } + */ + JsonNode partitionsNode = Optional.ofNullable(jsonMap.get("partitions")) + .orElseThrow(() -> new IOException("partitions is null")); + List partitions = objectMapper.convertValue(partitionsNode, new TypeReference>(){}); + + JsonNode tableIdDictNode = Optional.ofNullable(jsonMap.get("table_id_dict")) + .orElseThrow(() -> new IOException("tableIdDict is null")); + List tableIdDict = objectMapper.convertValue(tableIdDictNode, new TypeReference>(){}); + + JsonNode replicaDictNode = Optional.ofNullable(jsonMap.get("replica_dict")) + .orElseThrow(() -> new IOException("replicaDict is null")); + List replicaDict = objectMapper.convertValue(replicaDictNode, new TypeReference>(){}); + + final boolean isHashLikePartition = partitions.stream() + .map(obj -> (List) obj) + .filter(partition -> { + if (partition.size() <= 3) { + throw new IllegalArgumentException("partition size is not 3"); + } + return true; + }) + .allMatch(partition -> { + final byte[] highBound = partition.get(2).toString().getBytes(); + return Arrays.equals(highBound, Constants.EMPTY_STRING.getBytes()); + }); + return isHashLikePartition ? + createHashPartitionLocator(tableIdDict, replicaDict, partitions) : + createRangePartitionLocator(tableIdDict, replicaDict, partitions); + } catch (IllegalArgumentException e) { + throw new IOException("Invalid partition data: " + e.getMessage(), e); + } + } + + /** + * Creates a region locator for range partitions + * @param tableIdDict table ID dictionary + * @param replicaDict replica dictionary + * @param partitions list of partition data + * @return OHRegionLocator for range partitions + */ + private OHRegionLocator createRangePartitionLocator( + final List tableIdDict, + final List replicaDict, + final List partitions) { + if ((partitions.size() % tableIdDict.size()) != 0) { + throw new ObTableUnexpectedException( + "The number of partitions should be an integer multiple of the number of tables"); + } + final List startKeysList = new ArrayList<>(); + final List endKeysList = new ArrayList<>(); + // Currently based on SHARDING = 'ADAPTIVE' Table Group implementation for multi-CF, + // where one constraint is that Tables within the same Table Group have the same partitioning method, + // thus their associated partition boundaries are consistent. + // + // In native HBase, a Region can contain multiple CFs. + // Similarly, for OBKV-HBase, multiple CFs corresponding to related tablets also reside on the same machine. + // Therefore, here we maintain the same behavior by returning all partitions from just one table. + final int regionCountPerTable = partitions.size() / tableIdDict.size(); + + List oneTableLeaders = new ArrayList<>(); + for (int i = 0; i < regionCountPerTable; ++i) { + boolean isLeader = ((int) ((List) partitions.get(i)).get(4) == 1); + if (isLeader) { + oneTableLeaders.add(partitions.get(i)); + } + } + // Note that the number of leaders per single table != (regionCountPerTable / replicaDict.size()). + for (int i = 0; i < oneTableLeaders.size(); ++i) { + if (i == 0) { + startKeysList.add(HConstants.EMPTY_BYTE_ARRAY); + endKeysList.add(((List) oneTableLeaders.get(i)).get(2).toString().getBytes()); + } else if (i == oneTableLeaders.size() - 1) { + startKeysList.add(((List) oneTableLeaders.get(i - 1)).get(2).toString().getBytes()); + endKeysList.add(HConstants.EMPTY_BYTE_ARRAY); + } else { + startKeysList.add(((List) oneTableLeaders.get(i - 1)).get(2).toString().getBytes()); + endKeysList.add(((List) oneTableLeaders.get(i)).get(2).toString().getBytes()); + } + } + final byte[][] startKeys = startKeysList.toArray(new byte[0][]); + final byte[][] endKeys = endKeysList.toArray(new byte[0][]); + // Create region locations for all regions in one table + final List regionLocations = IntStream.range(0, regionCountPerTable) + .mapToObj(i -> { + final List partition = (List) partitions.get(Math.min(i, regionCountPerTable - 1)); + final int replicationIdx = (int) partition.get(3); + final List hostInfo = (List) replicaDict.get(replicationIdx); + + final ServerName serverName = ServerName.valueOf( + hostInfo.get(0).toString(), + (int) hostInfo.get(1), + i + ); + int boundIndex = i / replicaDict.size(); + final HRegionInfo regionInfo = new HRegionInfo( + TableName.valueOf(tableName), + startKeys[boundIndex], + endKeys[boundIndex] + ); + HRegionLocation location = new HRegionLocation(regionInfo, serverName, i); + Boolean role = (int) partition.get(4) == 1; + return new Pair(location, role); + }) + .collect(Collectors.toList()); + + return new OHRegionLocator(startKeys, endKeys, regionLocations, TableName.valueOf(tableName), client); + } + + /** + * Creates a region locator for hash partitions + * @param tableIdDict table ID dictionary + * @param replicaDict replica dictionary + * @param partitions list of partition data + * @return OHRegionLocator for hash partitions + */ + private OHRegionLocator createHashPartitionLocator( + final List tableIdDict, + final List replicaDict, + final List partitions) { + + final byte[][] startKeys = new byte[1][]; + final byte[][] endKeys = new byte[1][]; + startKeys[0] = HConstants.EMPTY_BYTE_ARRAY; + endKeys[0] = HConstants.EMPTY_BYTE_ARRAY; + final int regionCountPerTable = partitions.size() / tableIdDict.size(); + final List regionLocations = IntStream.range(0, regionCountPerTable) + .mapToObj(i -> { + final List partition = (List) partitions.get(i); + final int replicationIdx = (int) partition.get(3); + final List hostInfo = (List) replicaDict.get(replicationIdx); + + final ServerName serverName = ServerName.valueOf( + hostInfo.get(0).toString(), + (int) hostInfo.get(1), + i + ); + final HRegionInfo regionInfo = new HRegionInfo( + TableName.valueOf(tableName), + startKeys[0], + endKeys[0] + ); + HRegionLocation location = new HRegionLocation(regionInfo, serverName, i); + Boolean role = (int) partition.get(4) == 1; + return new Pair(location, role); + }) + .collect(Collectors.toList()); + + return new OHRegionLocator(startKeys, endKeys, regionLocations, TableName.valueOf(tableName), client); + } + + public OHRegionLocator getRegionLocator(final String tableName) throws IOException { + final ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + final Map requestData = new HashMap<>(); + requestData.put("table_name", tableName); + ObjectMapper objectMapper = new ObjectMapper(); + final String jsonData = objectMapper.writeValueAsString(requestData); + request.setData(jsonData); + + return execute(client, request); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java new file mode 100644 index 00000000..0e1e4732 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java @@ -0,0 +1,137 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; +import org.apache.hadoop.hbase.RegionMetrics; +import org.apache.hadoop.hbase.Size; + +import java.util.Map; + +public class OHRegionMetrics implements RegionMetrics { + private final String tablegroup; + private final byte[] name; // tablet_name, id in String + private final Size storeFileSize; // tablet storage used in ssTable + private final Size memStoreSize; // tablet storage used in memTable + + OHRegionMetrics(String tablegroup, byte[] name, Size storeFileSize, Size memStoreSize) { + this.tablegroup = tablegroup; + this.name = name; + this.storeFileSize = storeFileSize; + this.memStoreSize = memStoreSize; + } + + public String getTablegroup() { + return tablegroup; + } + + @Override + public byte[] getRegionName() { + return name; + } + + @Override + public int getStoreCount() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public int getStoreFileCount() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Size getStoreFileSize() { + return storeFileSize; + } + + @Override + public Size getMemStoreSize() { + return memStoreSize; + } + + @Override + public long getReadRequestCount() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public long getWriteRequestCount() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public long getFilteredReadRequestCount() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Size getStoreFileIndexSize() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Size getStoreFileRootLevelIndexSize() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Size getStoreFileUncompressedDataIndexSize() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Size getBloomFilterSize() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public long getCompactingCellCount() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public long getCompactedCellCount() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public long getCompletedSequenceId() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Map getStoreSequenceId() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public Size getUncompressedStoreFileSize() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public float getDataLocality() { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public long getLastMajorCompactionTimestamp() { + throw new FeatureNotSupportedException("does not support yet"); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java new file mode 100644 index 00000000..40235ef3 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java @@ -0,0 +1,97 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import org.apache.hadoop.hbase.RegionMetrics; +import org.apache.hadoop.hbase.Size; + +import java.io.IOException; +import java.util.*; + +public class OHRegionMetricsExecutor extends AbstractObTableMetaExecutor> { + private final ObTableClient tableClient; + + OHRegionMetricsExecutor(ObTableClient tableClient) { + this.tableClient = tableClient; + } + + @Override + public ObTableRpcMetaType getMetaType() throws IOException { + return ObTableRpcMetaType.HTABLE_REGION_METRICS; + } + + /* + * { + tableName: "tablegroup_name", + regionList:{ + "regions": [200051, 200052, 200053, 200191, 200192, 200193, ...], + "memTableSize":[123, 321, 321, 123, 321, 321, ...], + "ssTableSize":[5122, 4111, 5661, 5122, 4111, 5661, ...] + } + } + * */ + @Override + public List parse(ObTableMetaResponse response) throws IOException { + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(response.getData())) + .orElseThrow(() -> new IOException("jsonMap is null")); + JsonNode tableGroupNameNode = Optional.ofNullable(jsonMap.get("tableName")) + .orElseThrow(() -> new IOException("tableName is null")); + String tableGroupName = tableGroupNameNode.asText(); + JsonNode regionListNode = Optional.ofNullable(jsonMap.get("regionList")) + .orElseThrow(() -> new IOException("regionList is null")); + List regions = Optional.>ofNullable(objectMapper.convertValue(regionListNode.get("regions"), new TypeReference>() {})) + .orElseThrow(() -> new IOException("regions is null")); + List memTableSizeList = Optional.>ofNullable(objectMapper.convertValue(regionListNode.get("memTableSize"), new TypeReference>() {})) + .orElseThrow(() -> new IOException("memTableSize is null")); + List ssTableSizeList = Optional.>ofNullable(objectMapper.convertValue(regionListNode.get("ssTableSize"), new TypeReference>() {})) + .orElseThrow(() -> new IOException("ssTableSize is null")); + List metricsList = new ArrayList<>(); + if (regions.isEmpty() || regions.size() != memTableSizeList.size() || memTableSizeList.size() != ssTableSizeList.size()) { + throw new IOException("size length has to be the same"); + } + for (int i = 0; i < regions.size(); ++i) { + String name_str = Integer.toString(regions.get(i)); + byte[] name = name_str.getBytes(); + Size storeFileSize = new Size(((double) ssTableSizeList.get(i)) / (1024 * 1024) , Size.Unit.MEGABYTE); // The unit in original HBase is MEGABYTE, for us it is BYTE + Size memStoreSize = new Size(((double) memTableSizeList.get(i)) / (1024 * 1024), Size.Unit.MEGABYTE); // The unit in original HBase is MEGABYTE, for us it is BYTE + OHRegionMetrics ohRegionMetrics = new OHRegionMetrics(tableGroupName, name, storeFileSize, memStoreSize); + metricsList.add(ohRegionMetrics); + } + return metricsList; + } + + public List getRegionMetrics(String tableName) throws IOException { + ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestData = new HashMap<>(); + requestData.put("table_name", tableName); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); + request.setData(jsonData); + return execute(tableClient, request); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java new file mode 100644 index 00000000..9bf0075b --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java @@ -0,0 +1,74 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import org.apache.hadoop.hbase.TableNotDisabledException; +import org.apache.hadoop.hbase.TableNotEnabledException; +import org.apache.hadoop.hbase.TableNotFoundException; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class OHTableAccessControlExecutor extends AbstractObTableMetaExecutor { + private final ObTableClient tableClient; + private final ObTableRpcMetaType type; + + OHTableAccessControlExecutor(ObTableClient tableClient, ObTableRpcMetaType type) { + this.tableClient = tableClient; + this.type = type; + } + + @Override + public ObTableRpcMetaType getMetaType() throws IOException { + return this.type; + } + + @Override + public Void parse(ObTableMetaResponse response) throws IOException { + return null; + } + + public void enableTable(String tableName) throws IOException, TableNotFoundException, TableNotEnabledException { + ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestData = new HashMap<>(); + requestData.put("table_name", tableName); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); + request.setData(jsonData); + execute(tableClient, request); + } + + public void disableTable(String tableName) throws IOException, TableNotFoundException, TableNotDisabledException { + ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestData = new HashMap<>(); + requestData.put("table_name", tableName); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); + request.setData(jsonData); + execute(tableClient, request); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java new file mode 100644 index 00000000..dcf42681 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java @@ -0,0 +1,156 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import com.alipay.oceanbase.rpc.table.ObTable; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +public class OHTableDescriptorExecutor extends AbstractObTableMetaExecutor { + private final String tableName; + private final ObTableClient client; + + public OHTableDescriptorExecutor(String tableName, ObTableClient client) { + this.tableName = tableName; + this.client = client; + } + + @Override + public HTableDescriptor parse(ObTableMetaResponse response) throws IOException { + try { + final String jsonData = response.getData(); + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(jsonData)) + .orElseThrow(() -> new IOException("jsonMap is null")); + /* + { + "cfDescs": { + "cf1": { + "TTL":604800 + "maxVersions": 3 + }, + "cf2": { + "TTL":259200 + "maxVersions": 2 + } + }, + "tbDesc": { + "name":"test", + "state":"disable" ("enable") + } + } + */ + HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName)); + JsonNode cfDescsNode = Optional.ofNullable(jsonMap.get("cfDescs")) + .orElseThrow(() -> new IOException("cfDesc is null")); + Stream> stream = cfDescsNode.propertyStream(); + stream.forEach(entry -> { + String cfName = entry.getKey(); + JsonNode value = entry.getValue(); + int ttl = value.path("TTL").asInt(); + int maxVersions = value.path("maxVersions").asInt(); + HColumnDescriptor cf = new HColumnDescriptor(cfName); + cf.setTimeToLive(ttl); + cf.setMaxVersions(maxVersions); + tableDescriptor.addFamily(cf); + }); + return tableDescriptor; + } catch (IllegalArgumentException e) { + throw new IOException("Failed to parse response", e); + } + } + + @Override + public ObTableRpcMetaType getMetaType() throws IOException { + return ObTableRpcMetaType.HTABLE_GET_DESC; + } + + public HTableDescriptor getTableDescriptor() throws IOException { + final ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + final Map requestData = new HashMap<>(); + requestData.put("table_name", tableName); + ObjectMapper objectMapper = new ObjectMapper(); + final String jsonData = objectMapper.writeValueAsString(requestData); + request.setData(jsonData); + + return execute(client, request); + } + + public boolean isDisable() throws IOException { + boolean isDisable = false; + final ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + final Map requestData = new HashMap<>(); + requestData.put("table_name", tableName); + final ObjectMapper objectMapper = new ObjectMapper(); + final String jsonData = objectMapper.writeValueAsString(requestData); + request.setData(jsonData); + try { + ObTableMetaResponse response = innerExecute(client, request); + final String responseData = response.getData(); + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(responseData)) + .orElseThrow(() -> new IOException("jsonMap is null")); + JsonNode tbDesc = Optional.ofNullable(jsonMap.get("tableDesc")) + .orElseThrow(() -> new IOException("tableDesc is null")); + String state = tbDesc.get("state").asText(); + if (state.compareToIgnoreCase("disable") == 0) { + isDisable = true; + } else { + isDisable = false; + } + } catch (IOException e) { + throw e; + } + return isDisable; + } + + private ObTableMetaResponse innerExecute(ObTableClient client, ObTableMetaRequest request) + throws IOException { + if (request.getMetaType() != getMetaType()) { + throw new IOException("Invalid meta type, expected " + getMetaType()); + } + ObTable table = client.getRandomTable(); + ObTableMetaResponse response; + try { + response = (ObTableMetaResponse) client.executeWithRetry( + table, + request, + null /*tableName*/ + ); + } catch (Exception e) { + throw new IOException("Failed to execute request", e); + } + return response; + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java new file mode 100644 index 00000000..ffaaca18 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java @@ -0,0 +1,64 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public class OHTableExistsExecutor extends AbstractObTableMetaExecutor { + private final ObTableClient tableClient; + + OHTableExistsExecutor(ObTableClient tableClient) { + this.tableClient = tableClient; + } + + @Override + public ObTableRpcMetaType getMetaType() throws IOException { + return ObTableRpcMetaType.HTABLE_EXISTS; + } + + @Override + public Boolean parse(ObTableMetaResponse response) throws IOException { + String jsonData = response.getData(); + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode = Optional.ofNullable(objectMapper.readTree(jsonData)) + .orElseThrow(() -> new IOException("jsonMap is null")); + return jsonNode.get("exists").asBoolean(); + } + + public Boolean tableExists(String tableName) throws IOException { + ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestData = new HashMap<>(); + requestData.put("table_name", tableName); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); + request.setData(jsonData); + return execute(tableClient, request); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java index 5db940aa..9a3a56f2 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java @@ -19,8 +19,10 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.constant.Constants; +import com.alipay.oceanbase.hbase.OHTable; import com.google.common.base.Objects; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.TableName; import java.io.IOException; import java.util.Map; @@ -118,7 +120,7 @@ public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableCli OB_TABLE_CLIENT_INSTANCE.put(obTableClientKey, obTableClient); } } catch (Exception e) { - throw new IOException(e); + throw OHBaseExceptionUtil.convertTableException(e); } finally { lock.unlock(); } @@ -126,6 +128,22 @@ public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableCli return OB_TABLE_CLIENT_INSTANCE.get(obTableClientKey); } + public static ObTableClient getOrCreateObTableClientByTableName(TableName tableName, OHConnectionConfiguration connectionConfig) throws IllegalArgumentException, + IOException { + String tableNameString = tableName.getNameAsString(); + ObTableClient obTableClient = getOrCreateObTableClient( + OHTable.setUserDefinedNamespace(tableNameString, connectionConfig)); + ObTableClientManager.initTimeoutAndRetryTimes(obTableClient, connectionConfig); + return obTableClient; + } + + private static void initTimeoutAndRetryTimes(ObTableClient obTableClient, OHConnectionConfiguration ohConnectionConf) { + obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); + obTableClient.setRuntimeRetryTimes(ohConnectionConf.getNumRetries()); + obTableClient.setRuntimeMaxWait(ohConnectionConf.getOperationTimeout()); + obTableClient.setRuntimeBatchMaxWait(ohConnectionConf.getOperationTimeout()); + } + public static class ObTableClientKey { private String paramUrl; private String fullUserName; diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index cd2e7689..9473efd3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -20,9 +20,7 @@ import com.alipay.oceanbase.hbase.util.OHBufferedMutatorImpl; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; @@ -30,7 +28,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.concurrent.*; import static com.alipay.oceanbase.hbase.constants.OHConstants.SOCKET_TIMEOUT; @@ -81,7 +81,7 @@ public void testRefreshTableEntry() throws Exception { @After public void after() throws IOException { - hTable.close(); + if (hTable != null) hTable.close(); } @Test @@ -841,6 +841,249 @@ public void testBufferedMutatorConcurrent() throws Exception { } } + + /* + CREATE TABLEGROUP test_region_locator SHARDING = 'ADAPTIVE'; + CREATE TABLE `test_region_locator$family_region_locator` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = test_region_locator PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) + ); + */ + @Test + public void testRangePartitionWithRegionLocator() throws Exception { + final String tableNameStr = "test_region_locator"; + final String family = "family_region_locator"; + final int regionCount = 10; + final int rowsPerRegion = 5; + + final byte[][] splitPoints = new byte[][] { + Bytes.toBytes("c"), // p1: < 'c' + Bytes.toBytes("e"), // p2: < 'e' + Bytes.toBytes("g"), // p3: < 'g' + Bytes.toBytes("i"), // p4: < 'i' + Bytes.toBytes("l"), // p5: < 'l' + Bytes.toBytes("n"), // p6: < 'n' + Bytes.toBytes("p"), // p7: < 'p' + Bytes.toBytes("s"), // p8: < 's' + Bytes.toBytes("v") // p9: < 'v' + }; + + final TableName tableName = TableName.valueOf(tableNameStr); + final Configuration conf = ObHTableTestUtil.newConfiguration(); + connection = ConnectionFactory.createConnection(conf); + hTable = connection.getTable(tableName); + + try { + for (int i = 0; i < regionCount; i++) { + for (int j = 0; j < rowsPerRegion; j++) { + String rowKey; + if (i == 0) { + rowKey = "a" + j; + } else if (i == regionCount - 1) { + rowKey = "v" + j; + } else { + String baseKey = Bytes.toString(splitPoints[i - 1]); + rowKey = baseKey + j; + } + + Put put = new Put(Bytes.toBytes(rowKey)); + String value = String.format("value_%d_%d", i, j); + put.addColumn( + Bytes.toBytes(family), + Bytes.toBytes("q"), + Bytes.toBytes(value) + ); + hTable.put(put); + } + } + + try (RegionLocator locator = connection.getRegionLocator(tableName)) { + byte[][] startKeys = locator.getStartKeys(); + byte[][] endKeys = locator.getEndKeys(); + + Assert.assertEquals("Should have " + regionCount + " regions", + regionCount, startKeys.length); + + for (int i = 0; i < startKeys.length; i++) { + String startKeyStr = startKeys[i].length == 0 ? "-∞" : + Bytes.toString(startKeys[i]); + String endKeyStr = endKeys[i].length == 0 ? "+∞" : + Bytes.toString(endKeys[i]); + // 验证region边界 + if (i > 0) { + // 验证startKey与前一个endKey相同 + Assert.assertArrayEquals("Region " + i + " startKey should match previous endKey", + endKeys[i-1], startKeys[i]); + } + } + + for (int i = 0; i < startKeys.length; i++) { + Scan scan = new Scan(); + if (startKeys[i].length > 0) { + scan.setStartRow(startKeys[i]); + } + if (endKeys[i].length > 0) { + scan.setStopRow(endKeys[i]); + } + + String startKeyStr = startKeys[i].length == 0 ? "-∞" : + Bytes.toString(startKeys[i]); + String endKeyStr = endKeys[i].length == 0 ? "+∞" : + Bytes.toString(endKeys[i]); + + try (ResultScanner scanner = hTable.getScanner(scan)) { + List results = new ArrayList<>(); + for (Result result : scanner) { + results.add(result); + } + + Assert.assertEquals("Region " + i + " should have " + rowsPerRegion + " rows", + rowsPerRegion, results.size()); + + for (Result result : results) { + String rowKey = Bytes.toString(result.getRow()); + String value = Bytes.toString(result.getValue( + Bytes.toBytes(family), Bytes.toBytes("q"))); + if (startKeys[i].length > 0) { + Assert.assertTrue("Row key " + rowKey + " should be >= " + + Bytes.toString(startKeys[i]), + rowKey.compareTo(Bytes.toString(startKeys[i])) >= 0); + } + if (endKeys[i].length > 0) { + Assert.assertTrue("Row key " + rowKey + " should be < " + + Bytes.toString(endKeys[i]), + rowKey.compareTo(Bytes.toString(endKeys[i])) < 0); + } + } + } + } + } + } finally { + Optional.ofNullable(hTable).ifPresent(table -> { + try { + table.close(); + } catch (IOException e) { + e.printStackTrace(); + } + }); + Optional.ofNullable(connection).ifPresent(conn -> { + try { + conn.close(); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + } + + @Test + public void testHRegionLocation() throws IOException { + final String tableNameStr = "test_region_locator"; + + final byte[][] splitPoints = new byte[][] { + Bytes.toBytes("c"), // p1: < 'c' + Bytes.toBytes("e"), // p2: < 'e' + Bytes.toBytes("g"), // p3: < 'g' + Bytes.toBytes("i"), // p4: < 'i' + Bytes.toBytes("l"), // p5: < 'l' + Bytes.toBytes("n"), // p6: < 'n' + Bytes.toBytes("p"), // p7: < 'p' + Bytes.toBytes("s"), // p8: < 's' + Bytes.toBytes("v") // p9: < 'v' + }; + + final TableName tableName = TableName.valueOf(tableNameStr); + final Configuration conf = ObHTableTestUtil.newConfiguration(); + connection = ConnectionFactory.createConnection(conf); + hTable = connection.getTable(tableName); + // (min, c), [c, e), [e, g), [g, i), [i, l), [l, n), [n, p), [p, s), [s, v), [v, max) + try (RegionLocator locator = connection.getRegionLocator(tableName)) { + Assert.assertEquals(locator.getStartKeys().length, locator.getEndKeys().length); + Assert.assertEquals(locator.getStartKeys().length, 10); + HRegionLocation loc = locator.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY); + RegionInfo info = loc.getRegion(); + Assert.assertEquals(Arrays.toString(locator.getStartKeys()[0]), Arrays.toString(info.getStartKey())); + Assert.assertEquals(Arrays.toString(locator.getEndKeys()[0]), Arrays.toString(info.getEndKey())); + for (int i = 1; i < locator.getStartKeys().length; i++) { + loc = locator.getRegionLocation(splitPoints[i - 1]); + info = loc.getRegion(); + Assert.assertEquals(Arrays.toString(locator.getStartKeys()[i]), Arrays.toString(info.getStartKey())); + Assert.assertEquals(Arrays.toString(locator.getEndKeys()[i]), Arrays.toString(info.getEndKey())); + } + } finally { + Optional.ofNullable(hTable).ifPresent(table -> { + try { + table.close(); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + } + + @Test + public void testKeyPartitionWithRegionLocator() throws IOException { + final String tableNameStr = "test_multi_cf"; + final TableName tableName = TableName.valueOf(tableNameStr); + final Configuration conf = ObHTableTestUtil.newConfiguration(); + connection = ConnectionFactory.createConnection(conf); + hTable = connection.getTable(tableName); + RegionLocator locator = connection.getRegionLocator(tableName); + byte[][] startKeys = locator.getStartKeys(); + byte[][] endKeys = locator.getEndKeys(); + Assert.assertEquals("Should have 1 region", 1, startKeys.length); + Assert.assertEquals("Should have 1 region", 1, endKeys.length); + Assert.assertEquals(startKeys[0], endKeys[0]); + Assert.assertEquals(startKeys[0], HConstants.EMPTY_BYTE_ARRAY); + } + + @Test + public void testRangeKeySecondaryPartitionWithRegionLocator() throws IOException { + final String tableNameStr = "test_secondary_range_key"; + final TableName tableName = TableName.valueOf(tableNameStr); + final Configuration conf = ObHTableTestUtil.newConfiguration(); + connection = ConnectionFactory.createConnection(conf); + hTable = connection.getTable(tableName); + RegionLocator locator = connection.getRegionLocator(tableName); + byte[][] startKeys = locator.getStartKeys(); + byte[][] endKeys = locator.getEndKeys(); + Assert.assertEquals("Should have 1 region", 1, startKeys.length); + Assert.assertEquals("Should have 1 region", 1, endKeys.length); + Assert.assertEquals(startKeys[0], endKeys[0]); + Assert.assertEquals(startKeys[0], HConstants.EMPTY_BYTE_ARRAY); + } + + @Test + public void testKeyRangeSecondaryPartitionWithRegionLocator() throws IOException { + final String tableNameStr = "test_secondary_key_range"; + final TableName tableName = TableName.valueOf(tableNameStr); + final Configuration conf = ObHTableTestUtil.newConfiguration(); + connection = ConnectionFactory.createConnection(conf); + hTable = connection.getTable(tableName); + RegionLocator locator = connection.getRegionLocator(tableName); + byte[][] startKeys = locator.getStartKeys(); + byte[][] endKeys = locator.getEndKeys(); + Assert.assertEquals("Should have 1 region", 1, startKeys.length); + Assert.assertEquals("Should have 1 region", 1, endKeys.length); + Assert.assertEquals(startKeys[0], endKeys[0]); + Assert.assertEquals(startKeys[0], HConstants.EMPTY_BYTE_ARRAY); + } + + @Test public void testBufferedMutatorPeriodicFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 6d53616b..0d13e35a 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -18,15 +18,35 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.hbase.util.ResultSetPrinter; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.exception.ObTableGetException; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; + +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import java.io.IOException; -import java.util.concurrent.Executors; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.*; +import java.util.concurrent.*; +import java.util.stream.Collectors; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.executeSQL; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.*; +import static org.junit.Assert.assertFalse; +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; public class OHTableAdminInterfaceTest { public OHTablePool setUpLoadPool() throws IOException { @@ -39,6 +59,28 @@ public OHTablePool setUpLoadPool() throws IOException { return ohTablePool; } + public static void openHbaseAdminDDL() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + String stmt = "ALTER SYSTEM SET _enable_kv_hbase_admin_ddl = true;"; + conn.createStatement().execute(stmt); + } + + public static void closeHbaseAdminDDL() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + String stmt = "ALTER SYSTEM SET _enable_kv_hbase_admin_ddl = false;"; + conn.createStatement().execute(stmt); + } + + @BeforeClass + public static void before() throws Exception { + openHbaseAdminDDL(); + } + + @AfterClass + public static void finish() throws Exception { + closeHbaseAdminDDL(); + } + public OHTablePool setUpPool() throws IOException { Configuration c = ObHTableTestUtil.newConfiguration(); OHTablePool ohTablePool = new OHTablePool(c, 10); @@ -48,6 +90,59 @@ public OHTablePool setUpPool() throws IOException { return ohTablePool; } + enum ErrSimPoint { + EN_CREATE_HTABLE_TG_FINISH_ERR(2621), + EN_CREATE_HTABLE_CF_FINISH_ERR(2622), + EN_DISABLE_HTABLE_CF_FINISH_ERR(2623), + EN_DELETE_HTABLE_CF_FINISH_ERR(2624), + EN_DELETE_HTABLE_SKIP_CF_ERR(2625); + + private final int errCode; + + ErrSimPoint(int errCode) { + this.errCode = errCode; + } + + public int getErrCode() { + return errCode; + } + } + + private void setErrSimPoint(ErrSimPoint errSimPoint, boolean enable) { + java.sql.Connection connection = null; + java.sql.Statement statement = null; + + try { + connection = ObHTableTestUtil.getSysTenantConnection(); + statement = connection.createStatement(); + + String sql = String.format( + "alter system set_tp tp_no = %d, error_code = 4016, frequency = %d", + errSimPoint.getErrCode(), + enable ? 1 : 0 + ); + + statement.execute(sql); + } catch (Exception e) { + throw new RuntimeException("Error injection setup failed", e); + } finally { + if (statement != null) { + try { + statement.close(); + } catch (Exception e) { + // ignore + } + } + if (connection != null) { + try { + connection.close(); + } catch (Exception e) { + // ignore + } + } + } + } + @Test public void testGetStartEndKeysOHTableClientRange() throws Exception { // Init OHTableClient @@ -249,4 +344,1358 @@ public void testGetStartEndKeysOHTablePoolLoadNon() throws Exception { Assert.assertEquals(0, startEndKeys.getFirst()[0].length); Assert.assertEquals(0, startEndKeys.getSecond()[0].length); } + + public static void createTable(Admin admin, TableName tableName, String... columnFamilies) + throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + // Add column families + for (String cf : columnFamilies) { + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(cf)); + htd.addFamily(hcd); + } + // Create the table + admin.createTable(htd); + } + + @Test + public void testAdminEnDisableTable() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + Statement st = conn.createStatement(); + st.execute("CREATE DATABASE IF NOT EXISTS `en_dis`"); + st.close(); + conn.close(); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + try { + createTable(admin, TableName.valueOf("test_en_dis_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("en_dis", "test"), "cf1", "cf2", "cf3"); + assertTrue(admin.tableExists(TableName.valueOf("en_dis", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); + String kvAttrStrDefault = "{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\"}}"; + String kvAttributeDisable = "{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"disable\"}}"; + String kvAttributeEnable = "{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"enable\"}}"; + + checkKVAttributes("test_en_dis_tb$cf1", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf2", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf3", kvAttrStrDefault); + // 1. disable a non-existed table + { + IOException thrown = assertThrows(IOException.class, + () -> { + admin.disableTable(TableName.valueOf("tablegroup_not_exists")); + }); + assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + } + // 2. write an enabled table, should succeed + { + if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { + admin.enableTable(TableName.valueOf("test_en_dis_tb")); + } + checkKVAttributes("test_en_dis_tb$cf1", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf2", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf3", kvAttrStrDefault); + + batchInsert(10, "test_en_dis_tb"); + batchGet(10, "test_en_dis_tb"); + } + + // 3. disable a enable table + { + if (admin.isTableEnabled(TableName.valueOf("test_en_dis_tb"))) { + admin.disableTable(TableName.valueOf("test_en_dis_tb")); + } + checkKVAttributes("test_en_dis_tb$cf1", kvAttributeDisable); + checkKVAttributes("test_en_dis_tb$cf2", kvAttributeDisable); + checkKVAttributes("test_en_dis_tb$cf3", kvAttributeDisable); + // write and read disable table, should fail + try { + batchInsert(10, "test_en_dis_tb"); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + System.out.println(ex.getCause().getMessage()); + } + try { + batchGet(10, "test_en_dis_tb"); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, + ((ObTableException) ex.getCause()).getErrorCode()); + } + + } + + // 4. enable a disabled table + { + if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { + admin.enableTable(TableName.valueOf("test_en_dis_tb")); + } + checkKVAttributes("test_en_dis_tb$cf1", kvAttributeEnable); + checkKVAttributes("test_en_dis_tb$cf2", kvAttributeEnable); + checkKVAttributes("test_en_dis_tb$cf3", kvAttributeEnable); + // write an enabled table, should succeed + batchInsert(10, "test_en_dis_tb"); + batchGet(10, "test_en_dis_tb"); + } + + // 5. enable an enabled table + { + if (admin.isTableDisabled(TableName.valueOf("en_dis", "test"))) { + admin.enableTable(TableName.valueOf("en_dis", "test")); + } + checkKVAttributes("en_dis:test$cf1", kvAttrStrDefault); + checkKVAttributes("en_dis:test$cf2", kvAttrStrDefault); + checkKVAttributes("en_dis:test$cf3", kvAttrStrDefault); + try { + admin.enableTable(TableName.valueOf("en_dis", "test")); + Assert.fail(); + } catch (Exception ex) { + Assert.assertEquals(TableNotDisabledException.class, ex.getClass()); + } + } + + // 6. disable a disabled table + { + if (admin.isTableEnabled(TableName.valueOf("en_dis", "test"))) { + admin.disableTable(TableName.valueOf("en_dis", "test")); + } + checkKVAttributes("en_dis:test$cf1", kvAttributeDisable); + checkKVAttributes("en_dis:test$cf1", kvAttributeDisable); + checkKVAttributes("en_dis:test$cf1", kvAttributeDisable); + try { + admin.disableTable(TableName.valueOf("en_dis", "test")); + Assert.fail(); + } catch (Exception ex) { + Assert.assertEquals(TableNotEnabledException.class, ex.getClass()); + } + } + + } catch (Exception e) { + e.printStackTrace(); + throw e; + } finally { + admin.deleteTable(TableName.valueOf("test_en_dis_tb")); + assertFalse(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); + admin.deleteTable(TableName.valueOf("en_dis", "test")); + assertFalse(admin.tableExists(TableName.valueOf("en_dis", "test"))); + } + } + + private void batchGet(int rows, String tablegroup) throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Table table = connection.getTable(TableName.valueOf(tablegroup)); + List batchLsit = new LinkedList<>(); + for (int i = 0; i < rows; ++i) { + Get get = new Get(toBytes("Key" + i)); + batchLsit.add(get); + if (i % 100 == 0) { // 100 rows one batch to avoid OB_TIMEOUT + Object[] results = new Object[batchLsit.size()]; + table.batch(batchLsit, results); + batchLsit.clear(); + } + } + Object[] results = new Object[batchLsit.size()]; + table.batch(batchLsit, results); + } + + @Test + public void testAdminGetRegionMetrics() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + Statement st = conn.createStatement(); + st.execute("CREATE TABLEGROUP IF NOT EXISTS test_get_region_metrics SHARDING = 'ADAPTIVE';\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_get_region_metrics$family_with_group1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_get_region_metrics PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_get_region_metrics$family_with_group2` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_get_region_metrics PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_get_region_metrics$family_with_group3` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_get_region_metrics PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE DATABASE IF NOT EXISTS `get_region`;\n" + + "use `get_region`;\n" + + "CREATE TABLEGROUP IF NOT EXISTS `get_region:test_multi_cf` SHARDING = 'ADAPTIVE';\n" + + "CREATE TABLE IF NOT EXISTS `get_region:test_multi_cf$family_with_group1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `get_region:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "CREATE TABLE IF NOT EXISTS `get_region:test_multi_cf$family_with_group2` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `get_region:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "CREATE TABLE IF NOT EXISTS `get_region:test_multi_cf$family_with_group3` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `get_region:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;"); + st.close(); + conn.close(); + String tablegroup1 = "test_get_region_metrics"; + String tablegroup2 = "get_region:test_multi_cf"; + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + // test tablegroup not existed + IOException thrown = assertThrows(IOException.class, + () -> { + admin.getRegionMetrics(null, TableName.valueOf("tablegroup_not_exists")); + }); + Assert.assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_HBASE_TABLE_NOT_EXISTS.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + + // test use serverName without tableName to get region metrics + assertThrows(FeatureNotSupportedException.class, + () -> { + admin.getRegionMetrics(ServerName.valueOf("localhost,1,1")); + }); + + // test single-thread getRegionMetrics after writing + batchInsert(100000, tablegroup1); + // test ServerName is any string + long start = System.currentTimeMillis(); + List metrics = admin.getRegionMetrics(ServerName.valueOf("localhost,1,1"), TableName.valueOf(tablegroup1)); + long cost = System.currentTimeMillis() - start; + System.out.println("get region metrics time usage: " + cost + "ms, tablegroup: " + tablegroup1); + assertEquals(30, metrics.size()); + + // test getRegionMetrics concurrently reading while writing + ExecutorService executorService = Executors.newFixedThreadPool(10); + CountDownLatch latch = new CountDownLatch(100); + List exceptionCatcher = new ArrayList<>(); + for (int i = 0; i < 100; ++i) { + int taskId = i; + executorService.submit(() -> { + try { + if (taskId % 2 == 1) { + List regionMetrics = null; + // test get regionMetrics from different namespaces + if (taskId % 3 != 0) { + long thrStart = System.currentTimeMillis(); + regionMetrics = admin.getRegionMetrics(null, TableName.valueOf(tablegroup1)); + long thrCost = System.currentTimeMillis() - thrStart; + System.out.println("task: " + taskId + ", get region metrics time usage: " + thrCost + "ms, tablegroup: " + tablegroup1); + if (regionMetrics.size() != 30) { + throw new ObTableGetException( + "the number of region metrics does not match the number of tablets, the number of region metrics: " + regionMetrics.size()); + } + } else { + long thrStart = System.currentTimeMillis(); + regionMetrics = admin.getRegionMetrics(null, TableName.valueOf(tablegroup2)); + long thrCost = System.currentTimeMillis() - thrStart; + System.out.println("task: " + taskId + ", get region metrics time usage: " + thrCost + "ms, tablegroup: " + tablegroup1); + if (regionMetrics.size() != 9) { + throw new ObTableGetException( + "the number of region metrics does not match the number of tablets, the number of region metrics: " + regionMetrics.size()); + } + } + } else { + try { + if (taskId % 8 == 0) { + batchInsert(1000, tablegroup2); + } else { + batchInsert(1000, tablegroup1); + } + } catch (Exception e) { + Exception originalCause = e; + while (originalCause.getCause() != null && originalCause.getCause() instanceof ObTableException) { + originalCause = (Exception) originalCause.getCause(); + } + if (originalCause instanceof ObTableException && ((ObTableException) originalCause).getErrorCode() == ResultCodes.OB_TIMEOUT.errorCode) { + // ignore + System.out.println("taskId: " + taskId + " OB_TIMEOUT"); + } else { + throw e; + } + } + System.out.println("task: " + taskId + ", batchInsert"); + } + } catch (Exception e) { + e.printStackTrace(); + exceptionCatcher.add(e); + } finally { + latch.countDown(); + } + }); + } + try { + latch.await(); + } catch (Exception e) { + e.printStackTrace(); + exceptionCatcher.add(e); + } + executorService.shutdownNow(); + Assert.assertTrue(exceptionCatcher.isEmpty()); + + // test getRegionMetrics from non-partitioned table + String non_part_tablegroup = "test_no_part"; + batchInsert(10000, non_part_tablegroup); + start = System.currentTimeMillis(); + metrics = admin.getRegionMetrics(null, TableName.valueOf(non_part_tablegroup)); + cost = System.currentTimeMillis() - start; + System.out.println("get region metrics time usage: " + cost + "ms, tablegroup: " + non_part_tablegroup); + assertEquals(3, metrics.size()); + } + + private void deleteTableIfExists(Admin admin, TableName tableName) throws Exception { + if (admin.tableExists(tableName)) { + if (admin.isTableEnabled(tableName)) { + admin.disableTable(tableName); + } + admin.deleteTable(tableName); + } + } + + private void deleteTableIfExists(Admin admin, String tableName) throws Exception { + deleteTableIfExists(admin, TableName.valueOf(tableName)); + } + + @Test + public void testAdminDeleteTable() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + Statement st = conn.createStatement(); + st.execute("CREATE DATABASE IF NOT EXISTS `del_tb`"); + st.close(); + conn.close(); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + try { + createTable(admin, TableName.valueOf("test_del_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("del_tb", "test"), "cf1", "cf2", "cf3"); + assertTrue(admin.tableExists(TableName.valueOf("del_tb", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("test_del_tb"))); + IOException thrown = assertThrows(IOException.class, + () -> { + admin.deleteTable(TableName.valueOf("tablegroup_not_exists")); + }); + Assert.assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + admin.deleteTable(TableName.valueOf("del_tb", "test")); + admin.deleteTable(TableName.valueOf("test_del_tb")); + assertFalse(admin.tableExists(TableName.valueOf("del_tb", "test"))); + assertFalse(admin.tableExists(TableName.valueOf("test_del_tb"))); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } finally { + deleteTableIfExists(admin, "test_del_tb"); + deleteTableIfExists(admin, "del_tb"); + } + } + + @Test + public void testAdminTableExists() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + Statement st = conn.createStatement(); + st.execute("CREATE DATABASE IF NOT EXISTS `exist_tb`"); + st.close(); + conn.close(); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + // TableName cannot contain $ symbol + Assert.assertThrows(IllegalArgumentException.class, + () -> { + TableName.valueOf("random_string$"); + }); + Assert.assertFalse(admin.tableExists(TableName.valueOf("tablegroup_not_exists"))); + createTable(admin, TableName.valueOf("test_exist_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("exist_tb", "test"), "cf1", "cf2", "cf3"); + Assert.assertTrue(admin.tableExists(TableName.valueOf("test_exist_tb"))); + Assert.assertTrue(admin.tableExists(TableName.valueOf("exist_tb", "test"))); + } + + private void batchInsert(int rows, String tablegroup) throws Exception { + byte[] family1 = Bytes.toBytes("cf1"); + byte[] family2 = Bytes.toBytes("cf2"); + byte[] family3 = Bytes.toBytes("cf3"); + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family2_column3 = "family2_column3".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family3_column2 = "family3_column2".getBytes(); + byte[] family3_column3 = "family3_column3".getBytes(); + byte[] family1_value1 = Bytes.toBytes("family1_value1"); + byte[] family1_value2 = Bytes.toBytes("family1_value2"); + byte[] family1_value3 = Bytes.toBytes("family1_value3"); + byte[] family2_value1 = Bytes.toBytes("family2_value1"); + byte[] family2_value2 = Bytes.toBytes("family2_value2"); + byte[] family2_value3 = Bytes.toBytes("family2_value3"); + byte[] family3_value1 = Bytes.toBytes("family3_value1"); + byte[] family3_value2 = Bytes.toBytes("family3_value2"); + byte[] family3_value3 = Bytes.toBytes("family3_value3"); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Table table = connection.getTable(TableName.valueOf(tablegroup)); + List batchLsit = new LinkedList<>(); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.addColumn(family1, family1_column1, family1_value1); + put.addColumn(family1, family1_column2, family1_value2); + put.addColumn(family1, family1_column3, family1_value3); + put.addColumn(family2, family2_column1, family2_value1); + put.addColumn(family2, family2_column2, family2_value2); + put.addColumn(family2, family2_column3, family2_value3); + put.addColumn(family3, family3_column1, family3_value1); + put.addColumn(family3, family3_column2, family3_value2); + put.addColumn(family3, family3_column3, family3_value3); + batchLsit.add(put); + if (i % 100 == 0) { // 100 rows one batch to avoid OB_TIMEOUT + Object[] results = new Object[batchLsit.size()]; + table.batch(batchLsit, results); + batchLsit.clear(); + } + } + Object[] results = new Object[batchLsit.size()]; + table.batch(batchLsit, results); + } + + @Test + public void testCreateDeleteTable() throws Exception { + TableName tableName = TableName.valueOf("testCreateTable"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + byte[] cf3 = Bytes.toBytes("cf3"); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + // 1. construct htable desc and column family desc + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + hcd1.setMaxVersions(2); + hcd1.setTimeToLive(172800); + + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + hcd2.setMaxVersions(1); + hcd2.setTimeToLive(86400); + + HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + + // 2. execute create table and check exists + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + htd.addFamily(hcd3); + admin.createTable(htd); + + // 3. check table creation success and correctness + assertTrue(admin.tableExists(tableName)); + // TODO: show create table, need to be replace by getDescriptor + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + String selectSql = "show create table " + tableName.getNameAsString() + "$" + Bytes.toString(cf1); + System.out.println("execute sql: " + selectSql); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery(selectSql); + ResultSetPrinter.print(resultSet); + + selectSql = "show create table " + tableName.getNameAsString() + "$" + Bytes.toString(cf2); + System.out.println("execute sql: " + selectSql); + resultSet = conn.createStatement().executeQuery(selectSql); + ResultSetPrinter.print(resultSet); + + selectSql = "show create table " + tableName.getNameAsString() + "$" + Bytes.toString(cf3); + System.out.println("execute sql: " + selectSql); + resultSet = conn.createStatement().executeQuery(selectSql); + ResultSetPrinter.print(resultSet); + + + // 4. test put/get some data + Table table = connection.getTable(tableName); + Put put = new Put(toBytes("Key" + 1)); + put.addColumn(cf1, "c1".getBytes(), "hello world".getBytes()); + put.addColumn(cf2, "c2".getBytes(), "hello world".getBytes()); + put.addColumn(cf3, "c3".getBytes(), "hello world".getBytes()); + table.put(put); + + Scan scan = new Scan(); + ResultScanner resultScanner = table.getScanner(scan); + List cells = getCellsFromScanner(resultScanner); + Assert.assertEquals(3, cells.size()); + + // 5. disable and delete table + admin.disableTable(tableName); + admin.deleteTable(tableName); + + // 5. test table exists after delete + admin.tableExists(tableName); + + // 6. recreate and delete table + admin.createTable(htd); + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + void testConcurCreateDelTablesHelper(List tableNames, Boolean ignoreException) throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + int tableNums = tableNames.size(); + + // use some column family desc + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + byte[] cf3 = Bytes.toBytes("cf3"); + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + hcd1.setMaxVersions(2); + hcd1.setTimeToLive(172800); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + hcd1.setMaxVersions(1); + hcd1.setTimeToLive(86400); + HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + List originalColumnDescriptors = Arrays.asList(hcd1, hcd2, hcd3); + + // 1. generate create table task, one task per table + List> tasks = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + int finalI = i; + tasks.add(()->{ + HTableDescriptor htd = new HTableDescriptor(tableNames.get(finalI)); + List columnDescriptors = new ArrayList<>(originalColumnDescriptors); + // 随机打乱列族顺序 + Collections.shuffle(columnDescriptors); + String shuffledCfNames = columnDescriptors.stream() + .map(hcd -> Bytes.toString(hcd.getName())) + .collect(Collectors.joining(", ")); + System.out.println("Table " + tableNames.get(finalI) + " shuffled column families: " + shuffledCfNames); + for (HColumnDescriptor hcd : columnDescriptors) { + htd.addFamily(hcd); + } + try { + admin.createTable(htd); + System.out.println("success to create table:" + tableNames.get(finalI)); + } catch (Exception e) { + System.out.println(e); + if (!ignoreException) { + throw e; + } + } + return null; + }); + } + + // 2. execute concurrent create table tasks + ExecutorService executorService = Executors.newFixedThreadPool(tableNums); + executorService.invokeAll(tasks); + executorService.shutdown(); + executorService.awaitTermination(1, TimeUnit.MINUTES); + + // 3. check table create success + for (int i = 0; i < tableNames.size(); i++) { + TableName tableName = tableNames.get(i); + assertTrue(admin.tableExists(tableName)); + } + + // 4. test put/get/delete some data + for (int i = 0; i < tableNames.size(); i++) { + Table table = connection.getTable(tableNames.get(i)); + Put put = new Put(toBytes("Key" + 1)); + put.addColumn(cf1, "c1".getBytes(), "hello world".getBytes()); + put.addColumn(cf2, "c2".getBytes(), "hello world".getBytes()); + put.addColumn(cf3, "c3".getBytes(), "hello world".getBytes()); + table.put(put); + + Scan scan = new Scan(); + ResultScanner resultScanner = table.getScanner(scan); + List cells = getCellsFromScanner(resultScanner); + Assert.assertEquals(3, cells.size()); + + table.delete(new Delete(toBytes("Key" + 1))); + } + + // 4. disable all tables; + List> disableTasks = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + int finalI = i; + disableTasks.add(()->{ + try { + admin.disableTable(tableNames.get(finalI)); + System.out.println("success to disable table:" + tableNames.get(finalI)); + } catch (Exception e) { + System.out.println(e); + if (!ignoreException) { + throw e; + } + } + return null; + }); + } + ExecutorService disExecutorService = Executors.newFixedThreadPool(tableNums); + disExecutorService.invokeAll(disableTasks); + disExecutorService.shutdown(); + disExecutorService.awaitTermination(1, TimeUnit.MINUTES); + + assertTrue(admin.isTableDisabled(tableNames.get(0))); + for (int i = 0; i < tableNames.size(); i++) { + TableName tableName = tableNames.get(i); + assertTrue(admin.isTableDisabled(tableName)); + } + + // 5. generate delete table task + List> delTasks = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + int finalI = i; + delTasks.add(()->{ + try { + admin.deleteTable(tableNames.get(finalI)); + System.out.println("success to drop table:" + tableNames.get(finalI)); + } catch (Exception e) { + System.out.println(e); + if (!ignoreException) { + throw e; + } + } + return null; + }); + } + + // 6. execute concurrent delete table tasks + ExecutorService delExecutorService = Executors.newFixedThreadPool(tableNums); + delExecutorService.invokeAll(delTasks); + delExecutorService.shutdown(); + delExecutorService.awaitTermination(1, TimeUnit.MINUTES); + + // 7. check table deletion success + for (int i = 0; i < tableNames.size(); i++) { + TableName tableName = tableNames.get(i); + assertFalse(admin.tableExists(tableName)); + } + } + + // 2. test concurrent create or delete different table + // step1: create different tables concurrently, it must succeed + // step2: execute put/read/delete on created table + // step3: delete different tables concurrently, it must succeed + @Test + public void testConcurCreateDelTables() throws Exception { + final int tableNums = 20; + List tableNames = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + tableNames.add(TableName.valueOf("testConcurCreateTable" + i)); + } + testConcurCreateDelTablesHelper(tableNames, false); + } + + // 3. test concurrent create or delete same table + // step1: create one table concurrently, only one table was successfully created + // step2: execute put/read/delete on created table + // step3: delete one table concurrently, the table will be deleted successfully + @Test + public void testConcurCreateOneTable() throws Exception { + final int taskNum = 20; + TableName tableName = TableName.valueOf("testConcurCreateOneTable"); + List tableNames = new ArrayList<>(); + for (int i = 0; i < taskNum; i++) { + tableNames.add(tableName); + } + testConcurCreateDelTablesHelper(tableNames, true); + } + + // 4. test the performance of concurrent create/delete table + @Test + public void testConcurCreateDelPerf() throws Exception { + final int tableNums = 100; + List tableNames = new ArrayList<>(); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + byte[] cf3 = Bytes.toBytes("cf3"); + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + hcd1.setMaxVersions(2); + hcd1.setTimeToLive(172800); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + hcd1.setMaxVersions(1); + hcd1.setTimeToLive(86400); + HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + + for (int i = 0; i < tableNums; i++) { + tableNames.add(TableName.valueOf("testConcurCreateDelPerf" + i)); + } + + List> tasks = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + int finalI = i; + tasks.add(()->{ + HTableDescriptor htd = new HTableDescriptor(tableNames.get(finalI)); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + htd.addFamily(hcd3); + try { + admin.createTable(htd); + } catch (Exception e) { + System.out.println(e); + } + return null; + }); + } + + // 2. execute concurrent create table tasks + long start = System.currentTimeMillis(); + ExecutorService executorService = Executors.newFixedThreadPool(tableNums); + executorService.invokeAll(tasks); + executorService.shutdown(); + executorService.awaitTermination(2, TimeUnit.MINUTES); + long duration = System.currentTimeMillis() - start; + System.out.println("create " + tableNums + " tables cost " + duration + " ms."); + + // 3. disable all tables; + for (int i = 0; i < tableNames.size(); i++) { + TableName tableName = tableNames.get(i); + admin.disableTable(tableName); + } + + // 4. generate delete table task + List> delTasks = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + int finalI = i; + delTasks.add(()->{ + try { + admin.deleteTable(tableNames.get(finalI)); + } catch (Exception e) { + System.out.println(e); + } + return null; + }); + } + + // 6. execute concurrent delete table tasks + start = System.currentTimeMillis(); + ExecutorService delExecutorService = Executors.newFixedThreadPool(tableNums); + delExecutorService.invokeAll(delTasks); + delExecutorService.shutdown(); + delExecutorService.awaitTermination(1, TimeUnit.MINUTES); + duration = System.currentTimeMillis() - start; + System.out.println("delete " + tableNums + " tables cost " + duration + " ms."); + } + + @Test + public void testHTableDDLDefense() throws Exception { + TableName tableName = TableName.valueOf("testHTableDefense"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + // 1. construct htable desc and column family desc + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + hcd1.setMaxVersions(2); + hcd1.setTimeToLive(172800); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + hcd1.setMaxVersions(1); + hcd1.setTimeToLive(86400); + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + + // 2. execute create table and check exists + try { + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + admin.createTable(htd); + assertTrue(admin.tableExists(tableName)); + + /// execute the following ddl stmt in created by admin table, should be prohibited + // 3. alter table add constraint + try { + executeSQL(conn, "alter table testHTableDefense$cf1 ADD CONSTRAINT cons1 CHECK(T < 0)", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + // 4. alter table add index + try { + executeSQL(conn, "alter table testHTableDefense$cf1 ADD INDEX idx_1(T)", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + // 5. alter table modify column to lob + try { + executeSQL(conn, "alter table testHTableDefense$cf1 MODIFY COLUMN V LONGTEXT", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + // 6. alter hbase admin table add fk + try { + executeSQL(conn, "alter table testHTableDefense$cf1 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(K) REFERENCES testHTableDefense$cf2(K)", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + // 7. create a normal table to refer to hbase admin table + try { + executeSQL(conn, "create table testHTableDefense_t1(a varbinary(1024) primary key, FOREIGN KEY(a) REFERENCES testHTableDefense$cf1(K));" , true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + // 8. alter a normal table to refer to hbase admin table + try { + executeSQL(conn, "create table testHTableDefense_t2(a varbinary(1024) primary key)", true); + executeSQL(conn, "alter table testHTableDefense_t2 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(a) REFERENCES testHTableDefense$cf1(K);", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + // 9. create a normal table A to refer to a table mock parent table B, and create table B using hbase admin + try { + executeSQL(conn, "SET foreign_key_checks = 0", true); + + executeSQL(conn, "create table testHTableDefense_t3(a varbinary(1024) primary key, FOREIGN KEY(a) REFERENCES testHTableDefense2$cf1(K));", true); + HTableDescriptor htd2 = new HTableDescriptor(TableName.valueOf("testHTableDefense2")); + HColumnDescriptor hcd4 = new HColumnDescriptor("cf1".getBytes()); + hcd4.setMaxVersions(2); + hcd4.setTimeToLive(172800); + htd2.addFamily(hcd4); + admin.createTable(htd2); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + } + + + // 10. create trigger + try { + executeSQL(conn, " CREATE TRIGGER hbase_trigger_1" + + " AFTER INSERT ON testHTableDefense$cf1 FOR EACH ROW" + + " BEGIN END", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + // 11. create view + try { + executeSQL(conn, " CREATE VIEW hbase_view_1 as select * from testHTableDefense$cf1", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + // 12. alter view + try { + executeSQL(conn, "ALTER VIEW hbase_view_1 as select * from testHTableDefense$cf1", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + // 13. create index + try { + executeSQL(conn, " CREATE INDEX testHTableDefense$cf1_idx_T on testHTableDefense$cf1(T)", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + + // 14. explicit create table and specify created_by:admin, should be prohibited + try { + executeSQL(conn, "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + // 15. alter table to created_by:admin, should be prohibited + try { + executeSQL(conn, "CREATE TABLE testHTableDefense$cf3(a int primary key)", true); + executeSQL(conn, "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("alter table kv attributes to created by admin not supported", e.getMessage()); + // clean table + String sql3 = "drop table if exists testHTableDefense$cf3"; + System.out.println("execute sql: " + sql3); + conn.createStatement().execute(sql3); + } + + // 16. disable a htable did not created by admin is not suppported + try { + executeSQL(conn, "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2", true); + executeSQL(conn, "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf4(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2", true); + admin.disableTable(TableName.valueOf("testHTableDefense2")); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + } + + // 17. delete a htable did not created by admin is not suppported + try { + executeSQL(conn, "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2", true); + executeSQL(conn, + "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf5(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2", true); + admin.deleteTable(TableName.valueOf("testHTableDefense2")); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + } + + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } finally { + admin.disableTable(tableName); + admin.deleteTable(tableName); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense2$cf4", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense2$cf5", true); + executeSQL(conn, "DROP TABLEGROUP IF EXISTS testHTableDefense2", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense_t1", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense_t2", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense_t3", true); + } + } + + void checkKVAttributes(String tableName, String kvAttributes) throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery( + "select kv_attributes from oceanbase.__all_table where table_name = '" + tableName + + "'"); + resultSet.next(); + String value = resultSet.getString(1); + Assert.assertEquals(kvAttributes, value); + Assert.assertFalse(resultSet.next()); + } + + // NOTE: observer should build with `-DOB_ERRSIM=ON` option, otherwise the test will fail + // This test verifies error injection scenarios for table operations + @Test + public void testCreateTableInjectError() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + byte[] tableName = Bytes.toBytes("test_create_table_inject_error"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + byte[] cf3 = Bytes.toBytes("cf3"); + + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + htd.addFamily(hcd3); + + try { + // 1. open err EN_CREATE_HTABLE_TG_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_TG_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.createTable(htd)); + assertFalse("Table should not exist after TG error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_TG_FINISH_ERR, false); + + // 2. open err EN_CREATE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_CF_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.createTable(htd)); + assertFalse("Table should not exist after CF error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_CF_FINISH_ERR, false); + + // 3. create table without error + admin.createTable(htd); + assertTrue("Table should exist after normal creation", + admin.tableExists(TableName.valueOf(tableName))); + assertEquals("Table should have 3 column families", 3, + admin.getTableDescriptor(TableName.valueOf(tableName)).getFamilies().size()); + + // 4. open err EN_DISABLE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_DISABLE_HTABLE_CF_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.disableTable(TableName.valueOf(tableName))); + assertFalse("Table should not be disabled after disable error injection", + admin.isTableDisabled(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_DISABLE_HTABLE_CF_FINISH_ERR, false); + + // 5. disable table without error + admin.disableTable(TableName.valueOf(tableName)); + assertTrue("Table should be disabled after normal disable", + admin.isTableDisabled(TableName.valueOf(tableName))); + + // 6. open err EN_DELETE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.deleteTable(TableName.valueOf(tableName))); + assertTrue("Table should still exist after delete error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, false); + + // 7. delete table without error + admin.deleteTable(TableName.valueOf(tableName)); + assertFalse("Table should not exist after normal delete", + admin.tableExists(TableName.valueOf(tableName))); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } finally { + if (admin.tableExists(TableName.valueOf(tableName))) { + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, false); + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_SKIP_CF_ERR, false); + admin.deleteTable(TableName.valueOf(tableName)); + } + } + } + + @Test + public void testHbaseAdminDDLKnob() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + try { + closeHbaseAdminDDL(); + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException) e.getCause()).getErrorCode()); + } + + try { + admin.disableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException) e.getCause()).getErrorCode()); + } + + try { + admin.enableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException) e.getCause()).getErrorCode()); + } + + try { + admin.deleteTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException) e.getCause()).getErrorCode()); + } + } catch (Exception e) { + e.printStackTrace(); + throw e; + } finally { + openHbaseAdminDDL(); + } + } + + @Test + public void testHbaseDDLException() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + // 1. create a created table + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableExistsException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 2. delete a non-exist table + try { + admin.deleteTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotFoundException.class); + } + + // 3. enable a enabled table + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + admin.enableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotDisabledException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 4. disable a disabled table + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + admin.disableTable(TableName.valueOf("t1")); + admin.disableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotEnabledException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 5. get htable descriptor from an uncreated table + try { + HTableDescriptor descriptor = admin.getTableDescriptor(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotFoundException.class); + } + + // 6. get region metrics from an uncreated table + try { + admin.getRegionMetrics(null, TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertEquals(e.getClass(), TableNotFoundException.class); + } + + // 6. create a table in an uncreated namespace + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableExistsException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 7. delete a non-exist table in an uncreated namespace + try { + admin.deleteTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotFoundException.class); + } + + // 8. enable a enabled table in an uncreated namespace + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + admin.enableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotDisabledException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 9. disable a disabled table in an uncreated namespace + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + admin.disableTable(TableName.valueOf("t1")); + admin.disableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotEnabledException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 10. get a table metrics from an uncreated namespace + try { + admin.getRegionMetrics(null, TableName.valueOf("n1:t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), NamespaceNotFoundException.class); + } + + // 11. check table exists from an uncreated namespace + try { + admin.tableExists(TableName.valueOf("n1:t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), NamespaceNotFoundException.class); + } + + } + + // Test cases for abnormal scene in CreateTableGroupHelper/DropTableGroupHelper + @Test + public void testCreateDropTableGroup() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + java.sql.Connection sysConn = ObHTableTestUtil.getSysConnection(); + String tenantName = "mysql"; + + byte[] tableName = Bytes.toBytes("test_create_drop_tg_helper"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + byte[] cf3 = Bytes.toBytes("cf3"); + + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + htd.addFamily(hcd3); + + try { + admin.createTable(htd); + + // 1. open err EN_DELETE_HTABLE_SKIP_CF_ERR, will skip delete cf table when delete hbase table + // and the subsequent delete htable operations will return OB_TABLEGROUP_NOT_EMPTY + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_SKIP_CF_ERR, true); + ObHTableTestUtil.executeIgnoreExpectedErrors(() -> admin.deleteTable(TableName.valueOf(tableName)), "OB_TABLEGROUP_NOT_EMPTY"); + assertTrue("Table should still exist after delete error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_SKIP_CF_ERR, false); + + // 2. create a database and set default tablegroup to test_create_drop_tg_helper, + // and the subsequent delete htable operation will return OB_TABLEGROUP_NOT_EMPTY + executeSQL(conn, "create database db_test_create_drop_tg_helper default tablegroup test_create_drop_tg_helper", true); + ObHTableTestUtil.executeIgnoreExpectedErrors(() -> admin.deleteTable(TableName.valueOf(tableName)), "OB_TABLEGROUP_NOT_EMPTY"); + executeSQL(conn, "drop database db_test_create_drop_tg_helper", true); + + // 3. set tenant's default tablegroup to test_create_drop_tg_helper, + // and the subsequent delete htable operation will return OB_TABLEGROUP_NOT_EMPTY + executeSQL(sysConn, String.format("alter tenant %s set default tablegroup = test_create_drop_tg_helper", tenantName), true); + ObHTableTestUtil.executeIgnoreExpectedErrors(() -> admin.deleteTable(TableName.valueOf(tableName)), "OB_TABLEGROUP_NOT_EMPTY"); + executeSQL(sysConn, String.format("alter tenant %s set default tablegroup = null", tenantName), true); + + } catch (Exception e) { + e.printStackTrace(); + throw e; + } finally { + executeSQL(conn, "drop database if exists db_test_create_drop_tg_helper", true); + executeSQL(sysConn, String.format("alter tenant %s set default tablegroup = null", tenantName), true); + if (admin.tableExists(TableName.valueOf(tableName))) { + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_SKIP_CF_ERR, false); + admin.deleteTable(TableName.valueOf(tableName)); + } + } + } + + private void checkDDLStmtStr(List> ddlStmts) throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery( + String.format("select operation_type, ddl_stmt_str from oceanbase.__all_ddl_operation order by gmt_modified desc limit %d", ddlStmts.size() + 2)); + Assert.assertTrue(resultSet.next()); + int operationType = resultSet.getInt("operation_type"); + Assert.assertEquals(1503, operationType); + String ddlStmtStr = resultSet.getString("ddl_stmt_str"); + Assert.assertEquals("", ddlStmtStr); + + for (int i = 0; i < ddlStmts.size(); i++) { + Assert.assertTrue(resultSet.next()); + operationType = resultSet.getInt("operation_type"); + Assert.assertEquals(ddlStmts.get(i).getKey().intValue(), operationType); + ddlStmtStr = resultSet.getString("ddl_stmt_str"); + Assert.assertEquals(ddlStmts.get(i).getValue(), ddlStmtStr); + } + Assert.assertTrue(resultSet.next()); + operationType = resultSet.getInt("operation_type"); + Assert.assertEquals(1503, operationType); + ddlStmtStr = resultSet.getString("ddl_stmt_str"); + Assert.assertEquals("", ddlStmtStr); + Assert.assertFalse(resultSet.next()); + } + + @Test + public void testDDLStmtStr() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + byte[] tableName = Bytes.toBytes("test_ddl_stmt_str"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + + try { + // 1. create hbase table + admin.createTable(htd); + List> expStmtStrs = new ArrayList<>(); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(4, "CREATE TABLE `test_ddl_stmt_str$cf1` (K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, T bigint NOT NULL, V varbinary(1048576) NOT NULL, " + + "PRIMARY KEY (K, Q, T)) TABLEGROUP = `test_ddl_stmt_str` kv_attributes = '{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\"}}'")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(4, "CREATE TABLE `test_ddl_stmt_str$cf2` (K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, T bigint NOT NULL, V varbinary(1048576) NOT NULL, " + + "PRIMARY KEY (K, Q, T)) TABLEGROUP = `test_ddl_stmt_str` kv_attributes = '{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\"}}'")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(302, "CREATE TABLEGROUP `test_ddl_stmt_str`")); + checkDDLStmtStr(expStmtStrs); + + // 2. disable hbase table + admin.disableTable(TableName.valueOf(tableName)); + expStmtStrs.clear(); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(3, "ALTER TABLE test_ddl_stmt_str$cf1 KV_ATTRIBUTES='{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"disable\"}}'")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(3, "ALTER TABLE test_ddl_stmt_str$cf2 KV_ATTRIBUTES='{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"disable\"}}'")); + checkDDLStmtStr(expStmtStrs); + + // 3. enable hbase table + admin.enableTable(TableName.valueOf(tableName)); + expStmtStrs.clear(); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(3, "ALTER TABLE test_ddl_stmt_str$cf1 KV_ATTRIBUTES='{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"enable\"}}'")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(3, "ALTER TABLE test_ddl_stmt_str$cf2 KV_ATTRIBUTES='{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"enable\"}}'")); + checkDDLStmtStr(expStmtStrs); + + // 4. delete hbase table + admin.disableTable(TableName.valueOf(tableName)); + admin.deleteTable(TableName.valueOf(tableName)); + expStmtStrs.clear(); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(303, "DROP TABLEGROUP `test_ddl_stmt_str`")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(2, "DROP TABLE `test`.`test_ddl_stmt_str$cf1`")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(2, "DROP TABLE `test`.`test_ddl_stmt_str$cf2`")); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } finally { + if (admin.tableExists(TableName.valueOf(tableName))) { + if (admin.isTableEnabled(TableName.valueOf(tableName))) { + admin.disableTable(TableName.valueOf(tableName)); + } + admin.deleteTable(TableName.valueOf(tableName)); + } + } + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java index 233e31ed..ce1bc3a1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java @@ -18,6 +18,9 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.junit.*; import java.util.LinkedList; @@ -67,6 +70,87 @@ public void testNew() throws Exception { assertTrue(true); } + + /* + CREATE TABLEGROUP test_desc SHARDING = 'ADAPTIVE'; + CREATE TABLE `test_desc$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = test_desc + KV_ATTRIBUTES ='{"Hbase": {"TimeToLive": 3600, "MaxVersions": 3}}' + PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) + ); + + CREATE TABLE `test_desc$family2` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = test_desc + KV_ATTRIBUTES ='{"Hbase": {"TimeToLive": 7200, "MaxVersions": 3}}' + PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) + ); + */ + @Test + public void testGetTableDescriptor() throws Exception { + final String tableNameStr = "test_desc"; + + OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient(tableNameStr); + hTable2.init(); + try { + { + HTableDescriptor descriptor1 = hTable2.getTableDescriptor(); + Assert.assertNotNull(descriptor1); + Assert.assertEquals(2, descriptor1.getColumnFamilyCount()); + Assert.assertTrue(descriptor1.hasFamily("family1".getBytes())); + Assert.assertTrue(descriptor1.hasFamily("family2".getBytes())); + Assert.assertFalse(descriptor1.hasFamily("family".getBytes())); + ColumnFamilyDescriptor family1 = descriptor1.getColumnFamily("family1".getBytes()); + ColumnFamilyDescriptor family2 = descriptor1.getColumnFamily("family2".getBytes()); + Assert.assertEquals(3600, family1.getTimeToLive()); + Assert.assertEquals(7200, family2.getTimeToLive()); + } + { + TableDescriptor descriptor2 = hTable2.getDescriptor(); + Assert.assertNotNull(descriptor2); + Assert.assertEquals(2, descriptor2.getColumnFamilyCount()); + Assert.assertTrue(descriptor2.hasColumnFamily("family1".getBytes())); + Assert.assertTrue(descriptor2.hasColumnFamily("family2".getBytes())); + Assert.assertFalse(descriptor2.hasColumnFamily("family".getBytes())); + ColumnFamilyDescriptor family1 = descriptor2.getColumnFamily("family1".getBytes()); + ColumnFamilyDescriptor family2 = descriptor2.getColumnFamily("family2".getBytes()); + Assert.assertEquals(3600, family1.getTimeToLive()); + Assert.assertEquals(7200, family2.getTimeToLive()); + } + } finally { + hTable2.close(); + } + } + @AfterClass public static void finish() throws Exception { hTable.close(); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java index fef4dce0..32449005 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.junit.*; @@ -330,4 +331,259 @@ public void testMultiCFPut() throws Throwable { public void testMultiCFPutBatch() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMltiCFPutBatchImpl); } + + @Test + public void testPutOpt() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartPutTest::testPutOptImpl); + FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMultiCFPutOptImpl); + } + + public static void testPutOptImpl(String tableName) throws Exception { + int NUM_QUALIFIERS = 10; + int NUM_PUTS = 10; + byte[] family = toBytes(getColumnFamilyName(tableName)); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + { // 单put，10个Qualifier + Put put = new Put(toBytes("row1")); + List qualifiers = new ArrayList<>(); + for (int i = 1; i <= NUM_QUALIFIERS; i++) { + byte[] qualifier = toBytes("q" + i); + byte[] value = toBytes("value" + i); + put.addColumn(family, qualifier, value); + qualifiers.add(Bytes.toString(qualifier)); + } + hTable.put(put); + // verify + Get get = new Get(Bytes.toBytes("row1")); + Result result = hTable.get(get); + Assert.assertEquals(NUM_QUALIFIERS, result.getFamilyMap(family).size()); + qualifiers.forEach(q -> + Assert.assertNotNull("Qualifier " + q + " not found", + result.getValue(family, Bytes.toBytes(q))) + ); + } + + { // 单put，10个Qualifier，指定相同timestamp + Put put = new Put(toBytes("row2")); + List qualifiers = new ArrayList<>(); + long timestamp = System.currentTimeMillis(); + for (int i = 1; i <= NUM_QUALIFIERS; i++) { + byte[] qualifier = toBytes("q" + i); + byte[] value = toBytes("value" + i); + put.addColumn(family, qualifier, timestamp, value); + qualifiers.add(Bytes.toString(qualifier)); + } + hTable.put(put); + // verify + Get get = new Get(Bytes.toBytes("row2")); + Result result = hTable.get(get); + Assert.assertEquals(NUM_QUALIFIERS, result.getFamilyMap(family).size()); + qualifiers.forEach(q -> + Assert.assertNotNull("Qualifier " + q + " not found", + result.getValue(family, Bytes.toBytes(q))) + ); + } + { // batch put，相同key， 多个Qualifier + byte[] rowKey = Bytes.toBytes("batch_row"); + List puts = new ArrayList<>(); + List qualifiers = new ArrayList<>(); + for (int i = 1; i <= NUM_PUTS; i++) { + Put put = new Put(rowKey); + byte[] qualifier = Bytes.toBytes("batch_q" + i); + put.addColumn(family, qualifier, Bytes.toBytes("batch_val" + i)); + puts.add(put); + qualifiers.add(Bytes.toString(qualifier)); + } + hTable.put(puts); + // verify + Get get = new Get(rowKey); + Result result = hTable.get(get); + Assert.assertEquals(NUM_QUALIFIERS, result.getFamilyMap(family).size()); + qualifiers.forEach(q -> + Assert.assertNotNull("Qualifier " + q + " not found", + result.getValue(family, Bytes.toBytes(q))) + ); + } + + { // batch put，相同key，多个Qualifier，指定相同timestamp + byte[] rowKey = Bytes.toBytes("batch_row_ts"); + List puts = new ArrayList<>(); + List qualifiers = new ArrayList<>(); + for (int i = 1; i <= NUM_PUTS; i++) { + Put put = new Put(rowKey); + byte[] qualifier = Bytes.toBytes("batch_ts_q" + i); + put.addColumn(family, qualifier, Bytes.toBytes("batch_val" + i)); + puts.add(put); + qualifiers.add(Bytes.toString(qualifier)); + } + hTable.put(puts); + // verify + Get get = new Get(rowKey); + Result result = hTable.get(get); + Assert.assertEquals(NUM_QUALIFIERS, result.getFamilyMap(family).size()); + qualifiers.forEach(q -> + Assert.assertNotNull("Qualifier " + q + " not found", + result.getValue(family, Bytes.toBytes(q))) + ); + } + } + + public static void testMultiCFPutOptImpl(Map.Entry> entry) throws Exception { + int NUM_QUALIFIERS = 10; + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); + hTable.init(); + { + Put put = new Put(toBytes("multi_cf_row")); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + // 单put，10个Qualifier + List qualifiers = new ArrayList<>(); + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] qualifier = toBytes("q" + i); + byte[] value = toBytes("value_" + i); + put.addColumn(family, qualifier, value); + qualifiers.add(Bytes.toString(qualifier)); + } + } + hTable.put(put); + // verify + Get get = new Get(Bytes.toBytes("multi_cf_row")); + Result result = hTable.get(get); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Assert.assertEquals(NUM_QUALIFIERS, + result.getFamilyMap(family).size()); + + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] val = result.getValue(family, Bytes.toBytes("q" + i)); + Assert.assertEquals( "value_" + i, + Bytes.toString(val)); + } + } + } + + { // 指定timestamp + long currentTimestamp = System.currentTimeMillis(); + Put put = new Put(toBytes("multi_cf_ts_row")); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + // 单put，10个Qualifier + List qualifiers = new ArrayList<>(); + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] qualifier = toBytes("q" + i); + byte[] value = toBytes("value_" + i); + put.addColumn(family, qualifier, currentTimestamp, value); + qualifiers.add(Bytes.toString(qualifier)); + } + } + hTable.put(put); + // verify + Get get = new Get(Bytes.toBytes("multi_cf_ts_row")); + Result result = hTable.get(get); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Assert.assertEquals(NUM_QUALIFIERS, + result.getFamilyMap(family).size()); + + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] val = result.getValue(family, Bytes.toBytes("q" + i)); + Assert.assertEquals( "value_" + i, + Bytes.toString(val)); + } + } + } + + { + byte[] ROW = Bytes.toBytes("batch_row"); + List puts = new ArrayList<>(); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Put put = new Put(ROW); + for (int i = 0; i < NUM_QUALIFIERS; i++) { + put.addColumn(family, + Bytes.toBytes("q" + i), + Bytes.toBytes("value_" + i)); + } + puts.add(put); + } + hTable.put(puts); + // verify + Get get = new Get(ROW); + Result result = hTable.get(get); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Assert.assertEquals(NUM_QUALIFIERS, + result.getFamilyMap(family).size()); + + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] val = result.getValue(family, Bytes.toBytes("q" + i)); + Assert.assertEquals( "value_" + i, + Bytes.toString(val)); + } + } + } + + { + long timestamp = System.currentTimeMillis(); + byte[] ROW = Bytes.toBytes("batch_ts_row"); + List puts = new ArrayList<>(); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + for (int i = 0; i < NUM_QUALIFIERS; i++) { + Put put = new Put(ROW); + put.addColumn(family, + Bytes.toBytes("q" + i), + timestamp, + Bytes.toBytes("value_" + i)); + puts.add(put); + } + } + hTable.put(puts); + // verify + Get get = new Get(ROW); + Result result = hTable.get(get); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Assert.assertEquals(NUM_QUALIFIERS, + result.getFamilyMap(family).size()); + + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] val = result.getValue(family, Bytes.toBytes("q" + i)); + Assert.assertEquals( "value_" + i, + Bytes.toString(val)); + } + } + } + + { + byte[] ROW = Bytes.toBytes("batch_row_2"); + List puts = new ArrayList<>(); + for (int i = 0; i < NUM_QUALIFIERS; i++) { + Put put = new Put(ROW); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + put.addColumn(family, + Bytes.toBytes("q" + i), + Bytes.toBytes("value_" + i)); + } + puts.add(put); + } + hTable.put(puts); + // verify + Get get = new Get(ROW); + Result result = hTable.get(get); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Assert.assertEquals(NUM_QUALIFIERS, + result.getFamilyMap(family).size()); + + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] val = result.getValue(family, Bytes.toBytes("q" + i)); + Assert.assertEquals( "value_" + i, + Bytes.toString(val)); + } + } + } + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java new file mode 100644 index 00000000..2af2bd53 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java @@ -0,0 +1,187 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_QUERY_HOT_ONLY; +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.*; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + + +public class OHTableShareStorageSeriesTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableType type : NORMAL_SERIES_PARTITIONED_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + for (TableType type : NORMAL_SERIES_PARTITIONED_TABLES) { + alterTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.addColumn(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. get, expect less than 100 key + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + } + Assert.assertTrue(getCount < recordCount); + + hTable.close(); + } + + public static void testScanImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.addColumn(family.getBytes(), (column).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. scan not specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 2. scan specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 3. scan specify versions + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setMaxVersions(2); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 4. scan specify time range + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setMaxVersions(2); + scan.setTimeStamp(curTs); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + hTable.close(); + } + + @Test + public void testGet() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageSeriesTest::testGetImpl); + } + + + @Test + public void testScan() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageSeriesTest::testScanImpl); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java new file mode 100644 index 00000000..d15aaf5d --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java @@ -0,0 +1,558 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.*; +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_OCEANBASE_SYS_PASSWORD; +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.ODP_MODE; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.*; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableShareStorageTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : NORMAL_PARTITIONED_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + for (TableTemplateManager.TableType type : NORMAL_PARTITIONED_TABLES) { + alterTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.addColumn(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. get, expect less than 100 key + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + } + System.out.println("getCount:" + getCount); + Assert.assertTrue(getCount < recordCount); + + hTable.close(); + } + + public static void testMultiCFGetImpl(Map.Entry> entry) throws Exception { + // 0. prepare data + long recordCount = 100; + String groupName = getTableName(entry.getKey()); + List tableNames = entry.getValue(); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.addColumn(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + } + + // 1. get specify column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + } + Assert.assertTrue(getCount < recordCount); + } + } + + // 2. get do not specify column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addFamily(family.getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + } + Assert.assertTrue(getCount < recordCount); + } + } + + // 3. get do not specify column family + { + long getCount = 0; + Get get = new Get(key.getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + Assert.assertTrue(getCount < recordCount); + } + + // 4. get specify multi cf and column + { + long getCount = 0; + Get get = new Get(key.getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addColumn(family.getBytes(), column.getBytes()); + } + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + Assert.assertTrue(getCount < recordCount); + } + + hTable.close(); + } + + public static void testBatchGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.addColumn(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value)); + hTable.put(put); + } + + List gets = new ArrayList<>(); + // 1. get, expect less than 100 key + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + String getKey = null; + if (i == 4) { + getKey = key + 101; + } else { + getKey = key + i; + } + Get get = new Get((getKey + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + gets.add(get); + } + Result[] results = hTable.get(gets); + for (Result result : results) { + getCount += result.size(); + } + Assert.assertTrue(getCount < recordCount); + + hTable.close(); + } + + public static void testScanImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.addColumn(family.getBytes(), (column).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. scan not specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 2. scan specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 3. scan specify versions + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + + // 4. scan specify filter + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + scan.addFamily(family.getBytes()); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(value))); + scan.setFilter(valueFilter); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 5. scan in reverse + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addFamily(family.getBytes()); + scan.setReversed(true); + try { + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getMessage().contains("secondary partitioned hbase table with reverse query not supported")); + } + } + hTable.close(); + } + + public static void testMultiCFScanImpl(Map.Entry> entry) throws Exception { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + List tableNames = entry.getValue(); + + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.addColumn(family.getBytes(), (column).getBytes(), curTs, toBytes(value)); + hTable.put(put); + } + } + + // 1. multi cf scan specify one cf and one column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + } + + // 2. multi cf scan specify one cf without specify column + { + for (String tableName : tableNames) { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + String family = getColumnFamilyName(tableName); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + } + + // 3. multi cf scan do not specify cf + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 4. multi cf scan specify multi cf and multi column + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + scan.addColumn(family.getBytes(), column.getBytes()); + } + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 5. multi cf scan specify versions + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 6. multi cf scan specify time range + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + scan.setTimeStamp(curTs); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 7. multi cf scan specify filter + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(value))); + scan.setFilter(valueFilter); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 8. multi cf scan using setStartRow/setEndRow + { + Scan scan = new Scan(); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setStartRow(key.getBytes()); + scan.setStopRow((key + recordCount).getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 9. multi cf scan using batch + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setBatch(2); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 10. multi cf scan with family scan and column-specific scan + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + for (int i = 0; i < tableNames.size(); i++) { + String family = getColumnFamilyName(tableNames.get(i)); + if (i % 2 == 0) { + scan.addFamily(family.getBytes()); + } else { + scan.addColumn(family.getBytes(), column.getBytes()); + } + } + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + hTable.close(); + } + + @Test + public void testGet() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageTest::testGetImpl); + } + + @Test + public void testMultiCFGet() throws Throwable { + FOR_EACH(group2tableNames, OHTableShareStorageTest::testMultiCFGetImpl); + } + + @Test + public void testBatchGet() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageTest::testBatchGetImpl); + } + + @Test + public void testScan() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageTest::testScanImpl); + } + + @Test + public void testMultiCFScan() throws Throwable { + FOR_EACH(group2tableNames, OHTableShareStorageTest::testMultiCFScanImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesDeleteTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesDeleteTest.java index 5669da70..3e85b904 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesDeleteTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesDeleteTest.java @@ -75,7 +75,7 @@ public static void testDeleteColumnImpl(String tableName) throws Exception { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[i].getBytes(), ts[i], toBytes(values[i])); + put.addColumn(family.getBytes(), columns[i].getBytes(), ts[i], toBytes(values[i])); hTable.put(put); } @@ -126,7 +126,7 @@ public static void testDeleteColumnsImpl(String tableName) throws Exception { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[i].getBytes(), ts[i] + i, toBytes(values[i] + i)); + put.addColumn(family.getBytes(), columns[i].getBytes(), ts[i] + i, toBytes(values[i] + i)); hTable.put(put); } @@ -177,7 +177,7 @@ public static void testDeleteFamilyImpl(String tableName) throws Exception { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[i].getBytes(), ts[i] + i, toBytes(values[i] + i)); + put.addColumn(family.getBytes(), columns[i].getBytes(), ts[i] + i, toBytes(values[i] + i)); hTable.put(put); } @@ -229,8 +229,8 @@ public static void testDeleteFamilyWithTimestampImpl(String tableName) throws Ex for (int i = 0; i < 3; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[0].getBytes(), curTs + i, toBytes(value + i)); - put.add(family.getBytes(), columns[1].getBytes(), curTs + i, toBytes(value + i)); + put.addColumn(family.getBytes(), columns[0].getBytes(), curTs + i, toBytes(value + i)); + put.addColumn(family.getBytes(), columns[1].getBytes(), curTs + i, toBytes(value + i)); hTable.put(put); } @@ -264,8 +264,8 @@ public static void testDeleteFamilyVersionImpl(String tableName) throws Exceptio for (int i = 0; i < 3; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[0].getBytes(), curTs + i, toBytes(value + i)); - put.add(family.getBytes(), columns[1].getBytes(), curTs + i, toBytes(value + i)); + put.addColumn(family.getBytes(), columns[0].getBytes(), curTs + i, toBytes(value + i)); + put.addColumn(family.getBytes(), columns[1].getBytes(), curTs + i, toBytes(value + i)); hTable.put(put); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index 693fa35a..2ee37e4e 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -57,6 +57,20 @@ public static void createTables(TableTemplateManager.TableType type, List tableNames, + Map> group2tableNames, boolean printSql) + throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // single cf table + if (tableNames != null) { + alterTables(conn, type, tableNames, printSql); + } + // multi cf table + if (group2tableNames != null) { + alterTables(conn, type, group2tableNames, printSql); + } + } + public static void createTables(Connection conn, TableTemplateManager.TableType type, List tableNames, boolean printSql) throws Exception { // create single cf table @@ -85,6 +99,31 @@ public static void createTables(Connection conn, TableTemplateManager.TableType } } + public static void alterTables(Connection conn, TableTemplateManager.TableType type, + List tableNames, boolean printSql) throws Exception { + // create single cf table + if (tableNames != null) { + String tableGroup = TableTemplateManager.getTableGroupName(type, false); + String tableName = TableTemplateManager.generateTableName(tableGroup, false, 1); + String sql = TableTemplateManager.getAlterTableSQL(type, tableName); + try { + System.out.println(sql); + conn.createStatement().execute(sql); + System.out.println("============= alter table: " + tableName + " table_group: " + + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table: " + tableName + " table_group: " + + getTableName(tableName) + " alter failed ============="); + } + } + } + } + public static void createTables(Connection conn, TableTemplateManager.TableType type, Map> group2tableNames, boolean printSql) throws Exception { if (group2tableNames != null) { TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); @@ -121,6 +160,30 @@ public static void createTables(Connection conn, TableTemplateManager.TableType } } + public static void alterTables(Connection conn, TableTemplateManager.TableType type, Map> group2tableNames, boolean printSql) throws Exception { + if (group2tableNames != null) { + String tableGroup = TableTemplateManager.getTableGroupName(type, true); + group2tableNames.put(tableGroup, new LinkedList<>()); + for (int i = 1; i <= 3; ++i) { + String tableName = TableTemplateManager.generateTableName(tableGroup, true, i); + String sql = TableTemplateManager.getAlterTableSQL(type, tableName); + try { + conn.createStatement().execute(sql); + System.out.println("============= alter table: " + tableName + + " table_group: " + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table: " + tableName + " table_group: " + getTableName(tableName) + " alter failed ============="); + } + } + } + } + } + public static void truncateTables(List tableNames, Map> group2tableNames) throws Exception { Connection conn = ObHTableTestUtil.getConnection(); diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index 81fd3688..ef6cf8e3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -55,7 +55,13 @@ public class ObHTableTestUtil { + "oceanbase?" + "useUnicode=TRUE&" + "characterEncoding=utf-8&" + "socketTimeout=3000000&" + "connectTimeout=60000"; + public static String SYS_TENANT_JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + + "oceanbase?" + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + "connectTimeout=60000"; + public static String SYS_TENANT_USER_NAME = "root@sys"; + public static String SYS_TENANT_PASSWORD = ""; public static String SQL_FORMAT = "truncate %s"; public static List tableNameList = new LinkedList(); public static Connection conn; @@ -103,6 +109,8 @@ public static Configuration newConfiguration() { Configuration conf = HBaseConfiguration.create(); conf.set(HBASE_OCEANBASE_FULL_USER_NAME, FULL_USER_NAME); conf.set(HBASE_OCEANBASE_PASSWORD, PASSWORD); + conf.set("rpc.execute.timeout", "20000"); + conf.set("rpc.operation.timeout", "18000"); if (ODP_MODE) { // ODP mode conf.set(HBASE_OCEANBASE_ODP_ADDR, ODP_ADDR); @@ -170,6 +178,17 @@ static public Connection getSysConnection() { } } + static public Connection getSysTenantConnection() { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + Connection conn = DriverManager + .getConnection(SYS_TENANT_JDBC_URL, SYS_TENANT_USER_NAME, SYS_TENANT_PASSWORD); + return conn; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @FunctionalInterface public interface CheckedConsumer { void accept(T t) throws Throwable; @@ -227,4 +246,46 @@ public static boolean secureCompare(byte[] a, byte[] b) { } return diff == 0; } -} \ No newline at end of file + + public static void executeSQL(Connection conn, String sql, boolean printSQL) throws SQLException { + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } + + @FunctionalInterface + public interface CheckedRunnable { + void run() throws Exception; + } + + public static void executeIgnoreUnexpectedError(CheckedRunnable operation) throws Exception { + executeIgnoreExpectedErrors(operation, "OB_ERR_UNEXPECTED"); + } + + public static void executeIgnoreExpectedErrors(CheckedRunnable operation, String... expectedErrorMessages) throws Exception { + try { + operation.run(); + } catch (Exception e) { + boolean shouldIgnore = false; + String[] messagesToCheck = { + e.getMessage(), + e.getCause() != null ? e.getCause().getMessage() : null + }; + + for (String expectedMessage : expectedErrorMessages) { + for (String actualMessage : messagesToCheck) { + if (actualMessage != null && actualMessage.contains(expectedMessage)) { + shouldIgnore = true; + break; + } + } + if (shouldIgnore) { + break; + } + } + + if (!shouldIgnore) { + throw e; + } + } + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index 75a3e32d..2263bfcc 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -59,6 +59,19 @@ public enum TableType { SECONDARY_PARTITIONED_TIME_RANGE_KEY, SECONDARY_PARTITIONED_TIME_KEY_RANGE); + public static List NORMAL_PARTITIONED_TABLES = Arrays + .asList( + SINGLE_PARTITIONED_REGULAR, + SECONDARY_PARTITIONED_RANGE_KEY, + SECONDARY_PARTITIONED_RANGE_KEY_GEN, + SECONDARY_PARTITIONED_KEY_RANGE, + SECONDARY_PARTITIONED_KEY_RANGE_GEN); + public static List NORMAL_SERIES_PARTITIONED_TABLES = Arrays + .asList( + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + public static List SERIES_TABLES = Arrays .asList( NON_PARTITIONED_TIME_SERIES, @@ -284,6 +297,57 @@ public enum TableType { + ") TABLEGROUP = %s"); } + private static final Map ALTER_SQL_TEMPLATES = new EnumMap( + TableType.class); + + static { + // 普通表一级分区模板 + ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR, + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + // 时序表一级分区模板 + ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_TIME_SERIES, + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + // 普通表RANGE-KEY分区（使用K） + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 合并GEN类型的注释处理 + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 普通表KEY-RANGE分区（使用K） + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 普通表KEY-RANGE分区（使用生成列） + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 时序表RANGE-KEY分区 + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 时序表KEY-RANGE分区 + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_KEY_RANGE, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + /* ------------------ CELL TTL ----------------*/ + ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR_CELL_TTL, + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + } + public static String getCreateTableSQL(TableType type, String tableName, TimeGenerator.TimeRange timeRange) { System.out.println(tableName); @@ -329,6 +393,12 @@ public static String getCreateTableSQL(TableType type, String tableName, return String.format(template, params); } + public static String getAlterTableSQL(TableType type, String tableName) { + System.out.println(tableName); + String template = ALTER_SQL_TEMPLATES.get(type); + return String.format(template, tableName); + } + private static String getGeneratedColumn(TableType type) { StringBuilder sb = new StringBuilder(); boolean needsKPrefix = type.name().startsWith("SECONDARY_PARTITIONED") diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index 11a06bde..3fd00d75 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -276,3 +276,109 @@ CREATE TABLE `n1:test$partitionFamily1` ( `V` varbinary(1024) DEFAULT NULL, PRIMARY KEY (`K`, `Q`, `T`) ) partition by key(`K`) partitions 17; + +CREATE TABLEGROUP test_region_locator SHARDING = 'ADAPTIVE'; +CREATE TABLE `test_region_locator$family_region_locator` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_region_locator PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) +); + +CREATE TABLEGROUP test_desc SHARDING = 'ADAPTIVE'; +CREATE TABLE `test_desc$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_desc +KV_ATTRIBUTES ='{"Hbase": {"TimeToLive": 3600, "MaxVersions": 3}}' +PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) +); + +CREATE TABLE `test_desc$family2` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_desc +KV_ATTRIBUTES ='{"Hbase": {"TimeToLive": 7200, "MaxVersions": 3}}' +PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) +); + +CREATE TABLEGROUP test_secondary_key_range SHARDING = 'ADAPTIVE'; +CREATE TABLE IF NOT EXISTS `test_secondary_key_range$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + `G` bigint(20) GENERATED ALWAYS AS (ABS(T)), + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_secondary_key_range PARTITION BY KEY(`K`) PARTITIONS 3 +SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE ( + SUBPARTITION `p1` VALUES LESS THAN (100), + SUBPARTITION `p2` VALUES LESS THAN (200), + SUBPARTITION `p3` VALUES LESS THAN MAXVALUE +); +CREATE TABLE IF NOT EXISTS `test_secondary_key_range$family2` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + `G` bigint(20) GENERATED ALWAYS AS (ABS(T)), + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_secondary_key_range PARTITION BY KEY(`K`) PARTITIONS 3 +SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE ( + SUBPARTITION `p1` VALUES LESS THAN (100), + SUBPARTITION `p2` VALUES LESS THAN (200), + SUBPARTITION `p3` VALUES LESS THAN MAXVALUE +); + +CREATE TABLEGROUP test_secondary_range_key SHARDING = 'ADAPTIVE'; +CREATE TABLE IF NOT EXISTS `test_secondary_range_key$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + `G` bigint(20) GENERATED ALWAYS AS (ABS(T)), + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_secondary_range_key PARTITION BY RANGE COLUMNS(`G`) +SUBPARTITION BY KEY(`K`) SUBPARTITIONS 3 +( PARTITION `p1` VALUES LESS THAN (100), + PARTITION `p2` VALUES LESS THAN (200), + PARTITION `p3` VALUES LESS THAN MAXVALUE +);