From 8554f316e233e890a2e4ae895648b3b13c97f819 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Thu, 15 May 2025 10:51:48 +0800 Subject: [PATCH 01/36] MetaExecutor interface definition and demo --- .../execute/AbstractObTableMetaExecutor.java | 31 +++++++ .../hbase/execute/ObTableMetaExecutor.java | 32 +++++++ .../hbase/util/OHConnectionImpl.java | 5 +- .../oceanbase/hbase/util/OHRegionLocator.java | 84 +++++++++++++++++++ .../hbase/util/OHRegionLocatorExecutor.java | 49 +++++++++++ 5 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java create mode 100644 src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java new file mode 100644 index 00000000..83495c19 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java @@ -0,0 +1,31 @@ +package com.alipay.oceanbase.hbase.execute; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.table.ObTable; + +import java.io.IOException; + +public abstract class AbstractObTableMetaExecutor implements ObTableMetaExecutor { + + @Override + public T execute(ObTableClient client, ObTableMetaRequest request) throws IOException { + if (request.getMetaType() != getMetaType()) { + throw new IOException("Invalid meta type, expected " + getMetaType()); + } + ObTable table = client.randomTable(); + ObTableMetaResponse response; + try { + response = (ObTableMetaResponse) client.executeWithRetry( + table, + request, + null /*tableName*/ + ); + } catch (Exception e) { + throw new IOException("Failed to execute request", e); + } + return parse(response); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java new file mode 100644 index 00000000..4e8a0ffb --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java @@ -0,0 +1,32 @@ +package com.alipay.oceanbase.hbase.execute; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; + +import java.io.IOException; + +public interface ObTableMetaExecutor { + /** + * 执行元数据请求 + * @param request 元数据请求 + * @return 解析后的元数据对象 + * @throws IOException 如果执行失败或解析失败 + */ + T execute(ObTableClient client, ObTableMetaRequest request) throws IOException; + + /** + * 解析元数据响应, 用户需要重写 + * @param response 元数据响应 + * @return 解析后的元数据对象 + * @throws IOException 如果解析失败 + */ + T parse(ObTableMetaResponse response) throws IOException; + + /** + * 获取元信息类型, 用户需要重写 + * @return 元信息类型 + */ + ObTableRpcMetaType getMetaType() throws IOException; +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index 41a0690a..545efa18 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -19,6 +19,7 @@ import com.alipay.oceanbase.hbase.OHTable; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.rpc.ObTableClient; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; @@ -140,7 +141,9 @@ public BufferedMutator getBufferedMutator(BufferedMutatorParams params) throws I @Override public RegionLocator getRegionLocator(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("not supported yet'"); + ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClient(null/*args*/); + OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(obTableClient); + return executor.getRegionLocator(String.valueOf(tableName)); } @Override diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java new file mode 100644 index 00000000..f4474d92 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java @@ -0,0 +1,84 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.rpc.ObTableClient; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.util.Pair; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class OHRegionLocator implements RegionLocator { + public OHRegionLocator(byte[][] startKeys, byte[][] endKeys) { + + } + + @Override + public HRegionLocation getRegionLocation(byte[] bytes) throws IOException { + return null; + } + + @Override + public HRegionLocation getRegionLocation(byte[] bytes, boolean b) throws IOException { + return null; + } + + @Override + public List getAllRegionLocations() throws IOException { + return Collections.emptyList(); + } + + /** + * Gets the starting row key for every region in the currently open table. + *

+ * This is mainly useful for the MapReduce integration. + * + * @return Array of region starting row keys + * @throws IOException if a remote or network exception occurs + */ + @Override + public byte[][] getStartKeys() throws IOException { + return null; + } + + /** + * Gets the ending row key for every region in the currently open table. + *

+ * This is mainly useful for the MapReduce integration. + * + * @return Array of region ending row keys + * @throws IOException if a remote or network exception occurs + */ + @Override + public byte[][] getEndKeys() throws IOException { + return null; + } + + /** + * Gets the starting and ending row keys for every region in the currently + * open table. + *

+ * This is mainly useful for the MapReduce integration. + * + * @return Pair of arrays of region starting and ending row keys + * @throws IOException if a remote or network exception occurs + */ + @Override + public Pair getStartEndKeys() throws IOException { + return null; + } + + @Override + public TableName getName() { + return null; + } + + private ObTableClient tableClient; + + @Override + public void close() throws IOException { + + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java new file mode 100644 index 00000000..0f896827 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java @@ -0,0 +1,49 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alibaba.fastjson.JSON; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.location.model.TableEntry; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class OHRegionLocatorExecutor extends AbstractObTableMetaExecutor { + OHRegionLocatorExecutor(ObTableClient client) { + this.client = client; + } + + @Override + public ObTableRpcMetaType getMetaType() { + return ObTableRpcMetaType.HTABLE_REGION_LOCATOR; + } + + @Override + public OHRegionLocator parse(ObTableMetaResponse response) throws IOException { + try { + String jsonData = response.getData(); + // process json + return new OHRegionLocator(null, null); + } catch (IllegalArgumentException e) { + throw new IOException("msg", e); + } + } + + public OHRegionLocator getRegionLocator(String tableName) throws IOException { + ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestData = new HashMap<>(); + requestData.put("table_name", tableName); + String jsonData = JSON.toJSONString(requestData); + request.setData(jsonData); + + return execute(client, request); + } + + private final ObTableClient client; +} From 633d3493705027f73f78b81c5fd77608a931cfeb Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 20 May 2025 11:02:46 +0800 Subject: [PATCH 02/36] delete table and table exists function --- .../execute/AbstractObTableMetaExecutor.java | 2 +- .../alipay/oceanbase/hbase/util/OHAdmin.java | 739 ++++++++++++++++++ .../hbase/util/OHConnectionImpl.java | 5 +- .../hbase/util/OHDeleteTableExecutor.java | 43 + .../hbase/util/OHTableExistsExecutor.java | 43 + 5 files changed, 829 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java index 83495c19..d6d99da1 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java @@ -15,7 +15,7 @@ public T execute(ObTableClient client, ObTableMetaRequest request) throws IOExce if (request.getMetaType() != getMetaType()) { throw new IOException("Invalid meta type, expected " + getMetaType()); } - ObTable table = client.randomTable(); + ObTable table = client.getRandomTable(); ObTableMetaResponse response; try { response = (ObTableMetaResponse) client.executeWithRetry( diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java new file mode 100644 index 00000000..9cdde55b --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -0,0 +1,739 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.bolt.transport.TransportCodes; +import com.alipay.oceanbase.rpc.exception.ObTableTransportException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.security.SecurityCapability; +import org.apache.hadoop.hbase.exceptions.TimeoutIOException; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos; +import org.apache.hadoop.hbase.quotas.QuotaFilter; +import org.apache.hadoop.hbase.quotas.QuotaRetriever; +import org.apache.hadoop.hbase.quotas.QuotaSettings; +import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; +import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException; +import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; +import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; +import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException; +import org.apache.hadoop.hbase.util.Pair; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; +import java.util.regex.Pattern; + +public class OHAdmin implements Admin { + private final ObTableClient tableClient; + OHAdmin(ObTableClient tableClient) { + this.tableClient = tableClient; + } + + @Override + public int getOperationTimeout() { + return 0; + } + + @Override + public void abort(String s, Throwable throwable) { + + } + + @Override + public boolean isAborted() { + return false; + } + + @Override + public Connection getConnection() { + return null; + } + + @Override + public boolean tableExists(TableName tableName) throws IOException { + OHTableExistsExecutor executor = new OHTableExistsExecutor(tableClient); + return executor.tableExists(tableName.getNameAsString()); + } + + @Override + public HTableDescriptor[] listTables() throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public HTableDescriptor[] listTables(Pattern pattern) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public HTableDescriptor[] listTables(String s) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public HTableDescriptor[] listTables(Pattern pattern, boolean b) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public HTableDescriptor[] listTables(String s, boolean b) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public TableName[] listTableNames() throws IOException { + return new TableName[0]; + } + + @Override + public TableName[] listTableNames(Pattern pattern) throws IOException { + return new TableName[0]; + } + + @Override + public TableName[] listTableNames(String s) throws IOException { + return new TableName[0]; + } + + @Override + public TableName[] listTableNames(Pattern pattern, boolean b) throws IOException { + return new TableName[0]; + } + + @Override + public TableName[] listTableNames(String s, boolean b) throws IOException { + return new TableName[0]; + } + + @Override + public HTableDescriptor getTableDescriptor(TableName tableName) throws TableNotFoundException, IOException { + return null; + } + + @Override + public void createTable(HTableDescriptor hTableDescriptor) throws IOException { + + } + + @Override + public void createTable(HTableDescriptor hTableDescriptor, byte[] bytes, byte[] bytes1, int i) throws IOException { + + } + + @Override + public void createTable(HTableDescriptor hTableDescriptor, byte[][] bytes) throws IOException { + + } + + @Override + public void createTableAsync(HTableDescriptor hTableDescriptor, byte[][] bytes) throws IOException { + + } + + @Override + public void deleteTable(TableName tableName) throws IOException { + OHDeleteTableExecutor executor = new OHDeleteTableExecutor(tableClient); + try { + executor.deleteTable(tableName.getNameAsString()); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } + } + + @Override + public HTableDescriptor[] deleteTables(String s) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public HTableDescriptor[] deleteTables(Pattern pattern) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public void truncateTable(TableName tableName, boolean b) throws IOException { + + } + + @Override + public void enableTable(TableName tableName) throws IOException { + + } + + @Override + public void enableTableAsync(TableName tableName) throws IOException { + + } + + @Override + public HTableDescriptor[] enableTables(String s) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public HTableDescriptor[] enableTables(Pattern pattern) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public void disableTableAsync(TableName tableName) throws IOException { + + } + + @Override + public void disableTable(TableName tableName) throws IOException { + + } + + @Override + public HTableDescriptor[] disableTables(String s) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public HTableDescriptor[] disableTables(Pattern pattern) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public boolean isTableEnabled(TableName tableName) throws IOException { + return false; + } + + @Override + public boolean isTableDisabled(TableName tableName) throws IOException { + return false; + } + + @Override + public boolean isTableAvailable(TableName tableName) throws IOException { + return false; + } + + @Override + public boolean isTableAvailable(TableName tableName, byte[][] bytes) throws IOException { + return false; + } + + @Override + public Pair getAlterStatus(TableName tableName) throws IOException { + return null; + } + + @Override + public Pair getAlterStatus(byte[] bytes) throws IOException { + return null; + } + + @Override + public void addColumn(TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException { + + } + + @Override + public void deleteColumn(TableName tableName, byte[] bytes) throws IOException { + + } + + @Override + public void modifyColumn(TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException { + + } + + @Override + public void closeRegion(String s, String s1) throws IOException { + + } + + @Override + public void closeRegion(byte[] bytes, String s) throws IOException { + + } + + @Override + public boolean closeRegionWithEncodedRegionName(String s, String s1) throws IOException { + return false; + } + + @Override + public void closeRegion(ServerName serverName, HRegionInfo hRegionInfo) throws IOException { + + } + + @Override + public List getOnlineRegions(ServerName serverName) throws IOException { + return Collections.emptyList(); + } + + @Override + public void flush(TableName tableName) throws IOException { + + } + + @Override + public void flushRegion(byte[] bytes) throws IOException { + + } + + @Override + public void compact(TableName tableName) throws IOException { + + } + + @Override + public void compactRegion(byte[] bytes) throws IOException { + + } + + @Override + public void compact(TableName tableName, byte[] bytes) throws IOException { + + } + + @Override + public void compactRegion(byte[] bytes, byte[] bytes1) throws IOException { + + } + + @Override + public void majorCompact(TableName tableName) throws IOException { + + } + + @Override + public void majorCompactRegion(byte[] bytes) throws IOException { + + } + + @Override + public void majorCompact(TableName tableName, byte[] bytes) throws IOException { + + } + + @Override + public void majorCompactRegion(byte[] bytes, byte[] bytes1) throws IOException { + + } + + @Override + public void compactRegionServer(ServerName serverName, boolean b) throws IOException, InterruptedException { + + } + + @Override + public void move(byte[] bytes, byte[] bytes1) throws IOException { + + } + + @Override + public void assign(byte[] bytes) throws IOException { + + } + + @Override + public void unassign(byte[] bytes, boolean b) throws IOException { + + } + + @Override + public void offline(byte[] bytes) throws IOException { + + } + + @Override + public boolean setBalancerRunning(boolean b, boolean b1) throws IOException { + return false; + } + + @Override + public boolean balancer() throws IOException { + return false; + } + + @Override + public boolean balancer(boolean b) throws IOException { + return false; + } + + @Override + public boolean isBalancerEnabled() throws IOException { + return false; + } + + @Override + public boolean normalize() throws IOException { + return false; + } + + @Override + public boolean isNormalizerEnabled() throws IOException { + return false; + } + + @Override + public boolean setNormalizerRunning(boolean b) throws IOException { + return false; + } + + @Override + public boolean enableCatalogJanitor(boolean b) throws IOException { + return false; + } + + @Override + public int runCatalogScan() throws IOException { + return 0; + } + + @Override + public boolean isCatalogJanitorEnabled() throws IOException { + return false; + } + + @Override + public void mergeRegions(byte[] bytes, byte[] bytes1, boolean b) throws IOException { + + } + + @Override + public void split(TableName tableName) throws IOException { + + } + + @Override + public void splitRegion(byte[] bytes) throws IOException { + + } + + @Override + public void split(TableName tableName, byte[] bytes) throws IOException { + + } + + @Override + public void splitRegion(byte[] bytes, byte[] bytes1) throws IOException { + + } + + @Override + public void modifyTable(TableName tableName, HTableDescriptor hTableDescriptor) throws IOException { + + } + + @Override + public void shutdown() throws IOException { + + } + + @Override + public void stopMaster() throws IOException { + + } + + @Override + public void stopRegionServer(String s) throws IOException { + + } + + @Override + public ClusterStatus getClusterStatus() throws IOException { + return null; + } + + @Override + public Configuration getConfiguration() { + return null; + } + + @Override + public void createNamespace(NamespaceDescriptor namespaceDescriptor) throws IOException { + + } + + @Override + public void modifyNamespace(NamespaceDescriptor namespaceDescriptor) throws IOException { + + } + + @Override + public void deleteNamespace(String s) throws IOException { + + } + + @Override + public NamespaceDescriptor getNamespaceDescriptor(String s) throws NamespaceNotFoundException, IOException { + return null; + } + + @Override + public NamespaceDescriptor[] listNamespaceDescriptors() throws IOException { + return new NamespaceDescriptor[0]; + } + + @Override + public HTableDescriptor[] listTableDescriptorsByNamespace(String s) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public TableName[] listTableNamesByNamespace(String s) throws IOException { + return new TableName[0]; + } + + @Override + public List getTableRegions(TableName tableName) throws IOException { + return Collections.emptyList(); + } + + @Override + public void close() throws IOException { + + } + + @Override + public HTableDescriptor[] getTableDescriptorsByTableName(List list) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public HTableDescriptor[] getTableDescriptors(List list) throws IOException { + return new HTableDescriptor[0]; + } + + @Override + public boolean abortProcedure(long l, boolean b) throws IOException { + return false; + } + + @Override + public ProcedureInfo[] listProcedures() throws IOException { + return new ProcedureInfo[0]; + } + + @Override + public Future abortProcedureAsync(long l, boolean b) throws IOException { + return null; + } + + @Override + public void rollWALWriter(ServerName serverName) throws IOException, FailedLogCloseException { + + } + + @Override + public String[] getMasterCoprocessors() throws IOException { + return new String[0]; + } + + @Override + public AdminProtos.GetRegionInfoResponse.CompactionState getCompactionState(TableName tableName) throws IOException { + return null; + } + + @Override + public AdminProtos.GetRegionInfoResponse.CompactionState getCompactionStateForRegion(byte[] bytes) throws IOException { + return null; + } + + @Override + public long getLastMajorCompactionTimestamp(TableName tableName) throws IOException { + return 0; + } + + @Override + public long getLastMajorCompactionTimestampForRegion(byte[] bytes) throws IOException { + return 0; + } + + @Override + public void snapshot(String s, TableName tableName) throws IOException, SnapshotCreationException, IllegalArgumentException { + + } + + @Override + public void snapshot(byte[] bytes, TableName tableName) throws IOException, SnapshotCreationException, IllegalArgumentException { + + } + + @Override + public void snapshot(String s, TableName tableName, HBaseProtos.SnapshotDescription.Type type) throws IOException, SnapshotCreationException, IllegalArgumentException { + + } + + @Override + public void snapshot(HBaseProtos.SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException, IllegalArgumentException { + + } + + @Override + public MasterProtos.SnapshotResponse takeSnapshotAsync(HBaseProtos.SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException { + return null; + } + + @Override + public boolean isSnapshotFinished(HBaseProtos.SnapshotDescription snapshotDescription) throws IOException, HBaseSnapshotException, UnknownSnapshotException { + return false; + } + + @Override + public void restoreSnapshot(byte[] bytes) throws IOException, RestoreSnapshotException { + + } + + @Override + public void restoreSnapshot(String s) throws IOException, RestoreSnapshotException { + + } + + @Override + public void restoreSnapshot(byte[] bytes, boolean b) throws IOException, RestoreSnapshotException { + + } + + @Override + public void restoreSnapshot(String s, boolean b) throws IOException, RestoreSnapshotException { + + } + + @Override + public void cloneSnapshot(byte[] bytes, TableName tableName) throws IOException, TableExistsException, RestoreSnapshotException { + + } + + @Override + public void cloneSnapshot(String s, TableName tableName) throws IOException, TableExistsException, RestoreSnapshotException { + + } + + @Override + public void execProcedure(String s, String s1, Map map) throws IOException { + + } + + @Override + public byte[] execProcedureWithRet(String s, String s1, Map map) throws IOException { + return new byte[0]; + } + + @Override + public boolean isProcedureFinished(String s, String s1, Map map) throws IOException { + return false; + } + + @Override + public List listSnapshots() throws IOException { + return Collections.emptyList(); + } + + @Override + public List listSnapshots(String s) throws IOException { + return Collections.emptyList(); + } + + @Override + public List listSnapshots(Pattern pattern) throws IOException { + return Collections.emptyList(); + } + + @Override + public List listTableSnapshots(String s, String s1) throws IOException { + return Collections.emptyList(); + } + + @Override + public List listTableSnapshots(Pattern pattern, Pattern pattern1) throws IOException { + return Collections.emptyList(); + } + + @Override + public void deleteSnapshot(byte[] bytes) throws IOException { + + } + + @Override + public void deleteSnapshot(String s) throws IOException { + + } + + @Override + public void deleteSnapshots(String s) throws IOException { + + } + + @Override + public void deleteSnapshots(Pattern pattern) throws IOException { + + } + + @Override + public void deleteTableSnapshots(String s, String s1) throws IOException { + + } + + @Override + public void deleteTableSnapshots(Pattern pattern, Pattern pattern1) throws IOException { + + } + + @Override + public void setQuota(QuotaSettings quotaSettings) throws IOException { + + } + + @Override + public QuotaRetriever getQuotaRetriever(QuotaFilter quotaFilter) throws IOException { + return null; + } + + @Override + public CoprocessorRpcChannel coprocessorService() { + return null; + } + + @Override + public CoprocessorRpcChannel coprocessorService(ServerName serverName) { + return null; + } + + @Override + public void updateConfiguration(ServerName serverName) throws IOException { + + } + + @Override + public void updateConfiguration() throws IOException { + + } + + @Override + public int getMasterInfoPort() throws IOException { + return 0; + } + + @Override + public List getSecurityCapabilities() throws IOException { + return Collections.emptyList(); + } + + @Override + public boolean[] setSplitOrMergeEnabled(boolean b, boolean b1, MasterSwitchType... masterSwitchTypes) throws IOException { + return new boolean[0]; + } + + @Override + public boolean isSplitOrMergeEnabled(MasterSwitchType masterSwitchType) throws IOException { + return false; + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index 545efa18..de6ae338 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -141,14 +141,15 @@ public BufferedMutator getBufferedMutator(BufferedMutatorParams params) throws I @Override public RegionLocator getRegionLocator(TableName tableName) throws IOException { - ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClient(null/*args*/); + ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClient(connectionConfig); OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(obTableClient); return executor.getRegionLocator(String.valueOf(tableName)); } @Override public Admin getAdmin() throws IOException { - throw new FeatureNotSupportedException("not supported yet'"); + ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClient(connectionConfig); + return new OHAdmin(obTableClient); } private void shutdownBatchPool(ExecutorService pool) { diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java new file mode 100644 index 00000000..af97b7a3 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java @@ -0,0 +1,43 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.annotation.JSONField; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class OHDeleteTableExecutor extends AbstractObTableMetaExecutor { + private final ObTableClient tableClient; + + OHDeleteTableExecutor(ObTableClient tableClient) { + this.tableClient = tableClient; + } + + @Override + public ObTableRpcMetaType getMetaType() { + return ObTableRpcMetaType.HTABLE_DELETE_TABLE; + } + + + @Override + public Void parse(ObTableMetaResponse response) throws IOException { + // do nothing, error will be thrown from table + return null; + } + + public Void deleteTable(String tableName) throws IOException { + ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestDataMap = new HashMap<>(); + requestDataMap.put("name", tableName); + String jsonData = JSON.toJSONString(requestDataMap); + request.setData(jsonData); + return execute(tableClient, request); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java new file mode 100644 index 00000000..a7a7af35 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java @@ -0,0 +1,43 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class OHTableExistsExecutor extends AbstractObTableMetaExecutor { + private final ObTableClient tableClient; + + OHTableExistsExecutor(ObTableClient tableClient) { + this.tableClient = tableClient; + } + + @Override + public ObTableRpcMetaType getMetaType() throws IOException { + return ObTableRpcMetaType.HTABLE_EXISTS; + } + + @Override + public Boolean parse(ObTableMetaResponse response) throws IOException { + String jsonData = response.getData(); + JSONObject object = JSONObject.parseObject(jsonData); + return object.getBoolean("exists"); + } + + public Boolean tableExists(String tableName) throws IOException { + ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestData = new HashMap<>(); + requestData.put("name", tableName); + String jsonData = JSON.toJSONString(requestData); + request.setData(jsonData); + return execute(tableClient, request); + } +} From aa5bc40140f18c14608b56baf4767064d192faa8 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 21 May 2025 14:11:30 +0800 Subject: [PATCH 03/36] region metrics test --- .../alipay/oceanbase/hbase/util/OHAdmin.java | 418 +++++++++++++++--- .../oceanbase/hbase/util/OHRegionMetrics.java | 120 +++++ .../hbase/util/OHRegionMetricsExecutor.java | 71 +++ .../hbase/OHTableAdminInterfaceTest.java | 124 ++++++ 4 files changed, 678 insertions(+), 55 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index 9cdde55b..89c4e408 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -2,21 +2,22 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.bolt.transport.TransportCodes; +import com.alipay.oceanbase.rpc.exception.FeatureNotSupportedException; import com.alipay.oceanbase.rpc.exception.ObTableTransportException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.client.replication.TableCFs; import org.apache.hadoop.hbase.client.security.SecurityCapability; import org.apache.hadoop.hbase.exceptions.TimeoutIOException; import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.quotas.QuotaFilter; import org.apache.hadoop.hbase.quotas.QuotaRetriever; import org.apache.hadoop.hbase.quotas.QuotaSettings; import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException; +import org.apache.hadoop.hbase.replication.ReplicationException; +import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; @@ -24,9 +25,7 @@ import org.apache.hadoop.hbase.util.Pair; import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.Future; import java.util.regex.Pattern; @@ -67,11 +66,21 @@ public HTableDescriptor[] listTables() throws IOException { return new HTableDescriptor[0]; } + @Override + public List listTableDescriptors() throws IOException { + return Collections.emptyList(); + } + @Override public HTableDescriptor[] listTables(Pattern pattern) throws IOException { return new HTableDescriptor[0]; } + @Override + public List listTableDescriptors(Pattern pattern) throws IOException { + return Collections.emptyList(); + } + @Override public HTableDescriptor[] listTables(String s) throws IOException { return new HTableDescriptor[0]; @@ -82,6 +91,11 @@ public HTableDescriptor[] listTables(Pattern pattern, boolean b) throws IOExcept return new HTableDescriptor[0]; } + @Override + public List listTableDescriptors(Pattern pattern, boolean b) throws IOException { + return Collections.emptyList(); + } + @Override public HTableDescriptor[] listTables(String s, boolean b) throws IOException { return new HTableDescriptor[0]; @@ -118,25 +132,30 @@ public HTableDescriptor getTableDescriptor(TableName tableName) throws TableNotF } @Override - public void createTable(HTableDescriptor hTableDescriptor) throws IOException { - + public TableDescriptor getDescriptor(TableName tableName) throws TableNotFoundException, IOException { + return null; } @Override - public void createTable(HTableDescriptor hTableDescriptor, byte[] bytes, byte[] bytes1, int i) throws IOException { + public void createTable(TableDescriptor tableDescriptor) throws IOException { } @Override - public void createTable(HTableDescriptor hTableDescriptor, byte[][] bytes) throws IOException { + public void createTable(TableDescriptor tableDescriptor, byte[] bytes, byte[] bytes1, int i) throws IOException { } @Override - public void createTableAsync(HTableDescriptor hTableDescriptor, byte[][] bytes) throws IOException { + public void createTable(TableDescriptor tableDescriptor, byte[][] bytes) throws IOException { } + @Override + public Future createTableAsync(TableDescriptor tableDescriptor, byte[][] bytes) throws IOException { + return null; + } + @Override public void deleteTable(TableName tableName) throws IOException { OHDeleteTableExecutor executor = new OHDeleteTableExecutor(tableClient); @@ -144,7 +163,7 @@ public void deleteTable(TableName tableName) throws IOException { executor.deleteTable(tableName.getNameAsString()); } catch (IOException e) { if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { throw new TimeoutIOException(e.getCause()); } else { throw e; @@ -152,6 +171,11 @@ public void deleteTable(TableName tableName) throws IOException { } } + @Override + public Future deleteTableAsync(TableName tableName) throws IOException { + return null; + } + @Override public HTableDescriptor[] deleteTables(String s) throws IOException { return new HTableDescriptor[0]; @@ -167,14 +191,19 @@ public void truncateTable(TableName tableName, boolean b) throws IOException { } + @Override + public Future truncateTableAsync(TableName tableName, boolean b) throws IOException { + return null; + } + @Override public void enableTable(TableName tableName) throws IOException { } @Override - public void enableTableAsync(TableName tableName) throws IOException { - + public Future enableTableAsync(TableName tableName) throws IOException { + return null; } @Override @@ -188,8 +217,8 @@ public HTableDescriptor[] enableTables(Pattern pattern) throws IOException { } @Override - public void disableTableAsync(TableName tableName) throws IOException { - + public Future disableTableAsync(TableName tableName) throws IOException { + return null; } @Override @@ -238,8 +267,13 @@ public Pair getAlterStatus(byte[] bytes) throws IOException { } @Override - public void addColumn(TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException { + public void addColumnFamily(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { + + } + @Override + public Future addColumnFamilyAsync(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { + return null; } @Override @@ -248,10 +282,25 @@ public void deleteColumn(TableName tableName, byte[] bytes) throws IOException { } @Override - public void modifyColumn(TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException { + public void deleteColumnFamily(TableName tableName, byte[] bytes) throws IOException { + + } + + @Override + public Future deleteColumnFamilyAsync(TableName tableName, byte[] bytes) throws IOException { + return null; + } + + @Override + public void modifyColumnFamily(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { } + @Override + public Future modifyColumnFamilyAsync(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { + return null; + } + @Override public void closeRegion(String s, String s1) throws IOException { @@ -277,6 +326,11 @@ public List getOnlineRegions(ServerName serverName) throws IOExcept return Collections.emptyList(); } + @Override + public List getRegions(ServerName serverName) throws IOException { + return Collections.emptyList(); + } + @Override public void flush(TableName tableName) throws IOException { @@ -287,6 +341,11 @@ public void flushRegion(byte[] bytes) throws IOException { } + @Override + public void flushRegionServer(ServerName serverName) throws IOException { + + } + @Override public void compact(TableName tableName) throws IOException { @@ -307,6 +366,16 @@ public void compactRegion(byte[] bytes, byte[] bytes1) throws IOException { } + @Override + public void compact(TableName tableName, CompactType compactType) throws IOException, InterruptedException { + + } + + @Override + public void compact(TableName tableName, byte[] bytes, CompactType compactType) throws IOException, InterruptedException { + + } + @Override public void majorCompact(TableName tableName) throws IOException { @@ -328,7 +397,22 @@ public void majorCompactRegion(byte[] bytes, byte[] bytes1) throws IOException { } @Override - public void compactRegionServer(ServerName serverName, boolean b) throws IOException, InterruptedException { + public void majorCompact(TableName tableName, CompactType compactType) throws IOException, InterruptedException { + + } + + @Override + public void majorCompact(TableName tableName, byte[] bytes, CompactType compactType) throws IOException, InterruptedException { + + } + + @Override + public void compactRegionServer(ServerName serverName) throws IOException { + + } + + @Override + public void majorCompactRegionServer(ServerName serverName) throws IOException { } @@ -353,17 +437,17 @@ public void offline(byte[] bytes) throws IOException { } @Override - public boolean setBalancerRunning(boolean b, boolean b1) throws IOException { + public boolean balancerSwitch(boolean b, boolean b1) throws IOException { return false; } @Override - public boolean balancer() throws IOException { + public boolean balance() throws IOException { return false; } @Override - public boolean balancer(boolean b) throws IOException { + public boolean balance(boolean b) throws IOException { return false; } @@ -372,6 +456,11 @@ public boolean isBalancerEnabled() throws IOException { return false; } + @Override + public CacheEvictionStats clearBlockCache(TableName tableName) throws IOException { + return null; + } + @Override public boolean normalize() throws IOException { return false; @@ -383,17 +472,17 @@ public boolean isNormalizerEnabled() throws IOException { } @Override - public boolean setNormalizerRunning(boolean b) throws IOException { + public boolean normalizerSwitch(boolean b) throws IOException { return false; } @Override - public boolean enableCatalogJanitor(boolean b) throws IOException { + public boolean catalogJanitorSwitch(boolean b) throws IOException { return false; } @Override - public int runCatalogScan() throws IOException { + public int runCatalogJanitor() throws IOException { return 0; } @@ -402,11 +491,36 @@ public boolean isCatalogJanitorEnabled() throws IOException { return false; } + @Override + public boolean cleanerChoreSwitch(boolean b) throws IOException { + return false; + } + + @Override + public boolean runCleanerChore() throws IOException { + return false; + } + + @Override + public boolean isCleanerChoreEnabled() throws IOException { + return false; + } + @Override public void mergeRegions(byte[] bytes, byte[] bytes1, boolean b) throws IOException { } + @Override + public Future mergeRegionsAsync(byte[] bytes, byte[] bytes1, boolean b) throws IOException { + return null; + } + + @Override + public Future mergeRegionsAsync(byte[][] bytes, boolean b) throws IOException { + return null; + } + @Override public void split(TableName tableName) throws IOException { @@ -428,8 +542,28 @@ public void splitRegion(byte[] bytes, byte[] bytes1) throws IOException { } @Override - public void modifyTable(TableName tableName, HTableDescriptor hTableDescriptor) throws IOException { + public Future splitRegionAsync(byte[] bytes, byte[] bytes1) throws IOException { + return null; + } + + @Override + public void modifyTable(TableName tableName, TableDescriptor tableDescriptor) throws IOException { + + } + + @Override + public void modifyTable(TableDescriptor tableDescriptor) throws IOException { + + } + + @Override + public Future modifyTableAsync(TableName tableName, TableDescriptor tableDescriptor) throws IOException { + return null; + } + @Override + public Future modifyTableAsync(TableDescriptor tableDescriptor) throws IOException { + return null; } @Override @@ -442,16 +576,35 @@ public void stopMaster() throws IOException { } + @Override + public boolean isMasterInMaintenanceMode() throws IOException { + return false; + } + @Override public void stopRegionServer(String s) throws IOException { } @Override - public ClusterStatus getClusterStatus() throws IOException { + public ClusterMetrics getClusterMetrics(EnumSet enumSet) throws IOException { return null; } + @Override + public List getRegionMetrics(ServerName serverName) throws IOException { + throw new FeatureNotSupportedException("does not support yet"); + } + + @Override + public List getRegionMetrics(ServerName serverName, TableName tableName) throws IOException { + OHRegionMetricsExecutor executor = new OHRegionMetricsExecutor(tableClient); + if (tableName == null) { + throw new FeatureNotSupportedException("does not support tableName is null"); + } + return executor.getRegionMetrics(tableName.getNameAsString()); + } + @Override public Configuration getConfiguration() { return null; @@ -462,16 +615,31 @@ public void createNamespace(NamespaceDescriptor namespaceDescriptor) throws IOEx } + @Override + public Future createNamespaceAsync(NamespaceDescriptor namespaceDescriptor) throws IOException { + return null; + } + @Override public void modifyNamespace(NamespaceDescriptor namespaceDescriptor) throws IOException { } + @Override + public Future modifyNamespaceAsync(NamespaceDescriptor namespaceDescriptor) throws IOException { + return null; + } + @Override public void deleteNamespace(String s) throws IOException { } + @Override + public Future deleteNamespaceAsync(String s) throws IOException { + return null; + } + @Override public NamespaceDescriptor getNamespaceDescriptor(String s) throws NamespaceNotFoundException, IOException { return null; @@ -487,6 +655,11 @@ public HTableDescriptor[] listTableDescriptorsByNamespace(String s) throws IOExc return new HTableDescriptor[0]; } + @Override + public List listTableDescriptorsByNamespace(byte[] bytes) throws IOException { + return Collections.emptyList(); + } + @Override public TableName[] listTableNamesByNamespace(String s) throws IOException { return new TableName[0]; @@ -497,6 +670,11 @@ public List getTableRegions(TableName tableName) throws IOException return Collections.emptyList(); } + @Override + public List getRegions(TableName tableName) throws IOException { + return Collections.emptyList(); + } + @Override public void close() throws IOException { @@ -507,6 +685,11 @@ public HTableDescriptor[] getTableDescriptorsByTableName(List list) t return new HTableDescriptor[0]; } + @Override + public List listTableDescriptors(List list) throws IOException { + return Collections.emptyList(); + } + @Override public HTableDescriptor[] getTableDescriptors(List list) throws IOException { return new HTableDescriptor[0]; @@ -518,13 +701,18 @@ public boolean abortProcedure(long l, boolean b) throws IOException { } @Override - public ProcedureInfo[] listProcedures() throws IOException { - return new ProcedureInfo[0]; + public Future abortProcedureAsync(long l, boolean b) throws IOException { + return null; } @Override - public Future abortProcedureAsync(long l, boolean b) throws IOException { - return null; + public String getProcedures() throws IOException { + return ""; + } + + @Override + public String getLocks() throws IOException { + return ""; } @Override @@ -533,17 +721,17 @@ public void rollWALWriter(ServerName serverName) throws IOException, FailedLogCl } @Override - public String[] getMasterCoprocessors() throws IOException { - return new String[0]; + public CompactionState getCompactionState(TableName tableName) throws IOException { + return null; } @Override - public AdminProtos.GetRegionInfoResponse.CompactionState getCompactionState(TableName tableName) throws IOException { + public CompactionState getCompactionState(TableName tableName, CompactType compactType) throws IOException { return null; } @Override - public AdminProtos.GetRegionInfoResponse.CompactionState getCompactionStateForRegion(byte[] bytes) throws IOException { + public CompactionState getCompactionStateForRegion(byte[] bytes) throws IOException { return null; } @@ -568,22 +756,22 @@ public void snapshot(byte[] bytes, TableName tableName) throws IOException, Snap } @Override - public void snapshot(String s, TableName tableName, HBaseProtos.SnapshotDescription.Type type) throws IOException, SnapshotCreationException, IllegalArgumentException { + public void snapshot(String s, TableName tableName, SnapshotType snapshotType) throws IOException, SnapshotCreationException, IllegalArgumentException { } @Override - public void snapshot(HBaseProtos.SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException, IllegalArgumentException { + public void snapshot(SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException, IllegalArgumentException { } @Override - public MasterProtos.SnapshotResponse takeSnapshotAsync(HBaseProtos.SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException { - return null; + public void snapshotAsync(SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException { + } @Override - public boolean isSnapshotFinished(HBaseProtos.SnapshotDescription snapshotDescription) throws IOException, HBaseSnapshotException, UnknownSnapshotException { + public boolean isSnapshotFinished(SnapshotDescription snapshotDescription) throws IOException, HBaseSnapshotException, UnknownSnapshotException { return false; } @@ -597,6 +785,11 @@ public void restoreSnapshot(String s) throws IOException, RestoreSnapshotExcepti } + @Override + public Future restoreSnapshotAsync(String s) throws IOException, RestoreSnapshotException { + return null; + } + @Override public void restoreSnapshot(byte[] bytes, boolean b) throws IOException, RestoreSnapshotException { @@ -607,23 +800,38 @@ public void restoreSnapshot(String s, boolean b) throws IOException, RestoreSnap } + @Override + public void restoreSnapshot(String s, boolean b, boolean b1) throws IOException, RestoreSnapshotException { + + } + @Override public void cloneSnapshot(byte[] bytes, TableName tableName) throws IOException, TableExistsException, RestoreSnapshotException { } + @Override + public void cloneSnapshot(String s, TableName tableName, boolean b) throws IOException, TableExistsException, RestoreSnapshotException { + + } + @Override public void cloneSnapshot(String s, TableName tableName) throws IOException, TableExistsException, RestoreSnapshotException { } + @Override + public Future cloneSnapshotAsync(String s, TableName tableName) throws IOException, TableExistsException { + return null; + } + @Override public void execProcedure(String s, String s1, Map map) throws IOException { } @Override - public byte[] execProcedureWithRet(String s, String s1, Map map) throws IOException { + public byte[] execProcedureWithReturn(String s, String s1, Map map) throws IOException { return new byte[0]; } @@ -633,27 +841,27 @@ public boolean isProcedureFinished(String s, String s1, Map map) } @Override - public List listSnapshots() throws IOException { + public List listSnapshots() throws IOException { return Collections.emptyList(); } @Override - public List listSnapshots(String s) throws IOException { + public List listSnapshots(String s) throws IOException { return Collections.emptyList(); } @Override - public List listSnapshots(Pattern pattern) throws IOException { + public List listSnapshots(Pattern pattern) throws IOException { return Collections.emptyList(); } @Override - public List listTableSnapshots(String s, String s1) throws IOException { + public List listTableSnapshots(String s, String s1) throws IOException { return Collections.emptyList(); } @Override - public List listTableSnapshots(Pattern pattern, Pattern pattern1) throws IOException { + public List listTableSnapshots(Pattern pattern, Pattern pattern1) throws IOException { return Collections.emptyList(); } @@ -697,6 +905,11 @@ public QuotaRetriever getQuotaRetriever(QuotaFilter quotaFilter) throws IOExcept return null; } + @Override + public List getQuota(QuotaFilter quotaFilter) throws IOException { + return Collections.emptyList(); + } + @Override public CoprocessorRpcChannel coprocessorService() { return null; @@ -718,22 +931,117 @@ public void updateConfiguration() throws IOException { } @Override - public int getMasterInfoPort() throws IOException { - return 0; + public List getSecurityCapabilities() throws IOException { + return Collections.emptyList(); } @Override - public List getSecurityCapabilities() throws IOException { - return Collections.emptyList(); + public boolean splitSwitch(boolean b, boolean b1) throws IOException { + return false; + } + + @Override + public boolean mergeSwitch(boolean b, boolean b1) throws IOException { + return false; } @Override - public boolean[] setSplitOrMergeEnabled(boolean b, boolean b1, MasterSwitchType... masterSwitchTypes) throws IOException { - return new boolean[0]; + public boolean isSplitEnabled() throws IOException { + return false; } @Override - public boolean isSplitOrMergeEnabled(MasterSwitchType masterSwitchType) throws IOException { + public boolean isMergeEnabled() throws IOException { return false; } + + @Override + public void addReplicationPeer(String s, ReplicationPeerConfig replicationPeerConfig, boolean b) throws IOException { + + } + + @Override + public void removeReplicationPeer(String s) throws IOException { + + } + + @Override + public void enableReplicationPeer(String s) throws IOException { + + } + + @Override + public void disableReplicationPeer(String s) throws IOException { + + } + + @Override + public ReplicationPeerConfig getReplicationPeerConfig(String s) throws IOException { + return null; + } + + @Override + public void updateReplicationPeerConfig(String s, ReplicationPeerConfig replicationPeerConfig) throws IOException { + + } + + @Override + public void appendReplicationPeerTableCFs(String s, Map> map) throws ReplicationException, IOException { + + } + + @Override + public void removeReplicationPeerTableCFs(String s, Map> map) throws ReplicationException, IOException { + + } + + @Override + public List listReplicationPeers() throws IOException { + return Collections.emptyList(); + } + + @Override + public List listReplicationPeers(Pattern pattern) throws IOException { + return Collections.emptyList(); + } + + @Override + public void decommissionRegionServers(List list, boolean b) throws IOException { + + } + + @Override + public List listDecommissionedRegionServers() throws IOException { + return Collections.emptyList(); + } + + @Override + public void recommissionRegionServer(ServerName serverName, List list) throws IOException { + + } + + @Override + public List listReplicatedTableCFs() throws IOException { + return Collections.emptyList(); + } + + @Override + public void enableTableReplication(TableName tableName) throws IOException { + + } + + @Override + public void disableTableReplication(TableName tableName) throws IOException { + + } + + @Override + public void clearCompactionQueues(ServerName serverName, Set set) throws IOException, InterruptedException { + + } + + @Override + public List clearDeadServers(List list) throws IOException { + return Collections.emptyList(); + } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java new file mode 100644 index 00000000..735bac9d --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java @@ -0,0 +1,120 @@ +package com.alipay.oceanbase.hbase.util; + +import org.apache.hadoop.hbase.RegionMetrics; +import org.apache.hadoop.hbase.Size; + +import java.util.Collections; +import java.util.Map; + +public class OHRegionMetrics implements RegionMetrics { + private final String tablegroup; + private final byte[] name; // tablet_name, id in String + private final Size storeFileSize; // tablet storage used in ssTable + private final Size memStoreSize; // tablet storage used in memTable + + OHRegionMetrics(String tablegroup, byte[] name, Size storeFileSize, Size memStoreSize) { + this.tablegroup = tablegroup; + this.name = name; + this.storeFileSize = storeFileSize; + this.memStoreSize = memStoreSize; + } + + public String getTablegroup() { + return tablegroup; + } + + @Override + public byte[] getRegionName() { + return name; + } + + @Override + public int getStoreCount() { + return 0; + } + + @Override + public int getStoreFileCount() { + return 0; + } + + @Override + public Size getStoreFileSize() { + return storeFileSize; + } + + @Override + public Size getMemStoreSize() { + return memStoreSize; + } + + @Override + public long getReadRequestCount() { + return 0; + } + + @Override + public long getWriteRequestCount() { + return 0; + } + + @Override + public long getFilteredReadRequestCount() { + return 0; + } + + @Override + public Size getStoreFileIndexSize() { + return null; + } + + @Override + public Size getStoreFileRootLevelIndexSize() { + return null; + } + + @Override + public Size getStoreFileUncompressedDataIndexSize() { + return null; + } + + @Override + public Size getBloomFilterSize() { + return null; + } + + @Override + public long getCompactingCellCount() { + return 0; + } + + @Override + public long getCompactedCellCount() { + return 0; + } + + @Override + public long getCompletedSequenceId() { + return 0; + } + + @Override + public Map getStoreSequenceId() { + return Collections.emptyMap(); + } + + @Override + public Size getUncompressedStoreFileSize() { + return null; + } + + @Override + public float getDataLocality() { + return 0; + } + + @Override + public long getLastMajorCompactionTimestamp() { + return 0; + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java new file mode 100644 index 00000000..58485d14 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java @@ -0,0 +1,71 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import org.apache.hadoop.hbase.RegionMetrics; +import org.apache.hadoop.hbase.Size; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class OHRegionMetricsExecutor extends AbstractObTableMetaExecutor> { + private final ObTableClient tableClient; + OHRegionMetricsExecutor(ObTableClient tableClient) { + this.tableClient = tableClient; + } + @Override + public ObTableRpcMetaType getMetaType() throws IOException { + return ObTableRpcMetaType.HTABLE_REGION_METRICS; + } + + /* + * { + tableName: "tablegroup_name", + regionList:{ + "regions": [200051, 200052, 200053, 200191, 200192, 200193, ...], + "memTableSize":[123, 321, 321, 123, 321, 321, ...], + "ssTableSize":[5122, 4111, 5661, 5122, 4111, 5661, ...] + } + } + * */ + @Override + public List parse(ObTableMetaResponse response) throws IOException { + List metricsList = new ArrayList<>(); + JSONObject metrcisJSONObject = JSON.parseObject(response.getData()); + String tableGroupName = metrcisJSONObject.getString("tableName"); + JSONObject regionList = metrcisJSONObject.getJSONObject("regionList"); + List regions = regionList.getJSONArray("regions").toJavaList(Integer.class); + List memTableSizeList = regionList.getJSONArray("memTableSize").toJavaList(Integer.class); + List ssTableSizeList = regionList.getJSONArray("ssTableSize").toJavaList(Integer.class); + if (regions.isEmpty() || regions.size() != memTableSizeList.size() || memTableSizeList.size() != ssTableSizeList.size()) { + throw new IOException("size length has to be the same"); + } + for (int i = 0; i < regions.size(); ++i) { + String name_str = Integer.toString(regions.get(i)); + byte[] name = name_str.getBytes(); + Size storeFileSize = new Size(((double) ssTableSizeList.get(i)) / (1024 * 1024) , Size.Unit.MEGABYTE); // The unit in original HBase is MEGABYTE, for us it is BYTE + Size memStoreSize = new Size(((double) memTableSizeList.get(i)) / (1024 * 1024), Size.Unit.MEGABYTE); // The unit in original HBase is MEGABYTE, for us it is BYTE + OHRegionMetrics ohRegionMetrics = new OHRegionMetrics(tableGroupName, name, storeFileSize, memStoreSize); + metricsList.add(ohRegionMetrics); + } + return metricsList; + } + + public List getRegionMetrics(String tableName) throws IOException { + ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestData = new HashMap<>(); + requestData.put("name", tableName); + String jsonData = JSON.toJSONString(requestData); + request.setData(jsonData); + return execute(tableClient, request); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 6d53616b..a3e603f7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -18,15 +18,31 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.rpc.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.RegionMetrics; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import org.codehaus.jackson.annotate.JsonTypeInfo; import org.junit.Assert; import org.junit.Test; import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertThrows; public class OHTableAdminInterfaceTest { public OHTablePool setUpLoadPool() throws IOException { @@ -249,4 +265,112 @@ public void testGetStartEndKeysOHTablePoolLoadNon() throws Exception { Assert.assertEquals(0, startEndKeys.getFirst()[0].length); Assert.assertEquals(0, startEndKeys.getSecond()[0].length); } + + @Test + public void testAdminGetRegionMetrics() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + IOException thrown = assertThrows(IOException.class, + () -> { + admin.getRegionMetrics(null, TableName.valueOf("tablegroup_not_exists")); + }); + Assert.assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + assertThrows(FeatureNotSupportedException.class, + () -> { + admin.getRegionMetrics(ServerName.valueOf("localhost,1,1")); + }); + // insert 300 thousand of rows in each table under tablegorup test_multi_cf + batchInsert(100000); + List metrics = admin.getRegionMetrics(null, TableName.valueOf("test_multi_cf")); + for (RegionMetrics regionMetrics : metrics) { + System.out.println("region name: " + regionMetrics.getNameAsString() + + ", storeFileSize: " + regionMetrics.getStoreFileSize() + + ", memFileSize: " + regionMetrics.getMemStoreSize()); + } + // concurrent read write + ExecutorService executorService = Executors.newFixedThreadPool(10); + CountDownLatch latch = new CountDownLatch(100); + List exceptionCatcher = new ArrayList<>(); + for (int i = 0; i < 100; ++i) { + int taskId = i; + executorService.submit(() -> { + try { + if (taskId % 2 == 1) { + List regionMetrics = admin.getRegionMetrics(null, TableName.valueOf("test_multi_cf")); + for (RegionMetrics m : regionMetrics) { + System.out.println("task: " + taskId + ", region name: " + m.getNameAsString() + + ", storeFileSize: " + m.getStoreFileSize() + + ", memFileSize: " + m.getMemStoreSize()); + } + } else { + batchInsert(1000); + System.out.println("task: " + taskId + ", batchInsert"); + } + } catch (Exception e) { + e.printStackTrace(); + exceptionCatcher.add(e); + } finally { + latch.countDown(); + } + }); + } + try { + latch.await(); + } catch (Exception e) { + e.printStackTrace(); + exceptionCatcher.add(e); + } + executorService.shutdownNow(); + Assert.assertTrue(exceptionCatcher.isEmpty()); + } + + private void batchInsert(int rows) throws Exception { + byte[] family1 = Bytes.toBytes("family_with_group1"); + byte[] family2 = Bytes.toBytes("family_with_group2"); + byte[] family3 = Bytes.toBytes("family_with_group3"); + byte[] family1_column1 = "family1_column1".getBytes(); + byte[] family1_column2 = "family1_column2".getBytes(); + byte[] family1_column3 = "family1_column3".getBytes(); + byte[] family2_column1 = "family2_column1".getBytes(); + byte[] family2_column2 = "family2_column2".getBytes(); + byte[] family2_column3 = "family2_column3".getBytes(); + byte[] family3_column1 = "family3_column1".getBytes(); + byte[] family3_column2 = "family3_column2".getBytes(); + byte[] family3_column3 = "family3_column3".getBytes(); + byte[] family1_value1 = Bytes.toBytes("family1_value1"); + byte[] family1_value2 = Bytes.toBytes("family1_value2"); + byte[] family1_value3 = Bytes.toBytes("family1_value3"); + byte[] family2_value1 = Bytes.toBytes("family2_value1"); + byte[] family2_value2 = Bytes.toBytes("family2_value2"); + byte[] family2_value3 = Bytes.toBytes("family2_value3"); + byte[] family3_value1 = Bytes.toBytes("family3_value1"); + byte[] family3_value2 = Bytes.toBytes("family3_value2"); + byte[] family3_value3 = Bytes.toBytes("family3_value3"); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Table table = connection.getTable(TableName.valueOf("test_multi_cf")); + List batchLsit = new LinkedList<>(); + for (int i = 0; i < rows; ++i) { + Put put = new Put(toBytes("Key" + i)); + put.addColumn(family1, family1_column1, family1_value1); + put.addColumn(family1, family1_column2, family1_value2); + put.addColumn(family1, family1_column3, family1_value3); + put.addColumn(family2, family2_column1, family2_value1); + put.addColumn(family2, family2_column2, family2_value2); + put.addColumn(family2, family2_column3, family2_value3); + put.addColumn(family3, family3_column1, family3_value1); + put.addColumn(family3, family3_column2, family3_value2); + put.addColumn(family3, family3_column3, family3_value3); + batchLsit.add(put); + if (i % 100 == 0) { // 100 rows one batch to avoid OB_TIMEOUT + Object[] results = new Object[batchLsit.size()]; + table.batch(batchLsit, results); + batchLsit.clear(); + } + } + Object[] results = new Object[batchLsit.size()]; + table.batch(batchLsit, results); + } } From 5712ffaf176745f4de4fc477a2ac79559675002a Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 26 May 2025 12:07:29 +0800 Subject: [PATCH 04/36] set not support for interfaces --- .../alipay/oceanbase/hbase/util/OHAdmin.java | 402 +++++++++--------- .../hbase/util/OHConnectionImpl.java | 8 +- .../oceanbase/hbase/util/OHRegionMetrics.java | 33 +- .../hbase/util/ObTableClientManager.java | 7 + .../hbase/OHTableAdminInterfaceTest.java | 7 +- .../OHTableTimeSeriesDeleteTest.java | 14 +- 6 files changed, 244 insertions(+), 227 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index 89c4e408..d31baa4e 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -2,7 +2,7 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.bolt.transport.TransportCodes; -import com.alipay.oceanbase.rpc.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import com.alipay.oceanbase.rpc.exception.ObTableTransportException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; @@ -31,28 +31,33 @@ public class OHAdmin implements Admin { private final ObTableClient tableClient; - OHAdmin(ObTableClient tableClient) { + private final OHConnectionImpl connection; + private boolean aborted = false; + OHAdmin(ObTableClient tableClient, OHConnectionImpl connection) { this.tableClient = tableClient; + this.connection = connection; } @Override public int getOperationTimeout() { - return 0; + return (int) tableClient.getRuntimeMaxWait(); } @Override - public void abort(String s, Throwable throwable) { - + public void abort(String msg, Throwable t) { + // do nothing, just throw the message and exception + this.aborted = true; + throw new RuntimeException(msg, t); } @Override public boolean isAborted() { - return false; + return this.aborted; } @Override public Connection getConnection() { - return null; + return this.connection; } @Override @@ -63,97 +68,97 @@ public boolean tableExists(TableName tableName) throws IOException { @Override public HTableDescriptor[] listTables() throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listTableDescriptors() throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] listTables(Pattern pattern) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listTableDescriptors(Pattern pattern) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] listTables(String s) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] listTables(Pattern pattern, boolean b) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listTableDescriptors(Pattern pattern, boolean b) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] listTables(String s, boolean b) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public TableName[] listTableNames() throws IOException { - return new TableName[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public TableName[] listTableNames(Pattern pattern) throws IOException { - return new TableName[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public TableName[] listTableNames(String s) throws IOException { - return new TableName[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public TableName[] listTableNames(Pattern pattern, boolean b) throws IOException { - return new TableName[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public TableName[] listTableNames(String s, boolean b) throws IOException { - return new TableName[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor getTableDescriptor(TableName tableName) throws TableNotFoundException, IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public TableDescriptor getDescriptor(TableName tableName) throws TableNotFoundException, IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void createTable(TableDescriptor tableDescriptor) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void createTable(TableDescriptor tableDescriptor, byte[] bytes, byte[] bytes1, int i) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void createTable(TableDescriptor tableDescriptor, byte[][] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future createTableAsync(TableDescriptor tableDescriptor, byte[][] bytes) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override @@ -173,422 +178,422 @@ public void deleteTable(TableName tableName) throws IOException { @Override public Future deleteTableAsync(TableName tableName) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] deleteTables(String s) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] deleteTables(Pattern pattern) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void truncateTable(TableName tableName, boolean b) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future truncateTableAsync(TableName tableName, boolean b) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void enableTable(TableName tableName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future enableTableAsync(TableName tableName) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] enableTables(String s) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] enableTables(Pattern pattern) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future disableTableAsync(TableName tableName) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void disableTable(TableName tableName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] disableTables(String s) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] disableTables(Pattern pattern) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isTableEnabled(TableName tableName) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isTableDisabled(TableName tableName) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isTableAvailable(TableName tableName) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isTableAvailable(TableName tableName, byte[][] bytes) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public Pair getAlterStatus(TableName tableName) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public Pair getAlterStatus(byte[] bytes) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void addColumnFamily(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future addColumnFamilyAsync(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void deleteColumn(TableName tableName, byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void deleteColumnFamily(TableName tableName, byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future deleteColumnFamilyAsync(TableName tableName, byte[] bytes) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void modifyColumnFamily(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future modifyColumnFamilyAsync(TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void closeRegion(String s, String s1) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void closeRegion(byte[] bytes, String s) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean closeRegionWithEncodedRegionName(String s, String s1) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void closeRegion(ServerName serverName, HRegionInfo hRegionInfo) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public List getOnlineRegions(ServerName serverName) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public List getRegions(ServerName serverName) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public void flush(TableName tableName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void flushRegion(byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void flushRegionServer(ServerName serverName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void compact(TableName tableName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void compactRegion(byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void compact(TableName tableName, byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void compactRegion(byte[] bytes, byte[] bytes1) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void compact(TableName tableName, CompactType compactType) throws IOException, InterruptedException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void compact(TableName tableName, byte[] bytes, CompactType compactType) throws IOException, InterruptedException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void majorCompact(TableName tableName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void majorCompactRegion(byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void majorCompact(TableName tableName, byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void majorCompactRegion(byte[] bytes, byte[] bytes1) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void majorCompact(TableName tableName, CompactType compactType) throws IOException, InterruptedException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void majorCompact(TableName tableName, byte[] bytes, CompactType compactType) throws IOException, InterruptedException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void compactRegionServer(ServerName serverName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void majorCompactRegionServer(ServerName serverName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void move(byte[] bytes, byte[] bytes1) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void assign(byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void unassign(byte[] bytes, boolean b) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void offline(byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean balancerSwitch(boolean b, boolean b1) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean balance() throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean balance(boolean b) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isBalancerEnabled() throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public CacheEvictionStats clearBlockCache(TableName tableName) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean normalize() throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isNormalizerEnabled() throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean normalizerSwitch(boolean b) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean catalogJanitorSwitch(boolean b) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public int runCatalogJanitor() throws IOException { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isCatalogJanitorEnabled() throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean cleanerChoreSwitch(boolean b) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean runCleanerChore() throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isCleanerChoreEnabled() throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void mergeRegions(byte[] bytes, byte[] bytes1, boolean b) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future mergeRegionsAsync(byte[] bytes, byte[] bytes1, boolean b) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future mergeRegionsAsync(byte[][] bytes, boolean b) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void split(TableName tableName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void splitRegion(byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void split(TableName tableName, byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void splitRegion(byte[] bytes, byte[] bytes1) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future splitRegionAsync(byte[] bytes, byte[] bytes1) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void modifyTable(TableName tableName, TableDescriptor tableDescriptor) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void modifyTable(TableDescriptor tableDescriptor) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future modifyTableAsync(TableName tableName, TableDescriptor tableDescriptor) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future modifyTableAsync(TableDescriptor tableDescriptor) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void shutdown() throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void stopMaster() throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isMasterInMaintenanceMode() throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void stopRegionServer(String s) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public ClusterMetrics getClusterMetrics(EnumSet enumSet) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override @@ -607,441 +612,440 @@ public List getRegionMetrics(ServerName serverName, TableName tab @Override public Configuration getConfiguration() { - return null; + return connection.getConfiguration(); } @Override public void createNamespace(NamespaceDescriptor namespaceDescriptor) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future createNamespaceAsync(NamespaceDescriptor namespaceDescriptor) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void modifyNamespace(NamespaceDescriptor namespaceDescriptor) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future modifyNamespaceAsync(NamespaceDescriptor namespaceDescriptor) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void deleteNamespace(String s) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future deleteNamespaceAsync(String s) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public NamespaceDescriptor getNamespaceDescriptor(String s) throws NamespaceNotFoundException, IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public NamespaceDescriptor[] listNamespaceDescriptors() throws IOException { - return new NamespaceDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] listTableDescriptorsByNamespace(String s) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listTableDescriptorsByNamespace(byte[] bytes) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public TableName[] listTableNamesByNamespace(String s) throws IOException { - return new TableName[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public List getTableRegions(TableName tableName) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public List getRegions(TableName tableName) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override - public void close() throws IOException { - + public synchronized void close() throws IOException { } @Override public HTableDescriptor[] getTableDescriptorsByTableName(List list) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listTableDescriptors(List list) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public HTableDescriptor[] getTableDescriptors(List list) throws IOException { - return new HTableDescriptor[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean abortProcedure(long l, boolean b) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future abortProcedureAsync(long l, boolean b) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public String getProcedures() throws IOException { - return ""; + throw new FeatureNotSupportedException("does not support yet"); } @Override public String getLocks() throws IOException { - return ""; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void rollWALWriter(ServerName serverName) throws IOException, FailedLogCloseException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public CompactionState getCompactionState(TableName tableName) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public CompactionState getCompactionState(TableName tableName, CompactType compactType) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public CompactionState getCompactionStateForRegion(byte[] bytes) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public long getLastMajorCompactionTimestamp(TableName tableName) throws IOException { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override public long getLastMajorCompactionTimestampForRegion(byte[] bytes) throws IOException { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void snapshot(String s, TableName tableName) throws IOException, SnapshotCreationException, IllegalArgumentException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void snapshot(byte[] bytes, TableName tableName) throws IOException, SnapshotCreationException, IllegalArgumentException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void snapshot(String s, TableName tableName, SnapshotType snapshotType) throws IOException, SnapshotCreationException, IllegalArgumentException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void snapshot(SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException, IllegalArgumentException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void snapshotAsync(SnapshotDescription snapshotDescription) throws IOException, SnapshotCreationException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isSnapshotFinished(SnapshotDescription snapshotDescription) throws IOException, HBaseSnapshotException, UnknownSnapshotException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void restoreSnapshot(byte[] bytes) throws IOException, RestoreSnapshotException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void restoreSnapshot(String s) throws IOException, RestoreSnapshotException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future restoreSnapshotAsync(String s) throws IOException, RestoreSnapshotException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void restoreSnapshot(byte[] bytes, boolean b) throws IOException, RestoreSnapshotException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void restoreSnapshot(String s, boolean b) throws IOException, RestoreSnapshotException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void restoreSnapshot(String s, boolean b, boolean b1) throws IOException, RestoreSnapshotException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void cloneSnapshot(byte[] bytes, TableName tableName) throws IOException, TableExistsException, RestoreSnapshotException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void cloneSnapshot(String s, TableName tableName, boolean b) throws IOException, TableExistsException, RestoreSnapshotException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void cloneSnapshot(String s, TableName tableName) throws IOException, TableExistsException, RestoreSnapshotException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public Future cloneSnapshotAsync(String s, TableName tableName) throws IOException, TableExistsException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void execProcedure(String s, String s1, Map map) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public byte[] execProcedureWithReturn(String s, String s1, Map map) throws IOException { - return new byte[0]; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isProcedureFinished(String s, String s1, Map map) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listSnapshots() throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listSnapshots(String s) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listSnapshots(Pattern pattern) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listTableSnapshots(String s, String s1) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listTableSnapshots(Pattern pattern, Pattern pattern1) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public void deleteSnapshot(byte[] bytes) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void deleteSnapshot(String s) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void deleteSnapshots(String s) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void deleteSnapshots(Pattern pattern) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void deleteTableSnapshots(String s, String s1) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void deleteTableSnapshots(Pattern pattern, Pattern pattern1) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void setQuota(QuotaSettings quotaSettings) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public QuotaRetriever getQuotaRetriever(QuotaFilter quotaFilter) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public List getQuota(QuotaFilter quotaFilter) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public CoprocessorRpcChannel coprocessorService() { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public CoprocessorRpcChannel coprocessorService(ServerName serverName) { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void updateConfiguration(ServerName serverName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void updateConfiguration() throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public List getSecurityCapabilities() throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean splitSwitch(boolean b, boolean b1) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean mergeSwitch(boolean b, boolean b1) throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isSplitEnabled() throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public boolean isMergeEnabled() throws IOException { - return false; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void addReplicationPeer(String s, ReplicationPeerConfig replicationPeerConfig, boolean b) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void removeReplicationPeer(String s) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void enableReplicationPeer(String s) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void disableReplicationPeer(String s) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public ReplicationPeerConfig getReplicationPeerConfig(String s) throws IOException { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public void updateReplicationPeerConfig(String s, ReplicationPeerConfig replicationPeerConfig) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void appendReplicationPeerTableCFs(String s, Map> map) throws ReplicationException, IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void removeReplicationPeerTableCFs(String s, Map> map) throws ReplicationException, IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listReplicationPeers() throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listReplicationPeers(Pattern pattern) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public void decommissionRegionServers(List list, boolean b) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listDecommissionedRegionServers() throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public void recommissionRegionServer(ServerName serverName, List list) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public List listReplicatedTableCFs() throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public void enableTableReplication(TableName tableName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void disableTableReplication(TableName tableName) throws IOException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public void clearCompactionQueues(ServerName serverName, Set set) throws IOException, InterruptedException { - + throw new FeatureNotSupportedException("does not support yet"); } @Override public List clearDeadServers(List list) throws IOException { - return Collections.emptyList(); + throw new FeatureNotSupportedException("does not support yet"); } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index de6ae338..79e1c1ee 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -142,6 +142,9 @@ public BufferedMutator getBufferedMutator(BufferedMutatorParams params) throws I @Override public RegionLocator getRegionLocator(TableName tableName) throws IOException { ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClient(connectionConfig); + int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + ObTableClientManager.initTimeoutAndRetryTimes(obTableClient, connectionConfig, numRetries); OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(obTableClient); return executor.getRegionLocator(String.valueOf(tableName)); } @@ -149,7 +152,10 @@ public RegionLocator getRegionLocator(TableName tableName) throws IOException { @Override public Admin getAdmin() throws IOException { ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClient(connectionConfig); - return new OHAdmin(obTableClient); + int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + ObTableClientManager.initTimeoutAndRetryTimes(obTableClient, connectionConfig, numRetries); + return new OHAdmin(obTableClient, this); } private void shutdownBatchPool(ExecutorService pool) { diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java index 735bac9d..8f161575 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java @@ -1,5 +1,6 @@ package com.alipay.oceanbase.hbase.util; +import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import org.apache.hadoop.hbase.RegionMetrics; import org.apache.hadoop.hbase.Size; @@ -30,12 +31,12 @@ public byte[] getRegionName() { @Override public int getStoreCount() { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override public int getStoreFileCount() { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override @@ -50,71 +51,71 @@ public Size getMemStoreSize() { @Override public long getReadRequestCount() { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override public long getWriteRequestCount() { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override public long getFilteredReadRequestCount() { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override public Size getStoreFileIndexSize() { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public Size getStoreFileRootLevelIndexSize() { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public Size getStoreFileUncompressedDataIndexSize() { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public Size getBloomFilterSize() { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public long getCompactingCellCount() { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override public long getCompactedCellCount() { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override public long getCompletedSequenceId() { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override public Map getStoreSequenceId() { - return Collections.emptyMap(); + throw new FeatureNotSupportedException("does not support yet"); } @Override public Size getUncompressedStoreFileSize() { - return null; + throw new FeatureNotSupportedException("does not support yet"); } @Override public float getDataLocality() { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } @Override public long getLastMajorCompactionTimestamp() { - return 0; + throw new FeatureNotSupportedException("does not support yet"); } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java index 5db940aa..c80558fa 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java @@ -126,6 +126,13 @@ public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableCli return OB_TABLE_CLIENT_INSTANCE.get(obTableClientKey); } + public static void initTimeoutAndRetryTimes(ObTableClient obTableClient, OHConnectionConfiguration ohConnectionConf, int numRetries) { + obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); + obTableClient.setRuntimeRetryTimes(numRetries); + obTableClient.setRuntimeMaxWait(ohConnectionConf.getOperationTimeout()); + obTableClient.setRuntimeBatchMaxWait(ohConnectionConf.getOperationTimeout()); + } + public static class ObTableClientKey { private String paramUrl; private String fullUserName; diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index a3e603f7..5ea1fde7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -18,7 +18,7 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; -import com.alipay.oceanbase.rpc.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import org.apache.hadoop.conf.Configuration; @@ -28,7 +28,6 @@ import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; -import org.codehaus.jackson.annotate.JsonTypeInfo; import org.junit.Assert; import org.junit.Test; @@ -281,7 +280,7 @@ public void testAdminGetRegionMetrics() throws Exception { () -> { admin.getRegionMetrics(ServerName.valueOf("localhost,1,1")); }); - // insert 300 thousand of rows in each table under tablegorup test_multi_cf + // insert 300 thousand of rows in each table under tablegroup test_multi_cf batchInsert(100000); List metrics = admin.getRegionMetrics(null, TableName.valueOf("test_multi_cf")); for (RegionMetrics regionMetrics : metrics) { @@ -289,7 +288,7 @@ public void testAdminGetRegionMetrics() throws Exception { + ", storeFileSize: " + regionMetrics.getStoreFileSize() + ", memFileSize: " + regionMetrics.getMemStoreSize()); } - // concurrent read write + // concurrently read while writing 150 thousand of rows to each table ExecutorService executorService = Executors.newFixedThreadPool(10); CountDownLatch latch = new CountDownLatch(100); List exceptionCatcher = new ArrayList<>(); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesDeleteTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesDeleteTest.java index 5669da70..3e85b904 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesDeleteTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesDeleteTest.java @@ -75,7 +75,7 @@ public static void testDeleteColumnImpl(String tableName) throws Exception { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[i].getBytes(), ts[i], toBytes(values[i])); + put.addColumn(family.getBytes(), columns[i].getBytes(), ts[i], toBytes(values[i])); hTable.put(put); } @@ -126,7 +126,7 @@ public static void testDeleteColumnsImpl(String tableName) throws Exception { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[i].getBytes(), ts[i] + i, toBytes(values[i] + i)); + put.addColumn(family.getBytes(), columns[i].getBytes(), ts[i] + i, toBytes(values[i] + i)); hTable.put(put); } @@ -177,7 +177,7 @@ public static void testDeleteFamilyImpl(String tableName) throws Exception { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[i].getBytes(), ts[i] + i, toBytes(values[i] + i)); + put.addColumn(family.getBytes(), columns[i].getBytes(), ts[i] + i, toBytes(values[i] + i)); hTable.put(put); } @@ -229,8 +229,8 @@ public static void testDeleteFamilyWithTimestampImpl(String tableName) throws Ex for (int i = 0; i < 3; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[0].getBytes(), curTs + i, toBytes(value + i)); - put.add(family.getBytes(), columns[1].getBytes(), curTs + i, toBytes(value + i)); + put.addColumn(family.getBytes(), columns[0].getBytes(), curTs + i, toBytes(value + i)); + put.addColumn(family.getBytes(), columns[1].getBytes(), curTs + i, toBytes(value + i)); hTable.put(put); } @@ -264,8 +264,8 @@ public static void testDeleteFamilyVersionImpl(String tableName) throws Exceptio for (int i = 0; i < 3; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[0].getBytes(), curTs + i, toBytes(value + i)); - put.add(family.getBytes(), columns[1].getBytes(), curTs + i, toBytes(value + i)); + put.addColumn(family.getBytes(), columns[0].getBytes(), curTs + i, toBytes(value + i)); + put.addColumn(family.getBytes(), columns[1].getBytes(), curTs + i, toBytes(value + i)); hTable.put(put); } From fe943d26a377298c432c8d12c437bf11eb8ad649 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 26 May 2025 17:55:50 +0800 Subject: [PATCH 05/36] adapt hbase namespace --- .../com/alipay/oceanbase/hbase/OHTable.java | 14 +- .../alipay/oceanbase/hbase/util/OHAdmin.java | 18 ++- .../hbase/util/OHConnectionConfiguration.java | 7 + .../hbase/util/OHConnectionImpl.java | 14 +- .../hbase/util/ObTableClientManager.java | 15 +- .../hbase/OHTableAdminInterfaceTest.java | 133 ++++++++++++++++-- 6 files changed, 167 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index f66dcd8c..345fd894 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -220,8 +220,7 @@ public OHTable(Configuration configuration, String tableName) throws IOException DEFAULT_HBASE_HTABLE_THREAD_KEEP_ALIVE_TIME); this.executePool = createDefaultThreadPoolExecutor(1, maxThreads, keepAliveTime); OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); - int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + int numRetries = ohConnectionConf.getNumRetries(); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); @@ -273,8 +272,7 @@ public OHTable(Configuration configuration, final byte[] tableName, this.executePool = executePool; this.cleanupPoolOnClose = false; OHConnectionConfiguration ohConnectionConf = new OHConnectionConfiguration(configuration); - int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + int numRetries = ohConnectionConf.getNumRetries(); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, ohConnectionConf)); this.obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); @@ -345,8 +343,7 @@ public OHTable(TableName tableName, Connection connection, DEFAULT_HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK); this.writeBufferSize = connectionConfig.getWriteBufferSize(); this.tableName = tableName.getName(); - int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + int numRetries = connectionConfig.getNumRetries(); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, connectionConfig)); this.obTableClient.setRpcExecuteTimeout(rpcTimeout); @@ -389,8 +386,7 @@ public OHTable(Connection connection, ObTableBuilderBase builder, this.putWriteBufferCheck = this.configuration.getInt(HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK, DEFAULT_HBASE_HTABLE_PUT_WRITE_BUFFER_CHECK); this.writeBufferSize = connectionConfig.getWriteBufferSize(); - int numRetries = configuration.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); + int numRetries = connectionConfig.getNumRetries(); this.obTableClient = ObTableClientManager.getOrCreateObTableClient(setUserDefinedNamespace( this.tableNameString, connectionConfig)); this.obTableClient.setRpcExecuteTimeout(rpcTimeout); @@ -462,7 +458,7 @@ private void finishSetUp() { WRITE_BUFFER_SIZE_DEFAULT); } - private OHConnectionConfiguration setUserDefinedNamespace(String tableNameString, + public static OHConnectionConfiguration setUserDefinedNamespace(String tableNameString, OHConnectionConfiguration ohConnectionConf) throws IllegalArgumentException { if (tableNameString.indexOf(':') != -1) { diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index d31baa4e..c2a41d57 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -30,17 +30,17 @@ import java.util.regex.Pattern; public class OHAdmin implements Admin { - private final ObTableClient tableClient; + private boolean aborted = false; private final OHConnectionImpl connection; - private boolean aborted = false; - OHAdmin(ObTableClient tableClient, OHConnectionImpl connection) { - this.tableClient = tableClient; + private final Configuration conf; + OHAdmin(OHConnectionImpl connection) { this.connection = connection; + this.conf = connection.getConfiguration(); } @Override public int getOperationTimeout() { - return (int) tableClient.getRuntimeMaxWait(); + return connection.getOHConnectionConfiguration().getOperationTimeout(); } @Override @@ -62,6 +62,8 @@ public Connection getConnection() { @Override public boolean tableExists(TableName tableName) throws IOException { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); OHTableExistsExecutor executor = new OHTableExistsExecutor(tableClient); return executor.tableExists(tableName.getNameAsString()); } @@ -163,6 +165,8 @@ public Future createTableAsync(TableDescriptor tableDescriptor, byte[][] b @Override public void deleteTable(TableName tableName) throws IOException { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); OHDeleteTableExecutor executor = new OHDeleteTableExecutor(tableClient); try { executor.deleteTable(tableName.getNameAsString()); @@ -603,10 +607,12 @@ public List getRegionMetrics(ServerName serverName) throws IOExce @Override public List getRegionMetrics(ServerName serverName, TableName tableName) throws IOException { - OHRegionMetricsExecutor executor = new OHRegionMetricsExecutor(tableClient); if (tableName == null) { throw new FeatureNotSupportedException("does not support tableName is null"); } + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHRegionMetricsExecutor executor = new OHRegionMetricsExecutor(tableClient); return executor.getRegionMetrics(tableName.getNameAsString()); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java index 9b98dd0b..3c966a18 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionConfiguration.java @@ -57,6 +57,7 @@ public class OHConnectionConfiguration { private final int rpcConnectTimeout; private final long writeBufferPeriodicFlushTimeoutMs; private final long writeBufferPeriodicFlushTimerTickMs; + private final int numRetries; public OHConnectionConfiguration(Configuration conf) { this.paramUrl = conf.get(HBASE_OCEANBASE_PARAM_URL); @@ -100,6 +101,8 @@ public OHConnectionConfiguration(Configuration conf) { } } this.rpcConnectTimeout = rpcConnectTimeout; + this.numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, + HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); this.scannerCaching = conf.getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, Integer.MAX_VALUE); this.scannerMaxResultSize = conf.getLong( @@ -209,4 +212,8 @@ public long getWriteBufferPeriodicFlushTimeoutMs() { public long getWriteBufferPeriodicFlushTimerTickMs() { return this.writeBufferPeriodicFlushTimerTickMs; } + + public int getNumRetries() { + return this.numRetries; + } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index 79e1c1ee..d134ca8d 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -141,21 +141,17 @@ public BufferedMutator getBufferedMutator(BufferedMutatorParams params) throws I @Override public RegionLocator getRegionLocator(TableName tableName) throws IOException { - ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClient(connectionConfig); - int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); - ObTableClientManager.initTimeoutAndRetryTimes(obTableClient, connectionConfig, numRetries); + // need to use new connection configuration + // to avoid change the database in original param url by namespace in tableName + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(obTableClient); return executor.getRegionLocator(String.valueOf(tableName)); } @Override public Admin getAdmin() throws IOException { - ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClient(connectionConfig); - int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, - HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); - ObTableClientManager.initTimeoutAndRetryTimes(obTableClient, connectionConfig, numRetries); - return new OHAdmin(obTableClient, this); + return new OHAdmin(this); } private void shutdownBatchPool(ExecutorService pool) { diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java index c80558fa..e0d50cea 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java @@ -19,8 +19,10 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.constant.Constants; +import com.alipay.oceanbase.hbase.OHTable; import com.google.common.base.Objects; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.TableName; import java.io.IOException; import java.util.Map; @@ -126,9 +128,18 @@ public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableCli return OB_TABLE_CLIENT_INSTANCE.get(obTableClientKey); } - public static void initTimeoutAndRetryTimes(ObTableClient obTableClient, OHConnectionConfiguration ohConnectionConf, int numRetries) { + public static ObTableClient getOrCreateObTableClientByTableName(TableName tableName, OHConnectionConfiguration connectionConfig) throws IllegalArgumentException, + IOException { + String tableNameString = tableName.getNameAsString(); + ObTableClient obTableClient = getOrCreateObTableClient( + OHTable.setUserDefinedNamespace(tableNameString, connectionConfig)); + ObTableClientManager.initTimeoutAndRetryTimes(obTableClient, connectionConfig); + return obTableClient; + } + + private static void initTimeoutAndRetryTimes(ObTableClient obTableClient, OHConnectionConfiguration ohConnectionConf) { obTableClient.setRpcExecuteTimeout(ohConnectionConf.getRpcTimeout()); - obTableClient.setRuntimeRetryTimes(numRetries); + obTableClient.setRuntimeRetryTimes(ohConnectionConf.getNumRetries()); obTableClient.setRuntimeMaxWait(ohConnectionConf.getOperationTimeout()); obTableClient.setRuntimeBatchMaxWait(ohConnectionConf.getOperationTimeout()); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 5ea1fde7..e4b2d3d7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.OHRegionMetrics; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import com.alipay.oceanbase.rpc.exception.ObTableException; @@ -32,6 +33,7 @@ import org.junit.Test; import java.io.IOException; +import java.sql.Statement; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; @@ -41,7 +43,8 @@ import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; import static org.apache.hadoop.hbase.util.Bytes.toBytes; -import static org.junit.Assert.assertThrows; +import static org.junit.Assert.*; +import static org.junit.Assert.assertFalse; public class OHTableAdminInterfaceTest { public OHTablePool setUpLoadPool() throws IOException { @@ -265,8 +268,36 @@ public void testGetStartEndKeysOHTablePoolLoadNon() throws Exception { Assert.assertEquals(0, startEndKeys.getSecond()[0].length); } + /* + * + CREATE DATABASE IF NOT EXISTS `n1`; + CREATE TABLEGROUP `n1:test_multi_cf` SHARDING = 'ADAPTIVE'; + CREATE TABLE `n1:test_multi_cf$family_with_group1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3; + CREATE TABLE `n1:test_multi_cf$family_with_group2` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3; + CREATE TABLE `n1:test_multi_cf$family_with_group3` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3; + * */ @Test public void testAdminGetRegionMetrics() throws Exception { + String tablegroup1 = "test_multi_cf"; + String tablegroup2 = "n1:test_multi_cf"; Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); @@ -281,8 +312,8 @@ public void testAdminGetRegionMetrics() throws Exception { admin.getRegionMetrics(ServerName.valueOf("localhost,1,1")); }); // insert 300 thousand of rows in each table under tablegroup test_multi_cf - batchInsert(100000); - List metrics = admin.getRegionMetrics(null, TableName.valueOf("test_multi_cf")); + batchInsert(100000, tablegroup1); + List metrics = admin.getRegionMetrics(null, TableName.valueOf(tablegroup1)); for (RegionMetrics regionMetrics : metrics) { System.out.println("region name: " + regionMetrics.getNameAsString() + ", storeFileSize: " + regionMetrics.getStoreFileSize() @@ -297,14 +328,25 @@ public void testAdminGetRegionMetrics() throws Exception { executorService.submit(() -> { try { if (taskId % 2 == 1) { - List regionMetrics = admin.getRegionMetrics(null, TableName.valueOf("test_multi_cf")); + List regionMetrics = null; + // test get regionMetrics from different namespaces + if (taskId % 3 != 0) { + regionMetrics = admin.getRegionMetrics(null, TableName.valueOf(tablegroup1)); + } else { + regionMetrics = admin.getRegionMetrics(null, TableName.valueOf(tablegroup2)); + } for (RegionMetrics m : regionMetrics) { - System.out.println("task: " + taskId + ", region name: " + m.getNameAsString() + System.out.println("task: " + taskId + ", tablegroup: " + ((OHRegionMetrics) m).getTablegroup() + + ", region name: " + m.getNameAsString() + ", storeFileSize: " + m.getStoreFileSize() + ", memFileSize: " + m.getMemStoreSize()); } } else { - batchInsert(1000); + if (taskId % 8 == 0) { + batchInsert(1000, tablegroup2); + } else { + batchInsert(1000, tablegroup1); + } System.out.println("task: " + taskId + ", batchInsert"); } } catch (Exception e) { @@ -325,7 +367,82 @@ public void testAdminGetRegionMetrics() throws Exception { Assert.assertTrue(exceptionCatcher.isEmpty()); } - private void batchInsert(int rows) throws Exception { + @Test + public void testAdminDeleteTable() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + Statement st = conn.createStatement(); + st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE DATABASE IF NOT EXISTS `n1`;\n" + + "use `n1`;\n" + + "CREATE TABLEGROUP IF NOT EXISTS `n1:test` SHARDING = 'ADAPTIVE';\n" + + "CREATE TABLE IF NOT EXISTS `n1:test$family_group` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `n1:test`;" + + "\n" + + "CREATE TABLE IF NOT EXISTS `n1:test$family1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `n1:test`;"); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + assertTrue(admin.tableExists(TableName.valueOf("n1", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); + IOException thrown = assertThrows(IOException.class, + () -> { + admin.deleteTable(TableName.valueOf("tablegroup_not_exists")); + }); + Assert.assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + admin.deleteTable(TableName.valueOf("n1", "test")); + admin.deleteTable(TableName.valueOf("test_multi_cf")); + assertFalse(admin.tableExists(TableName.valueOf("n1", "test"))); + assertFalse(admin.tableExists(TableName.valueOf("test_multi_cf"))); + } + + @Test + public void testAdminTableExists() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + Assert.assertFalse(admin.tableExists(TableName.valueOf("tablegroup_not_exists"))); + Assert.assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); + Assert.assertTrue(admin.tableExists(TableName.valueOf("n1", "test"))); + } + + private void batchInsert(int rows, String tablegroup) throws Exception { byte[] family1 = Bytes.toBytes("family_with_group1"); byte[] family2 = Bytes.toBytes("family_with_group2"); byte[] family3 = Bytes.toBytes("family_with_group3"); @@ -349,7 +466,7 @@ private void batchInsert(int rows) throws Exception { byte[] family3_value3 = Bytes.toBytes("family3_value3"); Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); - Table table = connection.getTable(TableName.valueOf("test_multi_cf")); + Table table = connection.getTable(TableName.valueOf(tablegroup)); List batchLsit = new LinkedList<>(); for (int i = 0; i < rows; ++i) { Put put = new Put(toBytes("Key" + i)); From 7fbe89499116e98b37554d238fbd1ef0392290c9 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 28 May 2025 10:49:34 +0800 Subject: [PATCH 06/36] update test --- .../hbase/util/OHDeleteTableExecutor.java | 2 +- .../hbase/util/OHRegionMetricsExecutor.java | 2 +- .../hbase/util/OHTableExistsExecutor.java | 2 +- .../hbase/OHTableAdminInterfaceTest.java | 137 ++++++++++++++---- 4 files changed, 113 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java index af97b7a3..47280128 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java @@ -35,7 +35,7 @@ public Void deleteTable(String tableName) throws IOException { ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); Map requestDataMap = new HashMap<>(); - requestDataMap.put("name", tableName); + requestDataMap.put("table_name", tableName); String jsonData = JSON.toJSONString(requestDataMap); request.setData(jsonData); return execute(tableClient, request); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java index 58485d14..430f568f 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java @@ -63,7 +63,7 @@ public List getRegionMetrics(String tableName) throws IOException ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); - requestData.put("name", tableName); + requestData.put("table_name", tableName); String jsonData = JSON.toJSONString(requestData); request.setData(jsonData); return execute(tableClient, request); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java index a7a7af35..7b827226 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java @@ -35,7 +35,7 @@ public Boolean tableExists(String tableName) throws IOException { ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); - requestData.put("name", tableName); + requestData.put("table_name", tableName); String jsonData = JSON.toJSONString(requestData); request.setData(jsonData); return execute(tableClient, request); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index e4b2d3d7..a499d73b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -268,34 +268,62 @@ public void testGetStartEndKeysOHTablePoolLoadNon() throws Exception { Assert.assertEquals(0, startEndKeys.getSecond()[0].length); } - /* - * - CREATE DATABASE IF NOT EXISTS `n1`; - CREATE TABLEGROUP `n1:test_multi_cf` SHARDING = 'ADAPTIVE'; - CREATE TABLE `n1:test_multi_cf$family_with_group1` ( - `K` varbinary(1024) NOT NULL, - `Q` varbinary(256) NOT NULL, - `T` bigint(20) NOT NULL, - `V` varbinary(1024) DEFAULT NULL, - PRIMARY KEY (`K`, `Q`, `T`) - ) TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3; - CREATE TABLE `n1:test_multi_cf$family_with_group2` ( - `K` varbinary(1024) NOT NULL, - `Q` varbinary(256) NOT NULL, - `T` bigint(20) NOT NULL, - `V` varbinary(1024) DEFAULT NULL, - PRIMARY KEY (`K`, `Q`, `T`) - ) TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3; - CREATE TABLE `n1:test_multi_cf$family_with_group3` ( - `K` varbinary(1024) NOT NULL, - `Q` varbinary(256) NOT NULL, - `T` bigint(20) NOT NULL, - `V` varbinary(1024) DEFAULT NULL, - PRIMARY KEY (`K`, `Q`, `T`) - ) TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3; - * */ @Test public void testAdminGetRegionMetrics() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + Statement st = conn.createStatement(); + st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE DATABASE IF NOT EXISTS `n1`;\n" + + "use `n1`;\n" + + "CREATE TABLEGROUP IF NOT EXISTS `n1:test_multi_cf` SHARDING = 'ADAPTIVE';\n" + + "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group2` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group3` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;"); + st.close(); + conn.close(); String tablegroup1 = "test_multi_cf"; String tablegroup2 = "n1:test_multi_cf"; Configuration conf = ObHTableTestUtil.newConfiguration(); @@ -319,7 +347,7 @@ public void testAdminGetRegionMetrics() throws Exception { + ", storeFileSize: " + regionMetrics.getStoreFileSize() + ", memFileSize: " + regionMetrics.getMemStoreSize()); } - // concurrently read while writing 150 thousand of rows to each table + // concurrently read while writing 150 thousand of rows to 2 tablegroups ExecutorService executorService = Executors.newFixedThreadPool(10); CountDownLatch latch = new CountDownLatch(100); List exceptionCatcher = new ArrayList<>(); @@ -415,6 +443,8 @@ public void testAdminDeleteTable() throws Exception { " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + ") TABLEGROUP = `n1:test`;"); + st.close(); + conn.close(); Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); @@ -434,9 +464,62 @@ public void testAdminDeleteTable() throws Exception { @Test public void testAdminTableExists() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + Statement st = conn.createStatement(); + st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE DATABASE IF NOT EXISTS `n1`;\n" + + "use `n1`;\n" + + "CREATE TABLEGROUP IF NOT EXISTS `n1:test` SHARDING = 'ADAPTIVE';\n" + + "CREATE TABLE IF NOT EXISTS `n1:test$family_group` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `n1:test`;" + + "\n" + + "CREATE TABLE IF NOT EXISTS `n1:test$family1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `n1:test`;"); + st.close(); + conn.close(); Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); + // TableName cannot contain $ symbol + Assert.assertThrows(IllegalArgumentException.class, + () -> { + TableName.valueOf("random_string$"); + }); Assert.assertFalse(admin.tableExists(TableName.valueOf("tablegroup_not_exists"))); Assert.assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); Assert.assertTrue(admin.tableExists(TableName.valueOf("n1", "test"))); From d398946699824b82d2dfc70dc3fc6bb20170b2a6 Mon Sep 17 00:00:00 2001 From: zjw Date: Thu, 29 May 2025 15:40:03 +0800 Subject: [PATCH 07/36] add enable disable to OHTableAccessControlExecutor --- .../util/OHTableAccessControlExecutor.java | 55 ++++++++ .../hbase/OHTableAdminInterfaceTest.java | 130 ++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java new file mode 100644 index 00000000..e2aadd0f --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java @@ -0,0 +1,55 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import org.apache.hadoop.hbase.TableNotDisabledException; +import org.apache.hadoop.hbase.TableNotEnabledException; +import org.apache.hadoop.hbase.TableNotFoundException; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class OHTableAccessControlExecutor extends AbstractObTableMetaExecutor { + private final ObTableClient tableClient; + private final ObTableRpcMetaType type; + + OHTableAccessControlExecutor(ObTableClient tableClient, ObTableRpcMetaType type) { + this.tableClient = tableClient; + this.type = type; + } + + @Override + public ObTableRpcMetaType getMetaType() throws IOException { + return this.type; + } + + @Override + public Boolean parse(ObTableMetaResponse response) throws IOException { + } + + public void enableTable(String tableName) throws IOException, TableNotFoundException, TableNotEnabledException { + ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestData = new HashMap<>(); + requestData.put("name", tableName); + String jsonData = JSON.toJSONString(requestData); + request.setData(jsonData); + execute(tableClient, request); + } + + public void disableTable(String tableName) throws IOException, TableNotFoundException, TableNotDisabledException { + ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestData = new HashMap<>(); + requestData.put("name", tableName); + String jsonData = JSON.toJSONString(requestData); + request.setData(jsonData); + execute(tableClient, request); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 6d53616b..82c9c01d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -18,15 +18,30 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.RegionMetrics; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; + import org.junit.Assert; +import org.junit.Assert.*; import org.junit.Test; + import java.io.IOException; +import java.sql.Statement; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.Executors; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.*; public class OHTableAdminInterfaceTest { public OHTablePool setUpLoadPool() throws IOException { @@ -249,4 +264,119 @@ public void testGetStartEndKeysOHTablePoolLoadNon() throws Exception { Assert.assertEquals(0, startEndKeys.getFirst()[0].length); Assert.assertEquals(0, startEndKeys.getSecond()[0].length); } + + @Test + public void testAdminEnDisableTable() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + Statement st = conn.createStatement(); + st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE DATABASE IF NOT EXISTS `n1`;\n" + + "use `n1`;\n" + + "CREATE TABLEGROUP IF NOT EXISTS `n1:test` SHARDING = 'ADAPTIVE';\n" + + "CREATE TABLE IF NOT EXISTS `n1:test$family_group` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `n1:test`;" + + "\n" + + "CREATE TABLE IF NOT EXISTS `n1:test$family1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `n1:test`;"); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + admin.disableTable(TableName.valueOf("test_multi_cf")); + assertTrue(admin.tableExists(TableName.valueOf("n1", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); + // disable a non-existed table + IOException thrown = assertThrows(IOException.class, + () -> { + admin.disableTable(TableName.valueOf("tablegroup_not_exists")); + }); + assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + + // write an enabled table, should succeed + batchInsert(10, "test_multi_ch"); + // disable a disabled table + thrown = assertThrows(IOException.class, + () -> { + admin.disableTable(TableName.valueOf("test_multi_cf")); + }); + assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + + // write an enabled table, should fail + batchInsert(10, "test_multi_ch"); + enDisableRead(10, "test_multi_ch"); + + // enable a disabled table + admin.enableTable(TableName.valueOf("test_multi_cf")); + + // write an enabled table, should succeed + batchInsert(10, "test_multi_ch"); + enDisableRead(10, "test_multi_ch"); + + // enable an enabled table + thrown = assertThrows(IOException.class, + () -> { + admin.disableTable(TableName.valueOf("n1", "test"); + }); + assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + + admin.deleteTable(TableName.valueOf("n1", "test")); + admin.deleteTable(TableName.valueOf("test_multi_cf")); + assertFalse(admin.tableExists(TableName.valueOf("n1", "test"))); + assertFalse(admin.tableExists(TableName.valueOf("test_multi_cf"))); + } + + private void enDisableRead(int rows, String tablegroup) throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Table table = connection.getTable(TableName.valueOf(tablegroup)); + List batchLsit = new LinkedList<>(); + for (int i = 0; i < rows; ++i) { + Get get = new Get(toBytes("Key" + i)); + batchLsit.add(get); + if (i % 100 == 0) { // 100 rows one batch to avoid OB_TIMEOUT + Object[] results = new Object[batchLsit.size()]; + table.batch(batchLsit, results); + batchLsit.clear(); + } + } + Object[] results = new Object[batchLsit.size()]; + table.batch(batchLsit, results); + } } From 62c5c583963b4ed0bd6401bb7f60bd5a2c9cee73 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Thu, 29 May 2025 20:12:02 +0800 Subject: [PATCH 08/36] create_table; region_locator; table_descriptor --- .../com/alipay/oceanbase/hbase/OHTable.java | 8 +- .../alipay/oceanbase/hbase/util/OHAdmin.java | 42 +++- .../hbase/util/OHConnectionImpl.java | 2 +- .../hbase/util/OHCreateTableExecutor.java | 69 +++++++ .../oceanbase/hbase/util/OHRegionLocator.java | 51 ++++- .../hbase/util/OHRegionLocatorExecutor.java | 191 ++++++++++++++++-- .../util/OHTableAccessControlExecutor.java | 3 +- .../hbase/util/OHTableDescriptorExecutor.java | 86 ++++++++ .../oceanbase/hbase/OHConnectionTest.java | 152 +++++++++++++- .../oceanbase/hbase/OHTableClientTest.java | 59 ++++++ src/test/java/unit_test_db.sql | 60 ++++++ 11 files changed, 691 insertions(+), 32 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 345fd894..83c2a9ab 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -499,13 +499,15 @@ public Configuration getConfiguration() { } @Override - public HTableDescriptor getTableDescriptor() { - throw new FeatureNotSupportedException("not supported yet."); + public HTableDescriptor getTableDescriptor() throws IOException { + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableNameString, obTableClient); + return executor.getTableDescriptor(); } @Override public TableDescriptor getDescriptor() throws IOException { - throw new FeatureNotSupportedException("not supported yet."); + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableNameString, obTableClient); + return executor.getTableDescriptor(); } /** diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index c2a41d57..63a942a2 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -135,17 +135,53 @@ public TableName[] listTableNames(String s, boolean b) throws IOException { @Override public HTableDescriptor getTableDescriptor(TableName tableName) throws TableNotFoundException, IOException { - throw new FeatureNotSupportedException("does not support yet"); + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableName.getNameAsString(), tableClient); + try { + return executor.getTableDescriptor(); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } } @Override public TableDescriptor getDescriptor(TableName tableName) throws TableNotFoundException, IOException { - throw new FeatureNotSupportedException("does not support yet"); + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableName.getNameAsString(), tableClient); + try { + return executor.getTableDescriptor(); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } } @Override public void createTable(TableDescriptor tableDescriptor) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableDescriptor.getTableName(), connectionConf); + OHCreateTableExecutor executor = new OHCreateTableExecutor(tableClient); + try { + executor.createTable(tableDescriptor, null); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } } @Override diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java index d134ca8d..fd666159 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHConnectionImpl.java @@ -145,7 +145,7 @@ public RegionLocator getRegionLocator(TableName tableName) throws IOException { // to avoid change the database in original param url by namespace in tableName OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); ObTableClient obTableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(obTableClient); + OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), obTableClient); return executor.getRegionLocator(String.valueOf(tableName)); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java new file mode 100644 index 00000000..a56e8097 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java @@ -0,0 +1,69 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.alibaba.fastjson.JSON; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; + +import java.io.IOException; +import java.util.*; + +public class OHCreateTableExecutor extends AbstractObTableMetaExecutor { + private final ObTableClient client; + + OHCreateTableExecutor(ObTableClient client) { + this.client = client; + } + + @Override + public ObTableRpcMetaType getMetaType() { + return ObTableRpcMetaType.HTABLE_CREATE_TABLE; + } + + @Override + public Void parse(ObTableMetaResponse response) throws IOException { + // success, do nothing + return null; + } + + public void createTable(TableDescriptor tableDescriptor, byte[][] splitKeys) throws IOException { + final ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + Map requestData = new HashMap<>(); + requestData.put("htable_name", tableDescriptor.getTableName().getName()); + Map> columnFamilies = new HashMap<>(); + for (ColumnFamilyDescriptor columnDescriptor : tableDescriptor.getColumnFamilies()) { + Map columnFamily = new HashMap<>(); + columnFamily.put("ttl", columnDescriptor.getTimeToLive()); + columnFamily.put("max_version", columnDescriptor.getMaxVersions()); + columnFamilies.put(columnDescriptor.getNameAsString(), columnFamily); + } + requestData.put("column_families", columnFamilies); + String jsonData = JSON.toJSONString(requestData); + request.setData(jsonData); + execute(client, request); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java index f4474d92..005982ee 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java @@ -1,9 +1,12 @@ package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.bolt.transport.TransportCodes; +import com.alipay.oceanbase.rpc.exception.ObTableTransportException; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.exceptions.TimeoutIOException; import org.apache.hadoop.hbase.util.Pair; import java.io.IOException; @@ -11,23 +14,53 @@ import java.util.List; public class OHRegionLocator implements RegionLocator { - public OHRegionLocator(byte[][] startKeys, byte[][] endKeys) { - + private byte[][] startKeys; + private byte[][] endKeys; + private TableName tableName; + + private List regionLocations; + + public OHRegionLocator(byte[][] startKeys, byte[][] endKeys, List regionLocations) { + this.startKeys = startKeys; + this.endKeys = endKeys; + this.regionLocations = regionLocations; } @Override public HRegionLocation getRegionLocation(byte[] bytes) throws IOException { + // check if bytes is in the range of startKeys and endKeys + for (HRegionLocation regionLocation : regionLocations) { + if (regionLocation.getRegionInfo().containsRow(bytes)) { + return regionLocation; + } + } return null; } @Override public HRegionLocation getRegionLocation(byte[] bytes, boolean b) throws IOException { - return null; + if (b) { + OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), tableClient); + try { + RegionLocator location = executor.getRegionLocator(tableName.toString()); + this.startKeys = location.getStartKeys(); + this.endKeys = location.getEndKeys(); + this.regionLocations = location.getAllRegionLocations(); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } + } + return getRegionLocation(bytes); } @Override public List getAllRegionLocations() throws IOException { - return Collections.emptyList(); + return regionLocations; } /** @@ -40,7 +73,7 @@ public List getAllRegionLocations() throws IOException { */ @Override public byte[][] getStartKeys() throws IOException { - return null; + return startKeys; } /** @@ -53,7 +86,7 @@ public byte[][] getStartKeys() throws IOException { */ @Override public byte[][] getEndKeys() throws IOException { - return null; + return endKeys; } /** @@ -67,18 +100,18 @@ public byte[][] getEndKeys() throws IOException { */ @Override public Pair getStartEndKeys() throws IOException { - return null; + return Pair.newPair(startKeys, endKeys); } @Override public TableName getName() { - return null; + return tableName; } private ObTableClient tableClient; @Override public void close() throws IOException { - + return; } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java index 0f896827..0b366e6a 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java @@ -1,49 +1,212 @@ package com.alipay.oceanbase.hbase.util; import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.constant.Constants; import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.location.model.TableEntry; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import org.apache.hadoop.hbase.*; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; public class OHRegionLocatorExecutor extends AbstractObTableMetaExecutor { - OHRegionLocatorExecutor(ObTableClient client) { + private final String tableName; + private final ObTableClient client; + + OHRegionLocatorExecutor(String tableName, ObTableClient client) { + this.tableName = tableName; this.client = client; } - + @Override public ObTableRpcMetaType getMetaType() { return ObTableRpcMetaType.HTABLE_REGION_LOCATOR; } + /** + * Parses the response and creates a region locator + * @param response response from the server + * @return OHRegionLocator + * @throws IOException if failed to parse the response + */ @Override public OHRegionLocator parse(ObTableMetaResponse response) throws IOException { try { - String jsonData = response.getData(); - // process json - return new OHRegionLocator(null, null); + final String jsonData = response.getData(); + final JSONObject jsonMap = Optional.ofNullable(JSON.parseObject(jsonData)) + .orElseThrow(() -> new IOException("jsonMap is null")); + /* + { + "table_id_dict": [1001, 1002], + "replica_dict": [ + ["127.0.0.1", 2881], + ["127.0.0.2", 2882], + ["127.0.0.3", 2883] + ], + "partitions": [ + // 表1001的3个分区，每个分区3副本 + [0, 50001, "rowkey_1", 0, 1], // leader + [0, 50001, "rowkey_1", 1, 0], // follower + [0, 50001, "rowkey_1", 2, 0], // follower + [0, 50002, "rowkey_2", 0, 1], + [0, 50002, "rowkey_2", 1, 0], + [0, 50002, "rowkey_2", 2, 0], + [0, 50003, "rowkey_3", 0, 1], + [0, 50003, "rowkey_3", 1, 0], + [0, 50003, "rowkey_3", 2, 0], + + // 表1002的3个分区，每个分区3副本 + [1, 50004, "rowkey_1", 0, 1], + [1, 50004, "rowkey_1", 1, 0], + [1, 50004, "rowkey_1", 2, 0], + [1, 50005, "rowkey_2", 0, 1], + [1, 50005, "rowkey_2", 1, 0], + [1, 50005, "rowkey_2", 2, 0], + [1, 50006, "rowkey_3", 0, 1], + [1, 50006, "rowkey_3", 1, 0], + [1, 50006, "rowkey_3", 2, 0] + ] + } + */ + + final List partitions = Optional.>ofNullable(jsonMap.getJSONArray("partitions")) + .orElseThrow(() -> new IOException("partitions is null")); + + final List tableIdDict = Optional.>ofNullable(jsonMap.getJSONArray("table_id_dict")) + .orElseThrow(() -> new IOException("tableIdDict is null")); + final List replicaDict = Optional.>ofNullable(jsonMap.getJSONArray("replica_dict")) + .orElseThrow(() -> new IOException("replicaDict is null")); + + final boolean isHashLikePartition = partitions.stream() + .map(obj -> (List) obj) + .filter(partition -> { + if (partition.size() <= 3) { + throw new IllegalArgumentException("partition size is not 3"); + } + return true; + }) + .allMatch(partition -> { + final byte[] highBound = partition.get(2).toString().getBytes(); + return Arrays.equals(highBound, Constants.EMPTY_STRING.getBytes()); + }); + return isHashLikePartition ? + createHashPartitionLocator(tableIdDict, replicaDict, partitions) : + createRangePartitionLocator(tableIdDict, replicaDict, partitions); } catch (IllegalArgumentException e) { - throw new IOException("msg", e); + throw new IOException("Invalid partition data: " + e.getMessage(), e); } } - public OHRegionLocator getRegionLocator(String tableName) throws IOException { - ObTableMetaRequest request = new ObTableMetaRequest(); + /** + * Creates a region locator for range partitions + * @param tableIdDict table ID dictionary + * @param replicaDict replica dictionary + * @param partitions list of partition data + * @return OHRegionLocator for range partitions + */ + private OHRegionLocator createRangePartitionLocator( + final List tableIdDict, + final List replicaDict, + final List partitions) { + final int partitionCount = partitions.size(); + final int regionCount = partitionCount + 1; + final byte[][] startKeys = new byte[regionCount][]; + final byte[][] endKeys = new byte[regionCount][]; + + for (int i = 0; i < regionCount; i++) { + if (i == 0) { + startKeys[i] = HConstants.EMPTY_BYTE_ARRAY; + endKeys[i] = ((List) partitions.get(i)).get(2).toString().getBytes(); + } else if (i == regionCount - 1) { + startKeys[i] = ((List) partitions.get(i - 1)).get(2).toString().getBytes(); + endKeys[i] = HConstants.EMPTY_BYTE_ARRAY; + } else { + startKeys[i] = ((List) partitions.get(i - 1)).get(2).toString().getBytes(); + endKeys[i] = ((List) partitions.get(i)).get(2).toString().getBytes(); + } + } + + // Create region locations for all regions + final List regionLocations = IntStream.range(0, regionCount) + .mapToObj(i -> { + final List partition = (List) partitions.get(Math.min(i, partitionCount - 1)); + final int replicationIdx = (int) partition.get(3); + final List hostInfo = (List) replicaDict.get(replicationIdx); + + final ServerName serverName = ServerName.valueOf( + hostInfo.get(0).toString(), + (int) hostInfo.get(1), + i + ); + final HRegionInfo regionInfo = new HRegionInfo( + TableName.valueOf(tableName), + startKeys[i], + endKeys[i] + ); + return new HRegionLocation(regionInfo, serverName, i); + }) + .collect(Collectors.toList()); + + return new OHRegionLocator(startKeys, endKeys, regionLocations); + } + + /** + * Creates a region locator for hash partitions + * @param tableIdDict table ID dictionary + * @param replicaDict replica dictionary + * @param partitions list of partition data + * @return OHRegionLocator for hash partitions + */ + private OHRegionLocator createHashPartitionLocator( + final List tableIdDict, + final List replicaDict, + final List partitions) { + + final byte[][] startKeys = new byte[1][]; + final byte[][] endKeys = new byte[1][]; + startKeys[0] = HConstants.EMPTY_BYTE_ARRAY; + endKeys[0] = HConstants.EMPTY_BYTE_ARRAY; + + final List regionLocations = IntStream.range(0, partitions.size()) + .mapToObj(i -> { + final List partition = (List) partitions.get(i); + final int replicationIdx = (int) partition.get(3); + final List hostInfo = (List) replicaDict.get(replicationIdx); + + final ServerName serverName = ServerName.valueOf( + hostInfo.get(0).toString(), + (int) hostInfo.get(1), + i + ); + final HRegionInfo regionInfo = new HRegionInfo( + TableName.valueOf(tableName), + startKeys[0], + endKeys[0] + ); + return new HRegionLocation(regionInfo, serverName, i); + }) + .collect(Collectors.toList()); + + return new OHRegionLocator(startKeys, endKeys, regionLocations); + } + + public OHRegionLocator getRegionLocator(final String tableName) throws IOException { + final ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); - Map requestData = new HashMap<>(); + final Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - String jsonData = JSON.toJSONString(requestData); + + final String jsonData = JSON.toJSONString(requestData); request.setData(jsonData); return execute(client, request); } - - private final ObTableClient client; } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java index e2aadd0f..06719ecf 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java @@ -30,7 +30,8 @@ public ObTableRpcMetaType getMetaType() throws IOException { } @Override - public Boolean parse(ObTableMetaResponse response) throws IOException { + public Void parse(ObTableMetaResponse response) throws IOException { + return null; } public void enableTable(String tableName) throws IOException, TableNotFoundException, TableNotEnabledException { diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java new file mode 100644 index 00000000..68bcbd13 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java @@ -0,0 +1,86 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; +import com.alipay.oceanbase.rpc.ObTableClient; +import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; +import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import sun.font.SunFontManager; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public class OHTableDescriptorExecutor extends AbstractObTableMetaExecutor { + private final String tableName; + private final ObTableClient client; + + public OHTableDescriptorExecutor(String tableName, ObTableClient client) { + this.tableName = tableName; + this.client = client; + } + + @Override + public HTableDescriptor parse(ObTableMetaResponse response) throws IOException { + try { + final String jsonData = response.getData(); + final JSONObject jsonMap = Optional.ofNullable(JSON.parseObject(jsonData)) + .orElseThrow(() -> new IOException("jsonMap is null")); + /* + { + "cfDesc": { + "cf1": { + "TTL":604800 + }, + "cf2": { + "TTL":259200 + } + }, + "tbDesc": { + "name":"test" + } + } + */ + HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName)); + JSONObject cfDesc = jsonMap.getJSONObject("cfDescs"); + if (cfDesc != null) { + for (Map.Entry entry : cfDesc.entrySet()) { + String cfName = entry.getKey(); + JSONObject attributes = (JSONObject) entry.getValue(); + HColumnDescriptor cf = new HColumnDescriptor(cfName); + cf.setTimeToLive(attributes.getIntValue("TTL")); + tableDescriptor.addFamily(cf); + } + } else { + throw new IOException("cfDesc is null"); + } + return tableDescriptor; + } catch (IllegalArgumentException e) { + throw new IOException("Failed to parse response", e); + } + } + + @Override + public ObTableRpcMetaType getMetaType() throws IOException { + return ObTableRpcMetaType.HTABLE_GET_DESC; + } + + + public HTableDescriptor getTableDescriptor() throws IOException { + final ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + final Map requestData = new HashMap<>(); + requestData.put("table_name", tableName); + + final String jsonData = JSON.toJSONString(requestData); + request.setData(jsonData); + + return execute(client, request); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index cd2e7689..92c41b1c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.*; import static com.alipay.oceanbase.hbase.constants.OHConstants.SOCKET_TIMEOUT; @@ -81,7 +82,7 @@ public void testRefreshTableEntry() throws Exception { @After public void after() throws IOException { - hTable.close(); + if (hTable != null) hTable.close(); } @Test @@ -841,6 +842,155 @@ public void testBufferedMutatorConcurrent() throws Exception { } } + + /* + CREATE TABLEGROUP test_region_locator SHARDING = 'ADAPTIVE'; + CREATE TABLE `test_region_locator$family_region_locator` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = test_region_locator PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) + ); + */ + @Test + public void testRangePartitionWithRegionLocator() throws Exception { + final String tableNameStr = "test_region_locator"; + final String family = "family_region_locator"; + final int regionCount = 10; + final int rowsPerRegion = 5; + + final byte[][] splitPoints = new byte[][] { + Bytes.toBytes("c"), // p1: < 'c' + Bytes.toBytes("e"), // p2: < 'e' + Bytes.toBytes("g"), // p3: < 'g' + Bytes.toBytes("i"), // p4: < 'i' + Bytes.toBytes("l"), // p5: < 'l' + Bytes.toBytes("n"), // p6: < 'n' + Bytes.toBytes("p"), // p7: < 'p' + Bytes.toBytes("s"), // p8: < 's' + Bytes.toBytes("v") // p9: < 'v' + }; + + final TableName tableName = TableName.valueOf(tableNameStr); + final Configuration conf = ObHTableTestUtil.newConfiguration(); + connection = ConnectionFactory.createConnection(conf); + hTable = connection.getTable(tableName); + + try { + for (int i = 0; i < regionCount; i++) { + for (int j = 0; j < rowsPerRegion; j++) { + String rowKey; + if (i == 0) { + rowKey = "a" + j; + } else if (i == regionCount - 1) { + rowKey = "v" + j; + } else { + String baseKey = Bytes.toString(splitPoints[i - 1]); + rowKey = baseKey + j; + } + + Put put = new Put(Bytes.toBytes(rowKey)); + String value = String.format("value_%d_%d", i, j); + put.addColumn( + Bytes.toBytes(family), + Bytes.toBytes("q"), + Bytes.toBytes(value) + ); + hTable.put(put); + } + } + + try (RegionLocator locator = connection.getRegionLocator(tableName)) { + byte[][] startKeys = locator.getStartKeys(); + byte[][] endKeys = locator.getEndKeys(); + + Assert.assertEquals("Should have " + regionCount + " regions", + regionCount, startKeys.length); + + for (int i = 0; i < startKeys.length; i++) { + String startKeyStr = startKeys[i].length == 0 ? "-∞" : + Bytes.toString(startKeys[i]); + String endKeyStr = endKeys[i].length == 0 ? "+∞" : + Bytes.toString(endKeys[i]); + // 验证region边界 + if (i > 0) { + // 验证startKey与前一个endKey相同 + Assert.assertArrayEquals("Region " + i + " startKey should match previous endKey", + endKeys[i-1], startKeys[i]); + } + } + + for (int i = 0; i < startKeys.length; i++) { + Scan scan = new Scan(); + if (startKeys[i].length > 0) { + scan.setStartRow(startKeys[i]); + } + if (endKeys[i].length > 0) { + scan.setStopRow(endKeys[i]); + } + + String startKeyStr = startKeys[i].length == 0 ? "-∞" : + Bytes.toString(startKeys[i]); + String endKeyStr = endKeys[i].length == 0 ? "+∞" : + Bytes.toString(endKeys[i]); + + try (ResultScanner scanner = hTable.getScanner(scan)) { + List results = new ArrayList<>(); + for (Result result : scanner) { + results.add(result); + } + + Assert.assertEquals("Region " + i + " should have " + rowsPerRegion + " rows", + rowsPerRegion, results.size()); + + for (Result result : results) { + String rowKey = Bytes.toString(result.getRow()); + String value = Bytes.toString(result.getValue( + Bytes.toBytes(family), Bytes.toBytes("q"))); + if (startKeys[i].length > 0) { + Assert.assertTrue("Row key " + rowKey + " should be >= " + + Bytes.toString(startKeys[i]), + rowKey.compareTo(Bytes.toString(startKeys[i])) >= 0); + } + if (endKeys[i].length > 0) { + Assert.assertTrue("Row key " + rowKey + " should be < " + + Bytes.toString(endKeys[i]), + rowKey.compareTo(Bytes.toString(endKeys[i])) < 0); + } + } + } + } + } + } finally { + Optional.ofNullable(hTable).ifPresent(table -> { + try { + table.close(); + } catch (IOException e) { + e.printStackTrace(); + } + }); + Optional.ofNullable(connection).ifPresent(conn -> { + try { + conn.close(); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + } + @Test public void testBufferedMutatorPeriodicFlush() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java index 233e31ed..93bd4725 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java @@ -18,6 +18,7 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import org.apache.hadoop.hbase.HTableDescriptor; import org.junit.*; import java.util.LinkedList; @@ -67,6 +68,64 @@ public void testNew() throws Exception { assertTrue(true); } + + /* + CREATE TABLEGROUP test_desc SHARDING = 'ADAPTIVE'; + CREATE TABLE `test_desc$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = test_desc PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) + ); + + CREATE TABLE `test_desc$family2` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) + ) TABLEGROUP = test_desc PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) + ); + */ + @Test + public void testGetTableDescriptor() throws Exception { + final String tableNameStr = "test_desc"; + + OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient(tableNameStr); + hTable2.init(); + try { + HTableDescriptor descriptor = hTable2.getTableDescriptor(); + Assert.assertNotNull(descriptor); + Assert.assertTrue(descriptor.hasFamily("family1".getBytes())); + Assert.assertTrue(descriptor.hasFamily("family2".getBytes())); + Assert.assertFalse(descriptor.hasFamily("family".getBytes())); + } finally { + hTable2.close(); + } + } + @AfterClass public static void finish() throws Exception { hTable.close(); diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index 11a06bde..f6b1068f 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -276,3 +276,63 @@ CREATE TABLE `n1:test$partitionFamily1` ( `V` varbinary(1024) DEFAULT NULL, PRIMARY KEY (`K`, `Q`, `T`) ) partition by key(`K`) partitions 17; + +CREATE TABLEGROUP test_region_locator SHARDING = 'ADAPTIVE'; +CREATE TABLE `test_region_locator$family_region_locator` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_region_locator PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) +); + +CREATE TABLEGROUP test_desc SHARDING = 'ADAPTIVE'; + +CREATE TABLE `test_desc$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_desc PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) +); + +CREATE TABLE `test_desc$family2` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_desc PARTITION BY RANGE COLUMNS(K) ( + PARTITION p1 VALUES LESS THAN ('c'), + PARTITION p2 VALUES LESS THAN ('e'), + PARTITION p3 VALUES LESS THAN ('g'), + PARTITION p4 VALUES LESS THAN ('i'), + PARTITION p5 VALUES LESS THAN ('l'), + PARTITION p6 VALUES LESS THAN ('n'), + PARTITION p7 VALUES LESS THAN ('p'), + PARTITION p8 VALUES LESS THAN ('s'), + PARTITION p9 VALUES LESS THAN ('v'), + PARTITION p10 VALUES LESS THAN (MAXVALUE) +); From e1335341c2d37f1e53a98c4521667e2106f3fd6d Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Fri, 30 May 2025 16:34:55 +0800 Subject: [PATCH 09/36] [Test] add create/delete table testcases --- .../hbase/OHTableAdminInterfaceTest.java | 296 +++++++++++++++++- 1 file changed, 290 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index f724d98f..209e6e8a 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -20,12 +20,11 @@ import com.alipay.oceanbase.hbase.util.OHRegionMetrics; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.hbase.util.ResultSetPrinter; import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.RegionMetrics; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -39,14 +38,13 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.*; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; import static org.junit.Assert.assertFalse; +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; public class OHTableAdminInterfaceTest { public OHTablePool setUpLoadPool() throws IOException { @@ -689,4 +687,290 @@ private void batchInsert(int rows, String tablegroup) throws Exception { Object[] results = new Object[batchLsit.size()]; table.batch(batchLsit, results); } + + @Test + public void testCreateDeleteTable() throws Exception { + TableName tableName = TableName.valueOf("testCreateTable"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + byte[] cf3 = Bytes.toBytes("cf3"); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + // 1. construct htable desc and column family desc + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + hcd1.setMaxVersions(2); + hcd1.setTimeToLive(172800); + + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + hcd2.setMaxVersions(1); + hcd2.setTimeToLive(86400); + + HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + + // 2. execute create table and check exists + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + htd.addFamily(hcd3); + admin.createTable(htd); + + // 3. check table creation success and correctness + assertTrue(admin.tableExists(tableName)); + // TODO: show create table, need to be replace by getDescriptor + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + String selectSql = "show create table " + tableName.getNameAsString() + "$" + cf1; + System.out.println("execute sql: " + selectSql); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery(selectSql); + ResultSetPrinter.print(resultSet); + + selectSql = "show create table " + tableName.getNameAsString() + "$" + cf2; + System.out.println("execute sql: " + selectSql); + resultSet = conn.createStatement().executeQuery(selectSql); + ResultSetPrinter.print(resultSet); + + selectSql = "show create table " + tableName.getNameAsString() + "$" + cf3; + System.out.println("execute sql: " + selectSql); + resultSet = conn.createStatement().executeQuery(selectSql); + ResultSetPrinter.print(resultSet); + + + // 4. test put/get some data + Table table = connection.getTable(tableName); + Put put = new Put(toBytes("Key" + 1)); + put.addColumn(cf1, "c1".getBytes(), "hello world".getBytes()); + put.addColumn(cf2, "c2".getBytes(), "hello world".getBytes()); + put.addColumn(cf3, "c3".getBytes(), "hello world".getBytes()); + table.put(put); + + Scan scan = new Scan(); + ResultScanner resultScanner = table.getScanner(scan); + List cells = getCellsFromScanner(resultScanner); + Assert.assertEquals(3, cells.size()); + + // 5. disable and delete table + admin.disableTable(tableName); + admin.deleteTable(tableName); + + // 5. test table exists after delete + admin.tableExists(tableName); + + // 6. recreate and delete table + admin.createTable(htd); + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + + void testConcurCreateDelTablesHelper(List tableNames, Boolean ignoreException) throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + int tableNums = tableNames.size(); + + // use some column family desc + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + byte[] cf3 = Bytes.toBytes("cf3"); + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + hcd1.setMaxVersions(2); + hcd1.setTimeToLive(172800); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + hcd1.setMaxVersions(1); + hcd1.setTimeToLive(86400); + HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + + // 1. generate create table task, one task per table + List> tasks = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + int finalI = i; + tasks.add(()->{ + HTableDescriptor htd = new HTableDescriptor(tableNames.get(finalI)); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + htd.addFamily(hcd3); + try { + admin.createTable(htd); + } catch (Exception e) { + System.out.println(e); + if (!ignoreException) { + throw e; + } + } + return null; + }); + } + + // 2. execute concurrent create table tasks + ExecutorService executorService = Executors.newFixedThreadPool(tableNums); + executorService.invokeAll(tasks); + executorService.shutdown(); + executorService.awaitTermination(1, TimeUnit.MINUTES); + + // 3. check table create success + for (int i = 0; i < tableNames.size(); i++) { + TableName tableName = tableNames.get(i); + assertTrue(admin.tableExists(tableName)); + } + + // 4. test put/get/delete some data + for (int i = 0; i < tableNames.size(); i++) { + Table table = connection.getTable(tableNames.get(i)); + Put put = new Put(toBytes("Key" + 1)); + put.addColumn(cf1, "c1".getBytes(), "hello world".getBytes()); + put.addColumn(cf2, "c2".getBytes(), "hello world".getBytes()); + put.addColumn(cf3, "c3".getBytes(), "hello world".getBytes()); + table.put(put); + + Scan scan = new Scan(); + ResultScanner resultScanner = table.getScanner(scan); + List cells = getCellsFromScanner(resultScanner); + Assert.assertEquals(3, cells.size()); + + table.delete(new Delete(toBytes("Key" + 1))); + } + + // 4. disable all tables; + for (int i = 0; i < tableNames.size(); i++) { + TableName tableName = tableNames.get(i); + admin.disableTable(tableName); + } + + // 5. generate delete table task + List> delTasks = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + int finalI = i; + delTasks.add(()->{ + try { + admin.deleteTable(tableNames.get(finalI)); + } catch (Exception e) { + System.out.println(e); + if (!ignoreException) { + throw e; + } + } + return null; + }); + } + + // 6. execute concurrent delete table tasks + ExecutorService delExecutorService = Executors.newFixedThreadPool(tableNums); + delExecutorService.invokeAll(delTasks); + delExecutorService.shutdown(); + delExecutorService.awaitTermination(1, TimeUnit.MINUTES); + + // 7. check table deletion success + for (int i = 0; i < tableNames.size(); i++) { + TableName tableName = tableNames.get(i); + assertFalse(admin.tableExists(tableName)); + } + } + + // 2. test concurrent create or delete different table + // step1: create different tables concurrently, it must succeed + // step2: execute put/read/delete on created table + // step3: delete different tables concurrently, it must succeed + @Test + public void testConcurCreateDelTables() throws Exception { + final int tableNums = 20; + List tableNames = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + tableNames.add(TableName.valueOf("testConcurCreateTable" + i)); + } + testConcurCreateDelTablesHelper(tableNames, false); + } + + // 3. test concurrent create or delete same table + // step1: create one table concurrently, only one table was successfully created + // step2: execute put/read/delete on created table + // step3: delete one table concurrently, the table will be deleted successfully + @Test + public void testConcurCreateOneTable() throws Exception { + final int taskNum = 20; + TableName tableName = TableName.valueOf("testConcurCreateOneTable"); + List tableNames = new ArrayList<>(); + for (int i = 0; i < taskNum; i++) { + tableNames.add(tableName); + } + testConcurCreateDelTablesHelper(tableNames, true); + } + + // 4. test the performance of concurrent create/delete table + @Test + public void testConcurCreateDelPerf() throws Exception { + final int tableNums = 100; + List tableNames = new ArrayList<>(); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + byte[] cf3 = Bytes.toBytes("cf3"); + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + hcd1.setMaxVersions(2); + hcd1.setTimeToLive(172800); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + hcd1.setMaxVersions(1); + hcd1.setTimeToLive(86400); + HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + + for (int i = 0; i < tableNums; i++) { + tableNames.add(TableName.valueOf("testConcurCreateDelPerf" + i)); + } + + List> tasks = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + int finalI = i; + tasks.add(()->{ + HTableDescriptor htd = new HTableDescriptor(tableNames.get(finalI)); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + htd.addFamily(hcd3); + try { + admin.createTable(htd); + } catch (Exception e) { + System.out.println(e); + } + return null; + }); + } + + // 2. execute concurrent create table tasks + long start = System.currentTimeMillis(); + ExecutorService executorService = Executors.newFixedThreadPool(tableNums); + executorService.invokeAll(tasks); + executorService.shutdown(); + executorService.awaitTermination(2, TimeUnit.MINUTES); + long duration = System.currentTimeMillis() - start; + System.out.println("create " + tableNums + " tables cost " + duration + " ms."); + + // 3. disable all tables; + for (int i = 0; i < tableNames.size(); i++) { + TableName tableName = tableNames.get(i); + admin.disableTable(tableName); + } + + // 4. generate delete table task + List> delTasks = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + int finalI = i; + delTasks.add(()->{ + try { + admin.deleteTable(tableNames.get(finalI)); + } catch (Exception e) { + System.out.println(e); + } + return null; + }); + } + + // 6. execute concurrent delete table tasks + start = System.currentTimeMillis(); + ExecutorService delExecutorService = Executors.newFixedThreadPool(tableNums); + delExecutorService.invokeAll(delTasks); + delExecutorService.shutdown(); + delExecutorService.awaitTermination(1, TimeUnit.MINUTES); + duration = System.currentTimeMillis() - start; + System.out.println("delete " + tableNums + " tables cost " + duration + " ms."); + } } From 40ae8e5355ffd90d8832fa4bce2418d2636bbf59 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Tue, 3 Jun 2025 15:29:31 +0800 Subject: [PATCH 10/36] add hash-like partition case for region locator --- .../hbase/util/OHTableDescriptorExecutor.java | 1 - .../oceanbase/hbase/OHConnectionTest.java | 17 +++++++++++++++++ .../hbase/OHTableAdminInterfaceTest.java | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java index 68bcbd13..94d7f754 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java @@ -10,7 +10,6 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; -import sun.font.SunFontManager; import java.io.IOException; import java.util.HashMap; diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 92c41b1c..31a6d800 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; @@ -30,6 +31,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.concurrent.*; @@ -990,6 +992,21 @@ public void testRangePartitionWithRegionLocator() throws Exception { }); } } + @Test + public void testKeyPartitionWithRegionLocator() throws IOException { + final String tableNameStr = "test_multi_cf"; + final TableName tableName = TableName.valueOf(tableNameStr); + final Configuration conf = ObHTableTestUtil.newConfiguration(); + connection = ConnectionFactory.createConnection(conf); + hTable = connection.getTable(tableName); + RegionLocator locator = connection.getRegionLocator(tableName); + byte[][] startKeys = locator.getStartKeys(); + byte[][] endKeys = locator.getEndKeys(); + Assert.assertEquals("Should have 1 region", 1, startKeys.length); + Assert.assertEquals("Should have 1 region", 1, endKeys.length); + Assert.assertEquals(startKeys[0], endKeys[0]); + Assert.assertEquals(startKeys[0], HConstants.EMPTY_BYTE_ARRAY); + } @Test public void testBufferedMutatorPeriodicFlush() throws Exception { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index f724d98f..ed656fbd 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -356,7 +356,7 @@ public void testAdminEnDisableTable() throws Exception { // enable an enabled table thrown = assertThrows(IOException.class, () -> { - admin.disableTable(TableName.valueOf("n1", "test"); + admin.disableTable(TableName.valueOf("n1", "test")); }); assertTrue(thrown.getCause() instanceof ObTableException); Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); From 5f90652e4b6305c352aadfb7c4f1c26d0df0192c Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Tue, 3 Jun 2025 19:37:21 +0800 Subject: [PATCH 11/36] fix region locator in flink --- .../oceanbase/hbase/util/OHRegionLocator.java | 12 +++--- .../hbase/util/OHRegionLocatorExecutor.java | 42 ++++++++++++------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java index 005982ee..ab99bc22 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java @@ -10,20 +10,24 @@ import org.apache.hadoop.hbase.util.Pair; import java.io.IOException; -import java.util.Collections; import java.util.List; public class OHRegionLocator implements RegionLocator { private byte[][] startKeys; private byte[][] endKeys; + private ObTableClient tableClient; private TableName tableName; private List regionLocations; - public OHRegionLocator(byte[][] startKeys, byte[][] endKeys, List regionLocations) { + public OHRegionLocator(byte[][] startKeys, byte[][] endKeys, + List regionLocations, + TableName tableName, ObTableClient tableClient) { this.startKeys = startKeys; this.endKeys = endKeys; this.regionLocations = regionLocations; + this.tableName = tableName; + this.tableClient = tableClient; } @Override @@ -39,7 +43,7 @@ public HRegionLocation getRegionLocation(byte[] bytes) throws IOException { @Override public HRegionLocation getRegionLocation(byte[] bytes, boolean b) throws IOException { - if (b) { + if (b || regionLocations.isEmpty()) { OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), tableClient); try { RegionLocator location = executor.getRegionLocator(tableName.toString()); @@ -108,8 +112,6 @@ public TableName getName() { return tableName; } - private ObTableClient tableClient; - @Override public void close() throws IOException { return; diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java index 0b366e6a..435cd6f4 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java @@ -6,6 +6,7 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.constant.Constants; import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; import com.alipay.oceanbase.rpc.location.model.TableEntry; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; @@ -116,28 +117,37 @@ private OHRegionLocator createRangePartitionLocator( final List tableIdDict, final List replicaDict, final List partitions) { - final int partitionCount = partitions.size(); - final int regionCount = partitionCount + 1; - final byte[][] startKeys = new byte[regionCount][]; - final byte[][] endKeys = new byte[regionCount][]; - - for (int i = 0; i < regionCount; i++) { + if ((partitions.size() % tableIdDict.size()) != 0) { + throw new ObTableUnexpectedException( + "The number of partitions should be an integer multiple of the number of tables"); + } + // the size of partitions the multiple of the number of zones, the number of tablets and the number of tables + final int regionCount = partitions.size() / tableIdDict.size(); // get tablet boundaries of leaders and followers + final List startKeysList = new ArrayList<>(); + final List endKeysList = new ArrayList<>(); + + for (int i = 0; i < regionCount; ++i) { + boolean isLeader = ((int) ((List) partitions.get(i)).get(4) == 1); + if (!isLeader) { // only record leader's boundary + continue; + } if (i == 0) { - startKeys[i] = HConstants.EMPTY_BYTE_ARRAY; - endKeys[i] = ((List) partitions.get(i)).get(2).toString().getBytes(); + startKeysList.add(HConstants.EMPTY_BYTE_ARRAY); + endKeysList.add(((List) partitions.get(i)).get(2).toString().getBytes()); } else if (i == regionCount - 1) { - startKeys[i] = ((List) partitions.get(i - 1)).get(2).toString().getBytes(); - endKeys[i] = HConstants.EMPTY_BYTE_ARRAY; + startKeysList.add(((List) partitions.get(i - 1)).get(2).toString().getBytes()); + endKeysList.add(HConstants.EMPTY_BYTE_ARRAY); } else { - startKeys[i] = ((List) partitions.get(i - 1)).get(2).toString().getBytes(); - endKeys[i] = ((List) partitions.get(i)).get(2).toString().getBytes(); + startKeysList.add(((List) partitions.get(i - 1)).get(2).toString().getBytes()); + endKeysList.add(((List) partitions.get(i)).get(2).toString().getBytes()); } } - + final byte[][] startKeys = startKeysList.toArray(new byte[0][]); + final byte[][] endKeys = endKeysList.toArray(new byte[0][]); // Create region locations for all regions final List regionLocations = IntStream.range(0, regionCount) .mapToObj(i -> { - final List partition = (List) partitions.get(Math.min(i, partitionCount - 1)); + final List partition = (List) partitions.get(Math.min(i, regionCount - 1)); final int replicationIdx = (int) partition.get(3); final List hostInfo = (List) replicaDict.get(replicationIdx); @@ -155,7 +165,7 @@ private OHRegionLocator createRangePartitionLocator( }) .collect(Collectors.toList()); - return new OHRegionLocator(startKeys, endKeys, regionLocations); + return new OHRegionLocator(startKeys, endKeys, regionLocations, TableName.valueOf(tableName), client); } /** @@ -195,7 +205,7 @@ private OHRegionLocator createHashPartitionLocator( }) .collect(Collectors.toList()); - return new OHRegionLocator(startKeys, endKeys, regionLocations); + return new OHRegionLocator(startKeys, endKeys, regionLocations, TableName.valueOf(tableName), client); } public OHRegionLocator getRegionLocator(final String tableName) throws IOException { From 61b48c2c07d80ecc29c86b5f23550de2bb1e400c Mon Sep 17 00:00:00 2001 From: GroundWu <1175416256@qq.com> Date: Wed, 4 Jun 2025 11:36:58 +0800 Subject: [PATCH 12/36] adapt for disable/enable table test (#228) * adapt for disable/enable table --- .../alipay/oceanbase/hbase/util/OHAdmin.java | 40 +++++- .../util/OHTableAccessControlExecutor.java | 5 +- .../hbase/util/OHTableDescriptorExecutor.java | 48 ++++++++ .../hbase/OHTableAdminInterfaceTest.java | 114 ++++++++++++------ 4 files changed, 165 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index 63a942a2..ba1e483f 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -4,6 +4,7 @@ import com.alipay.oceanbase.rpc.bolt.transport.TransportCodes; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import com.alipay.oceanbase.rpc.exception.ObTableTransportException; +import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; @@ -243,7 +244,19 @@ public Future truncateTableAsync(TableName tableName, boolean b) throws IO @Override public void enableTable(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_ENABLE_TABLE); + try { + executor.enableTable(tableName.getNameAsString()); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } } @Override @@ -268,7 +281,19 @@ public Future disableTableAsync(TableName tableName) throws IOException { @Override public void disableTable(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_DISABLE_TABLE); + try { + executor.disableTable(tableName.getNameAsString()); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } } @Override @@ -283,12 +308,19 @@ public HTableDescriptor[] disableTables(Pattern pattern) throws IOException { @Override public boolean isTableEnabled(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); + return isDisabled(tableName) == false; } @Override public boolean isTableDisabled(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); + return isDisabled(tableName) == true; + } + + private boolean isDisabled(TableName tableName) throws IOException { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableDescriptorExecutor tableDescriptor = new OHTableDescriptorExecutor(tableName.getNameAsString(), tableClient); + return tableDescriptor.isDisable(); } @Override diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java index 06719ecf..090da7e4 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java @@ -1,7 +1,6 @@ package com.alipay.oceanbase.hbase.util; import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -38,7 +37,7 @@ public void enableTable(String tableName) throws IOException, TableNotFoundExcep ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); - requestData.put("name", tableName); + requestData.put("table_name", tableName); String jsonData = JSON.toJSONString(requestData); request.setData(jsonData); execute(tableClient, request); @@ -48,7 +47,7 @@ public void disableTable(String tableName) throws IOException, TableNotFoundExce ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); - requestData.put("name", tableName); + requestData.put("table_name", tableName); String jsonData = JSON.toJSONString(requestData); request.setData(jsonData); execute(tableClient, request); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java index 94d7f754..dae981f0 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java @@ -7,6 +7,7 @@ import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import com.alipay.oceanbase.rpc.table.ObTable; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; @@ -82,4 +83,51 @@ public HTableDescriptor getTableDescriptor() throws IOException { return execute(client, request); } + + public boolean isDisable() throws IOException { + boolean isDisable = false; + final ObTableMetaRequest request = new ObTableMetaRequest(); + request.setMetaType(getMetaType()); + final Map requestData = new HashMap<>(); + requestData.put("table_name", tableName); + + final String jsonData = JSON.toJSONString(requestData); + request.setData(jsonData); + try { + ObTableMetaResponse response = innerExecute(client, request); + final String responseData = response.getData(); + final JSONObject jsonMap = Optional.ofNullable(JSON.parseObject(responseData)) + .orElseThrow(() -> new IOException("jsonMap is null")); + JSONObject tbDesc = jsonMap.getJSONObject("tableDesc"); + if (tbDesc != null) { + String state = tbDesc.getString("state"); + if (state.compareToIgnoreCase("disable") == 0) { + isDisable = true; + } else { + isDisable = false; + } + } + } catch (IOException e) { + throw e; + } + return isDisable; + } + + private ObTableMetaResponse innerExecute(ObTableClient client, ObTableMetaRequest request) throws IOException { + if (request.getMetaType() != getMetaType()) { + throw new IOException("Invalid meta type, expected " + getMetaType()); + } + ObTable table = client.getRandomTable(); + ObTableMetaResponse response; + try { + response = (ObTableMetaResponse) client.executeWithRetry( + table, + request, + null /*tableName*/ + ); + } catch (Exception e) { + throw new IOException("Failed to execute request", e); + } + return response; + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index baf66f58..5059d9e8 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -319,53 +319,97 @@ public void testAdminEnDisableTable() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); - admin.disableTable(TableName.valueOf("test_multi_cf")); assertTrue(admin.tableExists(TableName.valueOf("n1", "test"))); assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); - // disable a non-existed table - IOException thrown = assertThrows(IOException.class, - () -> { - admin.disableTable(TableName.valueOf("tablegroup_not_exists")); - }); - assertTrue(thrown.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + // 1. disable a non-existed table + { + IOException thrown = assertThrows(IOException.class, + () -> { + admin.disableTable(TableName.valueOf("tablegroup_not_exists")); + }); + assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + } + // 2. write an enabled table, should succeed + { + if (admin.isTableDisabled(TableName.valueOf("test_multi_cf"))) { + admin.enableTable(TableName.valueOf("test_multi_cf")); + } + batchInsert(10, "test_multi_cf"); + batchGet(10, "test_multi_cf"); + } - // write an enabled table, should succeed - batchInsert(10, "test_multi_ch"); - // disable a disabled table - thrown = assertThrows(IOException.class, - () -> { - admin.disableTable(TableName.valueOf("test_multi_cf")); - }); - assertTrue(thrown.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + // 3. disable a enable table + { + if (admin.isTableEnabled(TableName.valueOf("test_multi_cf"))) { + admin.disableTable(TableName.valueOf("test_multi_cf")); + } + // write and read disable table, should fail + try { + batchInsert(10, "test_multi_cf"); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + System.out.println(ex.getCause().getMessage()); + } + try { + batchGet(10, "test_multi_cf"); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, + ((ObTableException) ex.getCause()).getErrorCode()); + } - // write an enabled table, should fail - batchInsert(10, "test_multi_ch"); - enDisableRead(10, "test_multi_ch"); + } - // enable a disabled table - admin.enableTable(TableName.valueOf("test_multi_cf")); + // 4. enable a disabled table + { + if (admin.isTableDisabled(TableName.valueOf("test_multi_cf"))) { + admin.enableTable(TableName.valueOf("test_multi_cf")); + } + // write an enabled table, should succeed + batchInsert(10, "test_multi_cf"); + batchGet(10, "test_multi_cf"); + } - // write an enabled table, should succeed - batchInsert(10, "test_multi_ch"); - enDisableRead(10, "test_multi_ch"); + // 5. enable an enabled table + { + if (admin.isTableDisabled(TableName.valueOf("n1", "test"))) { + admin.enableTable(TableName.valueOf("n1", "test")); + } + try { + admin.enableTable(TableName.valueOf("n1", "test")); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, + ((ObTableException) ex.getCause()).getErrorCode()); + } + } - // enable an enabled table - thrown = assertThrows(IOException.class, - () -> { - admin.disableTable(TableName.valueOf("n1", "test")); - }); - assertTrue(thrown.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + // 6. disable a disabled table + { + if (admin.isTableEnabled(TableName.valueOf("n1", "test"))) { + admin.disableTable(TableName.valueOf("n1", "test")); + } + try { + admin.disableTable(TableName.valueOf("n1", "test")); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, + ((ObTableException) ex.getCause()).getErrorCode()); + } + } - admin.deleteTable(TableName.valueOf("n1", "test")); admin.deleteTable(TableName.valueOf("test_multi_cf")); - assertFalse(admin.tableExists(TableName.valueOf("n1", "test"))); assertFalse(admin.tableExists(TableName.valueOf("test_multi_cf"))); + admin.deleteTable(TableName.valueOf("n1", "test")); + assertFalse(admin.tableExists(TableName.valueOf("n1", "test"))); } - private void enDisableRead(int rows, String tablegroup) throws Exception { + private void batchGet(int rows, String tablegroup) throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Table table = connection.getTable(TableName.valueOf(tablegroup)); From 8a9968b8b6332ae99d50aa0be105763d48e4dd0c Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 4 Jun 2025 17:10:22 +0800 Subject: [PATCH 13/36] add hregionlocation test case --- .../oceanbase/hbase/OHConnectionTest.java | 51 +++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 31a6d800..3e4046ce 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -20,10 +20,7 @@ import com.alipay.oceanbase.hbase.util.OHBufferedMutatorImpl; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; @@ -992,6 +989,52 @@ public void testRangePartitionWithRegionLocator() throws Exception { }); } } + + @Test + public void testHRegionLocation() throws IOException { + final String tableNameStr = "test_region_locator"; + + final byte[][] splitPoints = new byte[][] { + Bytes.toBytes("c"), // p1: < 'c' + Bytes.toBytes("e"), // p2: < 'e' + Bytes.toBytes("g"), // p3: < 'g' + Bytes.toBytes("i"), // p4: < 'i' + Bytes.toBytes("l"), // p5: < 'l' + Bytes.toBytes("n"), // p6: < 'n' + Bytes.toBytes("p"), // p7: < 'p' + Bytes.toBytes("s"), // p8: < 's' + Bytes.toBytes("v") // p9: < 'v' + }; + + final TableName tableName = TableName.valueOf(tableNameStr); + final Configuration conf = ObHTableTestUtil.newConfiguration(); + connection = ConnectionFactory.createConnection(conf); + hTable = connection.getTable(tableName); + // (min, c), [c, e), [e, g), [g, i), [i, l), [l, n), [n, p), [p, s), [s, v), [v, max) + try (RegionLocator locator = connection.getRegionLocator(tableName)) { + Assert.assertEquals(locator.getStartKeys().length, locator.getEndKeys().length); + Assert.assertEquals(locator.getStartKeys().length, 10); + HRegionLocation loc = locator.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY); + RegionInfo info = loc.getRegion(); + Assert.assertEquals(Arrays.toString(locator.getStartKeys()[0]), Arrays.toString(info.getStartKey())); + Assert.assertEquals(Arrays.toString(locator.getEndKeys()[0]), Arrays.toString(info.getEndKey())); + for (int i = 1; i < locator.getStartKeys().length; i++) { + loc = locator.getRegionLocation(splitPoints[i - 1]); + info = loc.getRegion(); + Assert.assertEquals(Arrays.toString(locator.getStartKeys()[i]), Arrays.toString(info.getStartKey())); + Assert.assertEquals(Arrays.toString(locator.getEndKeys()[i]), Arrays.toString(info.getEndKey())); + } + } finally { + Optional.ofNullable(hTable).ifPresent(table -> { + try { + table.close(); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + } + @Test public void testKeyPartitionWithRegionLocator() throws IOException { final String tableNameStr = "test_multi_cf"; From 129a0f4c3c8fbbad9b3f89d9eb98f06a3c5cdf6c Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 4 Jun 2025 17:18:10 +0800 Subject: [PATCH 14/36] disable htable ddl --- .../alipay/oceanbase/hbase/util/OHAdmin.java | 87 ++++++++++--------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index ba1e483f..7d1dfde5 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -170,19 +170,20 @@ public TableDescriptor getDescriptor(TableName tableName) throws TableNotFoundEx @Override public void createTable(TableDescriptor tableDescriptor) throws IOException { - OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableDescriptor.getTableName(), connectionConf); - OHCreateTableExecutor executor = new OHCreateTableExecutor(tableClient); - try { - executor.createTable(tableDescriptor, null); - } catch (IOException e) { - if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { - throw new TimeoutIOException(e.getCause()); - } else { - throw e; - } - } + throw new FeatureNotSupportedException("does not support yet"); +// OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); +// ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableDescriptor.getTableName(), connectionConf); +// OHCreateTableExecutor executor = new OHCreateTableExecutor(tableClient); +// try { +// executor.createTable(tableDescriptor, null); +// } catch (IOException e) { +// if (e.getCause() instanceof ObTableTransportException +// && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { +// throw new TimeoutIOException(e.getCause()); +// } else { +// throw e; +// } +// } } @Override @@ -202,19 +203,20 @@ public Future createTableAsync(TableDescriptor tableDescriptor, byte[][] b @Override public void deleteTable(TableName tableName) throws IOException { - OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHDeleteTableExecutor executor = new OHDeleteTableExecutor(tableClient); - try { - executor.deleteTable(tableName.getNameAsString()); - } catch (IOException e) { - if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { - throw new TimeoutIOException(e.getCause()); - } else { - throw e; - } - } + throw new FeatureNotSupportedException("does not support yet"); +// OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); +// ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); +// OHDeleteTableExecutor executor = new OHDeleteTableExecutor(tableClient); +// try { +// executor.deleteTable(tableName.getNameAsString()); +// } catch (IOException e) { +// if (e.getCause() instanceof ObTableTransportException +// && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { +// throw new TimeoutIOException(e.getCause()); +// } else { +// throw e; +// } +// } } @Override @@ -244,19 +246,20 @@ public Future truncateTableAsync(TableName tableName, boolean b) throws IO @Override public void enableTable(TableName tableName) throws IOException { - OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_ENABLE_TABLE); - try { - executor.enableTable(tableName.getNameAsString()); - } catch (IOException e) { - if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { - throw new TimeoutIOException(e.getCause()); - } else { - throw e; - } - } + throw new FeatureNotSupportedException("does not support yet"); +// OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); +// ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); +// OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_ENABLE_TABLE); +// try { +// executor.enableTable(tableName.getNameAsString()); +// } catch (IOException e) { +// if (e.getCause() instanceof ObTableTransportException +// && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { +// throw new TimeoutIOException(e.getCause()); +// } else { +// throw e; +// } +// } } @Override @@ -308,12 +311,14 @@ public HTableDescriptor[] disableTables(Pattern pattern) throws IOException { @Override public boolean isTableEnabled(TableName tableName) throws IOException { - return isDisabled(tableName) == false; + throw new FeatureNotSupportedException("does not support yet"); +// return isDisabled(tableName) == false; } @Override public boolean isTableDisabled(TableName tableName) throws IOException { - return isDisabled(tableName) == true; + throw new FeatureNotSupportedException("does not support yet"); +// return isDisabled(tableName) == true; } private boolean isDisabled(TableName tableName) throws IOException { From 0319f357f858b320632ef6d4244177e8cda8817a Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 4 Jun 2025 17:48:27 +0800 Subject: [PATCH 15/36] optimize region metrics case --- .../hbase/OHTableAdminInterfaceTest.java | 190 ++++++++++-------- 1 file changed, 103 insertions(+), 87 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 5059d9e8..748d21c8 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -22,6 +22,7 @@ import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import com.alipay.oceanbase.hbase.util.ResultSetPrinter; import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.exception.ObTableGetException; import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; @@ -432,31 +433,49 @@ public void testAdminGetRegionMetrics() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + - "\n" + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + + "CREATE TABLEGROUP IF NOT EXISTS test_no_part SHARDING = 'ADAPTIVE';\n"+ + "CREATE TABLE IF NOT EXISTS `test_no_part$family_with_group1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_no_part;\n" + + "CREATE TABLE IF NOT EXISTS `test_no_part$family_with_group2` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_no_part;\n" + + "CREATE TABLE IF NOT EXISTS `test_no_part$family_with_group3` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = test_no_part;\n" + "CREATE DATABASE IF NOT EXISTS `n1`;\n" + "use `n1`;\n" + "CREATE TABLEGROUP IF NOT EXISTS `n1:test_multi_cf` SHARDING = 'ADAPTIVE';\n" + @@ -488,25 +507,30 @@ public void testAdminGetRegionMetrics() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); + // test tablegroup not existed IOException thrown = assertThrows(IOException.class, () -> { admin.getRegionMetrics(null, TableName.valueOf("tablegroup_not_exists")); }); Assert.assertTrue(thrown.getCause() instanceof ObTableException); Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + + // test use serverName without tableName to get region metrics assertThrows(FeatureNotSupportedException.class, () -> { admin.getRegionMetrics(ServerName.valueOf("localhost,1,1")); }); - // insert 300 thousand of rows in each table under tablegroup test_multi_cf + + // test single-thread getRegionMetrics after writing batchInsert(100000, tablegroup1); - List metrics = admin.getRegionMetrics(null, TableName.valueOf(tablegroup1)); - for (RegionMetrics regionMetrics : metrics) { - System.out.println("region name: " + regionMetrics.getNameAsString() - + ", storeFileSize: " + regionMetrics.getStoreFileSize() - + ", memFileSize: " + regionMetrics.getMemStoreSize()); - } - // concurrently read while writing 150 thousand of rows to 2 tablegroups + // test ServerName is any string + long start = System.currentTimeMillis(); + List metrics = admin.getRegionMetrics(ServerName.valueOf("localhost,1,1"), TableName.valueOf(tablegroup1)); + long cost = System.currentTimeMillis() - start; + System.out.println("get region metrics time usage: " + cost + "ms, tablegroup: " + tablegroup1); + assertEquals(30, metrics.size()); + + // test getRegionMetrics concurrently reading while writing ExecutorService executorService = Executors.newFixedThreadPool(10); CountDownLatch latch = new CountDownLatch(100); List exceptionCatcher = new ArrayList<>(); @@ -518,21 +542,42 @@ public void testAdminGetRegionMetrics() throws Exception { List regionMetrics = null; // test get regionMetrics from different namespaces if (taskId % 3 != 0) { + long thrStart = System.currentTimeMillis(); regionMetrics = admin.getRegionMetrics(null, TableName.valueOf(tablegroup1)); + long thrCost = System.currentTimeMillis() - thrStart; + System.out.println("task: " + taskId + ", get region metrics time usage: " + thrCost + "ms, tablegroup: " + tablegroup1); + if (regionMetrics.size() != 30) { + throw new ObTableGetException( + "the number of region metrics does not match the number of tablets, the number of region metrics: " + regionMetrics.size()); + } } else { + long thrStart = System.currentTimeMillis(); regionMetrics = admin.getRegionMetrics(null, TableName.valueOf(tablegroup2)); - } - for (RegionMetrics m : regionMetrics) { - System.out.println("task: " + taskId + ", tablegroup: " + ((OHRegionMetrics) m).getTablegroup() - + ", region name: " + m.getNameAsString() - + ", storeFileSize: " + m.getStoreFileSize() - + ", memFileSize: " + m.getMemStoreSize()); + long thrCost = System.currentTimeMillis() - thrStart; + System.out.println("task: " + taskId + ", get region metrics time usage: " + thrCost + "ms, tablegroup: " + tablegroup1); + if (regionMetrics.size() != 9) { + throw new ObTableGetException( + "the number of region metrics does not match the number of tablets, the number of region metrics: " + regionMetrics.size()); + } } } else { - if (taskId % 8 == 0) { - batchInsert(1000, tablegroup2); - } else { - batchInsert(1000, tablegroup1); + try { + if (taskId % 8 == 0) { + batchInsert(1000, tablegroup2); + } else { + batchInsert(1000, tablegroup1); + } + } catch (Exception e) { + Exception originalCause = e; + while (originalCause.getCause() != null && originalCause.getCause() instanceof ObTableException) { + originalCause = (Exception) originalCause.getCause(); + } + if (originalCause instanceof ObTableException && ((ObTableException) originalCause).getErrorCode() == ResultCodes.OB_TIMEOUT.errorCode) { + // ignore + System.out.println("taskId: " + taskId + " OB_TIMEOUT"); + } else { + throw e; + } } System.out.println("task: " + taskId + ", batchInsert"); } @@ -552,6 +597,15 @@ public void testAdminGetRegionMetrics() throws Exception { } executorService.shutdownNow(); Assert.assertTrue(exceptionCatcher.isEmpty()); + + // test getRegionMetrics from non-partitioned table + String non_part_tablegroup = "test_no_part"; + batchInsert(10000, non_part_tablegroup); + start = System.currentTimeMillis(); + metrics = admin.getRegionMetrics(null, TableName.valueOf(non_part_tablegroup)); + cost = System.currentTimeMillis() - start; + System.out.println("get region metrics time usage: " + cost + "ms, tablegroup: " + non_part_tablegroup); + assertEquals(3, metrics.size()); } @Test @@ -559,55 +613,50 @@ public void testAdminDeleteTable() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + - "\n" + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + "CREATE DATABASE IF NOT EXISTS `n1`;\n" + "use `n1`;\n" + - "CREATE TABLEGROUP IF NOT EXISTS `n1:test` SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `n1:test$family_group` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test`;" + - "\n" + - "CREATE TABLE IF NOT EXISTS `n1:test$family1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test`;"); + "CREATE TABLEGROUP IF NOT EXISTS `n1:test_multi_cf` SHARDING = 'ADAPTIVE';\n" + + "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group1` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group2` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;"); st.close(); conn.close(); Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); - assertTrue(admin.tableExists(TableName.valueOf("n1", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("n1", "test_multi_cf"))); assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); IOException thrown = assertThrows(IOException.class, () -> { @@ -615,9 +664,9 @@ public void testAdminDeleteTable() throws Exception { }); Assert.assertTrue(thrown.getCause() instanceof ObTableException); Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); - admin.deleteTable(TableName.valueOf("n1", "test")); + admin.deleteTable(TableName.valueOf("n1", "test_multi_cf")); admin.deleteTable(TableName.valueOf("test_multi_cf")); - assertFalse(admin.tableExists(TableName.valueOf("n1", "test"))); + assertFalse(admin.tableExists(TableName.valueOf("n1", "test_multi_cf"))); assertFalse(admin.tableExists(TableName.valueOf("test_multi_cf"))); } @@ -626,49 +675,16 @@ public void testAdminTableExists() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + - "\n" + "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + + ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + "CREATE DATABASE IF NOT EXISTS `n1`;\n" + "use `n1`;\n" + - "CREATE TABLEGROUP IF NOT EXISTS `n1:test` SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `n1:test$family_group` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test`;" + - "\n" + - "CREATE TABLE IF NOT EXISTS `n1:test$family1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test`;"); + "CREATE TABLEGROUP IF NOT EXISTS `n1:test_multi_cf` SHARDING = 'ADAPTIVE';"); st.close(); conn.close(); Configuration conf = ObHTableTestUtil.newConfiguration(); @@ -680,8 +696,8 @@ public void testAdminTableExists() throws Exception { TableName.valueOf("random_string$"); }); Assert.assertFalse(admin.tableExists(TableName.valueOf("tablegroup_not_exists"))); + Assert.assertTrue(admin.tableExists(TableName.valueOf("n1", "test_multi_cf"))); Assert.assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); - Assert.assertTrue(admin.tableExists(TableName.valueOf("n1", "test"))); } private void batchInsert(int rows, String tablegroup) throws Exception { From cc25a35de97d8191873b7bfb8b006dd184cb84fa Mon Sep 17 00:00:00 2001 From: GroundWu <1175416256@qq.com> Date: Wed, 4 Jun 2025 17:56:44 +0800 Subject: [PATCH 16/36] add hbase put case for perf opt (#231) --- .../OHTableSecondaryPartPutTest.java | 256 ++++++++++++++++++ 1 file changed, 256 insertions(+) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java index fef4dce0..32449005 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.junit.*; @@ -330,4 +331,259 @@ public void testMultiCFPut() throws Throwable { public void testMultiCFPutBatch() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMltiCFPutBatchImpl); } + + @Test + public void testPutOpt() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartPutTest::testPutOptImpl); + FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMultiCFPutOptImpl); + } + + public static void testPutOptImpl(String tableName) throws Exception { + int NUM_QUALIFIERS = 10; + int NUM_PUTS = 10; + byte[] family = toBytes(getColumnFamilyName(tableName)); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + { // 单put，10个Qualifier + Put put = new Put(toBytes("row1")); + List qualifiers = new ArrayList<>(); + for (int i = 1; i <= NUM_QUALIFIERS; i++) { + byte[] qualifier = toBytes("q" + i); + byte[] value = toBytes("value" + i); + put.addColumn(family, qualifier, value); + qualifiers.add(Bytes.toString(qualifier)); + } + hTable.put(put); + // verify + Get get = new Get(Bytes.toBytes("row1")); + Result result = hTable.get(get); + Assert.assertEquals(NUM_QUALIFIERS, result.getFamilyMap(family).size()); + qualifiers.forEach(q -> + Assert.assertNotNull("Qualifier " + q + " not found", + result.getValue(family, Bytes.toBytes(q))) + ); + } + + { // 单put，10个Qualifier，指定相同timestamp + Put put = new Put(toBytes("row2")); + List qualifiers = new ArrayList<>(); + long timestamp = System.currentTimeMillis(); + for (int i = 1; i <= NUM_QUALIFIERS; i++) { + byte[] qualifier = toBytes("q" + i); + byte[] value = toBytes("value" + i); + put.addColumn(family, qualifier, timestamp, value); + qualifiers.add(Bytes.toString(qualifier)); + } + hTable.put(put); + // verify + Get get = new Get(Bytes.toBytes("row2")); + Result result = hTable.get(get); + Assert.assertEquals(NUM_QUALIFIERS, result.getFamilyMap(family).size()); + qualifiers.forEach(q -> + Assert.assertNotNull("Qualifier " + q + " not found", + result.getValue(family, Bytes.toBytes(q))) + ); + } + { // batch put，相同key， 多个Qualifier + byte[] rowKey = Bytes.toBytes("batch_row"); + List puts = new ArrayList<>(); + List qualifiers = new ArrayList<>(); + for (int i = 1; i <= NUM_PUTS; i++) { + Put put = new Put(rowKey); + byte[] qualifier = Bytes.toBytes("batch_q" + i); + put.addColumn(family, qualifier, Bytes.toBytes("batch_val" + i)); + puts.add(put); + qualifiers.add(Bytes.toString(qualifier)); + } + hTable.put(puts); + // verify + Get get = new Get(rowKey); + Result result = hTable.get(get); + Assert.assertEquals(NUM_QUALIFIERS, result.getFamilyMap(family).size()); + qualifiers.forEach(q -> + Assert.assertNotNull("Qualifier " + q + " not found", + result.getValue(family, Bytes.toBytes(q))) + ); + } + + { // batch put，相同key，多个Qualifier，指定相同timestamp + byte[] rowKey = Bytes.toBytes("batch_row_ts"); + List puts = new ArrayList<>(); + List qualifiers = new ArrayList<>(); + for (int i = 1; i <= NUM_PUTS; i++) { + Put put = new Put(rowKey); + byte[] qualifier = Bytes.toBytes("batch_ts_q" + i); + put.addColumn(family, qualifier, Bytes.toBytes("batch_val" + i)); + puts.add(put); + qualifiers.add(Bytes.toString(qualifier)); + } + hTable.put(puts); + // verify + Get get = new Get(rowKey); + Result result = hTable.get(get); + Assert.assertEquals(NUM_QUALIFIERS, result.getFamilyMap(family).size()); + qualifiers.forEach(q -> + Assert.assertNotNull("Qualifier " + q + " not found", + result.getValue(family, Bytes.toBytes(q))) + ); + } + } + + public static void testMultiCFPutOptImpl(Map.Entry> entry) throws Exception { + int NUM_QUALIFIERS = 10; + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); + hTable.init(); + { + Put put = new Put(toBytes("multi_cf_row")); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + // 单put，10个Qualifier + List qualifiers = new ArrayList<>(); + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] qualifier = toBytes("q" + i); + byte[] value = toBytes("value_" + i); + put.addColumn(family, qualifier, value); + qualifiers.add(Bytes.toString(qualifier)); + } + } + hTable.put(put); + // verify + Get get = new Get(Bytes.toBytes("multi_cf_row")); + Result result = hTable.get(get); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Assert.assertEquals(NUM_QUALIFIERS, + result.getFamilyMap(family).size()); + + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] val = result.getValue(family, Bytes.toBytes("q" + i)); + Assert.assertEquals( "value_" + i, + Bytes.toString(val)); + } + } + } + + { // 指定timestamp + long currentTimestamp = System.currentTimeMillis(); + Put put = new Put(toBytes("multi_cf_ts_row")); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + // 单put，10个Qualifier + List qualifiers = new ArrayList<>(); + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] qualifier = toBytes("q" + i); + byte[] value = toBytes("value_" + i); + put.addColumn(family, qualifier, currentTimestamp, value); + qualifiers.add(Bytes.toString(qualifier)); + } + } + hTable.put(put); + // verify + Get get = new Get(Bytes.toBytes("multi_cf_ts_row")); + Result result = hTable.get(get); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Assert.assertEquals(NUM_QUALIFIERS, + result.getFamilyMap(family).size()); + + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] val = result.getValue(family, Bytes.toBytes("q" + i)); + Assert.assertEquals( "value_" + i, + Bytes.toString(val)); + } + } + } + + { + byte[] ROW = Bytes.toBytes("batch_row"); + List puts = new ArrayList<>(); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Put put = new Put(ROW); + for (int i = 0; i < NUM_QUALIFIERS; i++) { + put.addColumn(family, + Bytes.toBytes("q" + i), + Bytes.toBytes("value_" + i)); + } + puts.add(put); + } + hTable.put(puts); + // verify + Get get = new Get(ROW); + Result result = hTable.get(get); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Assert.assertEquals(NUM_QUALIFIERS, + result.getFamilyMap(family).size()); + + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] val = result.getValue(family, Bytes.toBytes("q" + i)); + Assert.assertEquals( "value_" + i, + Bytes.toString(val)); + } + } + } + + { + long timestamp = System.currentTimeMillis(); + byte[] ROW = Bytes.toBytes("batch_ts_row"); + List puts = new ArrayList<>(); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + for (int i = 0; i < NUM_QUALIFIERS; i++) { + Put put = new Put(ROW); + put.addColumn(family, + Bytes.toBytes("q" + i), + timestamp, + Bytes.toBytes("value_" + i)); + puts.add(put); + } + } + hTable.put(puts); + // verify + Get get = new Get(ROW); + Result result = hTable.get(get); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Assert.assertEquals(NUM_QUALIFIERS, + result.getFamilyMap(family).size()); + + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] val = result.getValue(family, Bytes.toBytes("q" + i)); + Assert.assertEquals( "value_" + i, + Bytes.toString(val)); + } + } + } + + { + byte[] ROW = Bytes.toBytes("batch_row_2"); + List puts = new ArrayList<>(); + for (int i = 0; i < NUM_QUALIFIERS; i++) { + Put put = new Put(ROW); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + put.addColumn(family, + Bytes.toBytes("q" + i), + Bytes.toBytes("value_" + i)); + } + puts.add(put); + } + hTable.put(puts); + // verify + Get get = new Get(ROW); + Result result = hTable.get(get); + for (String tableName : entry.getValue()) { + byte[] family = toBytes(getColumnFamilyName(tableName)); + Assert.assertEquals(NUM_QUALIFIERS, + result.getFamilyMap(family).size()); + + for (int i = 0; i < NUM_QUALIFIERS; i++) { + byte[] val = result.getValue(family, Bytes.toBytes("q" + i)); + Assert.assertEquals( "value_" + i, + Bytes.toString(val)); + } + } + } + } } From 5121a8dbcf09aaf524d99c2fcc361d21537a0847 Mon Sep 17 00:00:00 2001 From: WeiXinChan Date: Sun, 27 Apr 2025 17:59:53 +0800 Subject: [PATCH 17/36] adapt read hot only --- .../com/alipay/oceanbase/hbase/OHTable.java | 4 + .../hbase/constants/OHConstants.java | 25 +- .../OHTableShareStorageSeriesTest.java | 187 ++++++ .../secondary/OHTableShareStorageTest.java | 558 ++++++++++++++++++ .../hbase/util/ObHTableSecondaryPartUtil.java | 63 ++ .../hbase/util/ObHTableTestUtil.java | 2 + .../hbase/util/TableTemplateManager.java | 70 +++ 7 files changed, 899 insertions(+), 10 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 83c2a9ab..106058eb 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1948,6 +1948,8 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE)); obTableQuery.setObKVParams(buildOBKVParams(scan)); obTableQuery.setScanRangeColumns("K", "Q", "T"); + byte[] hotOnly = scan.getAttribute(HBASE_HTABLE_QUERY_HOT_ONLY); + obTableQuery.setHotOnly(hotOnly != null && Arrays.equals(hotOnly, "true".getBytes())); return obTableQuery; } @@ -1965,6 +1967,8 @@ private ObTableQuery buildObTableQuery(final Get get, Collection columnQ } obTableQuery.setObKVParams(buildOBKVParams(get)); obTableQuery.setScanRangeColumns("K", "Q", "T"); + byte[] hotOnly = get.getAttribute(HBASE_HTABLE_QUERY_HOT_ONLY); + obTableQuery.setHotOnly(hotOnly != null && Arrays.equals(hotOnly, "true".getBytes())); return obTableQuery; } diff --git a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java index 19311744..deb94f75 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java +++ b/src/main/java/com/alipay/oceanbase/hbase/constants/OHConstants.java @@ -23,17 +23,17 @@ public final class OHConstants { /** - * ocenbase hbase root server http url + * oceanbase hbase root server http url */ public static final String HBASE_OCEANBASE_PARAM_URL = "hbase.oceanbase.paramURL"; /** - * ocenbase hbase connect server username + * oceanbase hbase connect server username */ public static final String HBASE_OCEANBASE_FULL_USER_NAME = "hbase.oceanbase.fullUserName"; /** - * ocenbase hbase connect server password + * oceanbase hbase connect server password */ public static final String HBASE_OCEANBASE_PASSWORD = "hbase.oceanbase.password"; @@ -48,39 +48,39 @@ public final class OHConstants { public static final String HBASE_OCEANBASE_SYS_PASSWORD = "hbase.oceanbase.sysPassword"; /** - * ocenbase hbase connect server password + * oceanbase hbase connect server password */ public static final String HBASE_OCEANBASE_BATCH_EXECUTOR = "hbase.oceanbase.batch.executor"; /** - * ocenbase hbase connect server ODP address + * oceanbase hbase connect server ODP address */ public static final String HBASE_OCEANBASE_ODP_ADDR = "hbase.oceanbase.odpAddr"; /** - * ocenbase hbase connect server ODP port + * oceanbase hbase connect server ODP port */ public static final String HBASE_OCEANBASE_ODP_PORT = "hbase.oceanbase.odpPort"; /** - * ocenbase hbase connect server ODP mode + * oceanbase hbase connect server ODP mode */ public static final String HBASE_OCEANBASE_ODP_MODE = "hbase.oceanbase.odpMode"; /** - * ocenbase hbase connect server database + * oceanbase hbase connect server database */ public static final String HBASE_OCEANBASE_DATABASE = "hbase.oceanbase.database"; /** - * ocenbase hbase model rowkey column is consist of following column + * oceanbase hbase model rowkey column is consist of following column * K, Q, T hbase value */ public static final String[] ROW_KEY_COLUMNS = new String[] { "K", "Q", "T" }; /** - * ocenbase hbase model value column is consist of following column + * oceanbase hbase model value column is consist of following column * V hbase value */ public static final String[] V_COLUMNS = new String[] { "V" }; @@ -124,6 +124,11 @@ public final class OHConstants { */ public static final String DEFAULT_HBASE_HTABLE_TEST_LOAD_SUFFIX = "_t"; + /** + * use to specify whether to query only the data in hot storage when performing a query. + */ + public static final String HBASE_HTABLE_QUERY_HOT_ONLY = "hbase.htable.query.hot_only"; + /*-------------------------------------------------------------------------------------------------------------*/ /** diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java new file mode 100644 index 00000000..53fb2746 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java @@ -0,0 +1,187 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_QUERY_HOT_ONLY; +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.*; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + + +public class OHTableShareStorageSeriesTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableType type : NORMAL_SERIES_PARTITIONED_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + for (TableType type : NORMAL_SERIES_PARTITIONED_TABLES) { + alterTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. get, expect less than 100 key + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + } + Assert.assertTrue(getCount < recordCount); + + hTable.close(); + } + + public static void testScanImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. scan not specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 2. scan specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 3. scan specify versions + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setMaxVersions(2); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 4. scan specify time range + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setMaxVersions(2); + scan.setTimeStamp(curTs); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + hTable.close(); + } + + @Test + public void testGet() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageSeriesTest::testGetImpl); + } + + + @Test + public void testScan() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageSeriesTest::testScanImpl); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java new file mode 100644 index 00000000..684f5c53 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java @@ -0,0 +1,558 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.*; +import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_OCEANBASE_SYS_PASSWORD; +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.ODP_MODE; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.*; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableShareStorageTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : NORMAL_PARTITIONED_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + for (TableTemplateManager.TableType type : NORMAL_PARTITIONED_TABLES) { + alterTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. get, expect less than 100 key + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + } + System.out.println("getCount:" + getCount); + Assert.assertTrue(getCount < recordCount); + + hTable.close(); + } + + public static void testMultiCFGetImpl(Map.Entry> entry) throws Exception { + // 0. prepare data + long recordCount = 100; + String groupName = getTableName(entry.getKey()); + List tableNames = entry.getValue(); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + } + + // 1. get specify column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + } + Assert.assertTrue(getCount < recordCount); + } + } + + // 2. get do not specify column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + Get get = new Get((key + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addFamily(family.getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + } + Assert.assertTrue(getCount < recordCount); + } + } + + // 3. get do not specify column family + { + long getCount = 0; + Get get = new Get(key.getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + Assert.assertTrue(getCount < recordCount); + } + + // 4. get specify multi cf and column + { + long getCount = 0; + Get get = new Get(key.getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addColumn(family.getBytes(), column.getBytes()); + } + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + getCount += cells.length; + System.out.println("getCount:" + getCount); + Assert.assertTrue(getCount < recordCount); + } + + hTable.close(); + } + + public static void testBatchGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value)); + hTable.put(put); + } + + List gets = new ArrayList<>(); + // 1. get, expect less than 100 key + long getCount = 0; + for (int i = 0; i < recordCount; i++) { + String getKey = null; + if (i == 4) { + getKey = key + 101; + } else { + getKey = key + i; + } + Get get = new Get((getKey + i).getBytes()); + get.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + get.addColumn(family.getBytes(), (column + i).getBytes()); + gets.add(get); + } + Result[] results = hTable.get(gets); + for (Result result : results) { + getCount += result.size(); + } + Assert.assertTrue(getCount < recordCount); + + hTable.close(); + } + + public static void testScanImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String family = getColumnFamilyName(tableName); + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column).getBytes(), curTs, toBytes(value + i)); + hTable.put(put); + } + + // 1. scan not specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 2. scan specify column + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 3. scan specify versions + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + + // 4. scan specify filter + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + scan.addFamily(family.getBytes()); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(value))); + scan.setFilter(valueFilter); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 5. scan in reverse + { + Scan scan = new Scan(key.getBytes(), (key+recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addFamily(family.getBytes()); + scan.setReversed(true); + try { + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getMessage().contains("secondary partitioned hbase table with reverse query not supported")); + } + } + hTable.close(); + } + + public static void testMultiCFScanImpl(Map.Entry> entry) throws Exception { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + + // 0. prepare data - 100 key + long recordCount = 100; + String key = "Key"; + String column = "Column"; + String value = "Value"; + long curTs = System.currentTimeMillis(); + List tableNames = entry.getValue(); + + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < recordCount; i++) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), (column).getBytes(), curTs, toBytes(value)); + hTable.put(put); + } + } + + // 1. multi cf scan specify one cf and one column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.addColumn(family.getBytes(), column.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + } + + // 2. multi cf scan specify one cf without specify column + { + for (String tableName : tableNames) { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + String family = getColumnFamilyName(tableName); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + } + + // 3. multi cf scan do not specify cf + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 4. multi cf scan specify multi cf and multi column + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + scan.addColumn(family.getBytes(), column.getBytes()); + } + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 5. multi cf scan specify versions + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 6. multi cf scan specify time range + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + scan.setTimeStamp(curTs); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 7. multi cf scan specify filter + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(value))); + scan.setFilter(valueFilter); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 8. multi cf scan using setStartRow/setEndRow + { + Scan scan = new Scan(); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setStartRow(key.getBytes()); + scan.setStopRow((key + recordCount).getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 9. multi cf scan using batch + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + scan.setBatch(2); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + + // 10. multi cf scan with family scan and column-specific scan + { + Scan scan = new Scan(key.getBytes(), (key + recordCount).getBytes()); + scan.setAttribute(HBASE_HTABLE_QUERY_HOT_ONLY, "true".getBytes()); + for (int i = 0; i < tableNames.size(); i++) { + String family = getColumnFamilyName(tableNames.get(i)); + if (i % 2 == 0) { + scan.addFamily(family.getBytes()); + } else { + scan.addColumn(family.getBytes(), column.getBytes()); + } + } + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + count++; + } + } + Assert.assertTrue(count < recordCount); + } + hTable.close(); + } + + @Test + public void testGet() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageTest::testGetImpl); + } + + @Test + public void testMultiCFGet() throws Throwable { + FOR_EACH(group2tableNames, OHTableShareStorageTest::testMultiCFGetImpl); + } + + @Test + public void testBatchGet() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageTest::testBatchGetImpl); + } + + @Test + public void testScan() throws Throwable { + FOR_EACH(tableNames, OHTableShareStorageTest::testScanImpl); + } + + @Test + public void testMultiCFScan() throws Throwable { + FOR_EACH(group2tableNames, OHTableShareStorageTest::testMultiCFScanImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index 693fa35a..2ee37e4e 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -57,6 +57,20 @@ public static void createTables(TableTemplateManager.TableType type, List tableNames, + Map> group2tableNames, boolean printSql) + throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // single cf table + if (tableNames != null) { + alterTables(conn, type, tableNames, printSql); + } + // multi cf table + if (group2tableNames != null) { + alterTables(conn, type, group2tableNames, printSql); + } + } + public static void createTables(Connection conn, TableTemplateManager.TableType type, List tableNames, boolean printSql) throws Exception { // create single cf table @@ -85,6 +99,31 @@ public static void createTables(Connection conn, TableTemplateManager.TableType } } + public static void alterTables(Connection conn, TableTemplateManager.TableType type, + List tableNames, boolean printSql) throws Exception { + // create single cf table + if (tableNames != null) { + String tableGroup = TableTemplateManager.getTableGroupName(type, false); + String tableName = TableTemplateManager.generateTableName(tableGroup, false, 1); + String sql = TableTemplateManager.getAlterTableSQL(type, tableName); + try { + System.out.println(sql); + conn.createStatement().execute(sql); + System.out.println("============= alter table: " + tableName + " table_group: " + + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table: " + tableName + " table_group: " + + getTableName(tableName) + " alter failed ============="); + } + } + } + } + public static void createTables(Connection conn, TableTemplateManager.TableType type, Map> group2tableNames, boolean printSql) throws Exception { if (group2tableNames != null) { TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); @@ -121,6 +160,30 @@ public static void createTables(Connection conn, TableTemplateManager.TableType } } + public static void alterTables(Connection conn, TableTemplateManager.TableType type, Map> group2tableNames, boolean printSql) throws Exception { + if (group2tableNames != null) { + String tableGroup = TableTemplateManager.getTableGroupName(type, true); + group2tableNames.put(tableGroup, new LinkedList<>()); + for (int i = 1; i <= 3; ++i) { + String tableName = TableTemplateManager.generateTableName(tableGroup, true, i); + String sql = TableTemplateManager.getAlterTableSQL(type, tableName); + try { + conn.createStatement().execute(sql); + System.out.println("============= alter table: " + tableName + + " table_group: " + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table: " + tableName + " table_group: " + getTableName(tableName) + " alter failed ============="); + } + } + } + } + } + public static void truncateTables(List tableNames, Map> group2tableNames) throws Exception { Connection conn = ObHTableTestUtil.getConnection(); diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index 81fd3688..b2c94e6c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -103,6 +103,8 @@ public static Configuration newConfiguration() { Configuration conf = HBaseConfiguration.create(); conf.set(HBASE_OCEANBASE_FULL_USER_NAME, FULL_USER_NAME); conf.set(HBASE_OCEANBASE_PASSWORD, PASSWORD); + conf.set("rpc.execute.timeout", "20000"); + conf.set("rpc.operation.timeout", "18000"); if (ODP_MODE) { // ODP mode conf.set(HBASE_OCEANBASE_ODP_ADDR, ODP_ADDR); diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index 75a3e32d..2263bfcc 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -59,6 +59,19 @@ public enum TableType { SECONDARY_PARTITIONED_TIME_RANGE_KEY, SECONDARY_PARTITIONED_TIME_KEY_RANGE); + public static List NORMAL_PARTITIONED_TABLES = Arrays + .asList( + SINGLE_PARTITIONED_REGULAR, + SECONDARY_PARTITIONED_RANGE_KEY, + SECONDARY_PARTITIONED_RANGE_KEY_GEN, + SECONDARY_PARTITIONED_KEY_RANGE, + SECONDARY_PARTITIONED_KEY_RANGE_GEN); + public static List NORMAL_SERIES_PARTITIONED_TABLES = Arrays + .asList( + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + public static List SERIES_TABLES = Arrays .asList( NON_PARTITIONED_TIME_SERIES, @@ -284,6 +297,57 @@ public enum TableType { + ") TABLEGROUP = %s"); } + private static final Map ALTER_SQL_TEMPLATES = new EnumMap( + TableType.class); + + static { + // 普通表一级分区模板 + ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR, + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + // 时序表一级分区模板 + ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_TIME_SERIES, + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + // 普通表RANGE-KEY分区（使用K） + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 合并GEN类型的注释处理 + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 普通表KEY-RANGE分区（使用K） + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 普通表KEY-RANGE分区（使用生成列） + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 时序表RANGE-KEY分区 + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + // 时序表KEY-RANGE分区 + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_KEY_RANGE, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + /* ------------------ CELL TTL ----------------*/ + ALTER_SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR_CELL_TTL, + "ALTER TABLE `%s` ALTER PARTITION p0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + + ALTER_SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, + "ALTER TABLE `%s` ALTER SUBPARTITION p1sp0 STORAGE_CACHE_POLICY='hot';"); + } + public static String getCreateTableSQL(TableType type, String tableName, TimeGenerator.TimeRange timeRange) { System.out.println(tableName); @@ -329,6 +393,12 @@ public static String getCreateTableSQL(TableType type, String tableName, return String.format(template, params); } + public static String getAlterTableSQL(TableType type, String tableName) { + System.out.println(tableName); + String template = ALTER_SQL_TEMPLATES.get(type); + return String.format(template, tableName); + } + private static String getGeneratedColumn(TableType type) { StringBuilder sb = new StringBuilder(); boolean needsKPrefix = type.name().startsWith("SECONDARY_PARTITIONED") From a0f324ab77d4bae3f180a482740fa4cc4fe396c5 Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Wed, 4 Jun 2025 20:50:36 +0800 Subject: [PATCH 18/36] [Test] fix test compile error --- .../com/oceanbase/example/SimpleHBaseClientDemo.java | 6 ++++-- .../hbase/secondary/OHTableShareStorageSeriesTest.java | 4 ++-- .../hbase/secondary/OHTableShareStorageTest.java | 10 +++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/example/simple-hbase-demo/src/main/java/com/oceanbase/example/SimpleHBaseClientDemo.java b/example/simple-hbase-demo/src/main/java/com/oceanbase/example/SimpleHBaseClientDemo.java index 0ab3b588..e0ff821c 100644 --- a/example/simple-hbase-demo/src/main/java/com/oceanbase/example/SimpleHBaseClientDemo.java +++ b/example/simple-hbase-demo/src/main/java/com/oceanbase/example/SimpleHBaseClientDemo.java @@ -19,6 +19,8 @@ import com.alipay.oceanbase.hbase.OHTableClient; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; @@ -49,7 +51,7 @@ public static void simpleTest() throws Exception { byte[] rowKey = toBytes("rowKey1"); byte[] column = toBytes("column1"); Put put = new Put(rowKey); - put.add(family, column, System.currentTimeMillis(), toBytes("value1")); + put.addColumn(family, column, System.currentTimeMillis(), toBytes("value1")); hTable.put(put); // 3. get data like hbase @@ -58,7 +60,7 @@ public static void simpleTest() throws Exception { Result r = hTable.get(get); if (!r.isEmpty()) { Cell cell = r.rawCells()[0]; - System.out.printf("column1: " + CellUtil.cloneQualifier(r)); + System.out.printf("column1: " + CellUtil.cloneQualifier(cell)); } // 4. close diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java index 53fb2746..2af2bd53 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageSeriesTest.java @@ -75,7 +75,7 @@ public static void testGetImpl(String tableName) throws Exception { long curTs = System.currentTimeMillis(); for (int i = 0; i < recordCount; i++) { Put put = new Put(toBytes(key + i)); - put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + put.addColumn(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); hTable.put(put); } @@ -107,7 +107,7 @@ public static void testScanImpl(String tableName) throws Exception { long curTs = System.currentTimeMillis(); for (int i = 0; i < recordCount; i++) { Put put = new Put(toBytes(key + i)); - put.add(family.getBytes(), (column).getBytes(), curTs, toBytes(value + i)); + put.addColumn(family.getBytes(), (column).getBytes(), curTs, toBytes(value + i)); hTable.put(put); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java index 684f5c53..d15aaf5d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableShareStorageTest.java @@ -83,7 +83,7 @@ public static void testGetImpl(String tableName) throws Exception { long curTs = System.currentTimeMillis(); for (int i = 0; i < recordCount; i++) { Put put = new Put(toBytes(key + i)); - put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + put.addColumn(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); hTable.put(put); } @@ -119,7 +119,7 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr String family = getColumnFamilyName(tableName); for (int i = 0; i < recordCount; i++) { Put put = new Put(toBytes(key + i)); - put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); + put.addColumn(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value + i)); hTable.put(put); } } @@ -204,7 +204,7 @@ public static void testBatchGetImpl(String tableName) throws Exception { long curTs = System.currentTimeMillis(); for (int i = 0; i < recordCount; i++) { Put put = new Put(toBytes(key + i)); - put.add(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value)); + put.addColumn(family.getBytes(), (column + i).getBytes(), curTs, toBytes(value)); hTable.put(put); } @@ -245,7 +245,7 @@ public static void testScanImpl(String tableName) throws Exception { long curTs = System.currentTimeMillis(); for (int i = 0; i < recordCount; i++) { Put put = new Put(toBytes(key + i)); - put.add(family.getBytes(), (column).getBytes(), curTs, toBytes(value + i)); + put.addColumn(family.getBytes(), (column).getBytes(), curTs, toBytes(value + i)); hTable.put(put); } @@ -354,7 +354,7 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th String family = getColumnFamilyName(tableName); for (int i = 0; i < recordCount; i++) { Put put = new Put(toBytes(key + i)); - put.add(family.getBytes(), (column).getBytes(), curTs, toBytes(value)); + put.addColumn(family.getBytes(), (column).getBytes(), curTs, toBytes(value)); hTable.put(put); } } From 38a97ac1f1c2b9141213c157acb28bc2e1cf7e96 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Thu, 5 Jun 2025 11:16:37 +0800 Subject: [PATCH 19/36] add getDesc test case --- .../alipay/oceanbase/hbase/OHTableClient.java | 3 +- .../alipay/oceanbase/hbase/util/OHAdmin.java | 6 ++- .../oceanbase/hbase/OHTableClientTest.java | 39 +++++++++++++++---- src/test/java/unit_test_db.sql | 9 +++-- 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java b/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java index 770d523c..8a2c2f89 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTableClient.java @@ -223,7 +223,8 @@ public HTableDescriptor getTableDescriptor() throws IOException { @Override public TableDescriptor getDescriptor() throws IOException { - return null; + checkStatus(); + return ohTable.getDescriptor(); } @Override diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index ba1e483f..99f43267 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -145,6 +145,8 @@ public HTableDescriptor getTableDescriptor(TableName tableName) throws TableNotF if (e.getCause() instanceof ObTableTransportException && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { throw new TimeoutIOException(e.getCause()); + } else if (e.getCause().getMessage().contains("OB_TABLEGROUP_NOT_EXIST")) { + throw new TableNotFoundException(tableName); } else { throw e; } @@ -152,7 +154,7 @@ public HTableDescriptor getTableDescriptor(TableName tableName) throws TableNotF } @Override - public TableDescriptor getDescriptor(TableName tableName) throws TableNotFoundException, IOException { + public TableDescriptor getDescriptor(TableName tableName) throws IOException { OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); OHTableDescriptorExecutor executor = new OHTableDescriptorExecutor(tableName.getNameAsString(), tableClient); @@ -162,6 +164,8 @@ public TableDescriptor getDescriptor(TableName tableName) throws TableNotFoundEx if (e.getCause() instanceof ObTableTransportException && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { throw new TimeoutIOException(e.getCause()); + } else if (e.getCause().getMessage().contains("OB_TABLEGROUP_NOT_EXIST")) { + throw new TableNotFoundException(tableName); } else { throw e; } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java index 93bd4725..ce1bc3a1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java @@ -19,6 +19,8 @@ import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptor; import org.junit.*; import java.util.LinkedList; @@ -77,7 +79,9 @@ public void testNew() throws Exception { `T` bigint(20) NOT NULL, `V` varbinary(1024) DEFAULT NULL, PRIMARY KEY (`K`, `Q`, `T`) - ) TABLEGROUP = test_desc PARTITION BY RANGE COLUMNS(K) ( + ) TABLEGROUP = test_desc + KV_ATTRIBUTES ='{"Hbase": {"TimeToLive": 3600, "MaxVersions": 3}}' + PARTITION BY RANGE COLUMNS(K) ( PARTITION p1 VALUES LESS THAN ('c'), PARTITION p2 VALUES LESS THAN ('e'), PARTITION p3 VALUES LESS THAN ('g'), @@ -96,7 +100,9 @@ PARTITION p10 VALUES LESS THAN (MAXVALUE) `T` bigint(20) NOT NULL, `V` varbinary(1024) DEFAULT NULL, PRIMARY KEY (`K`, `Q`, `T`) - ) TABLEGROUP = test_desc PARTITION BY RANGE COLUMNS(K) ( + ) TABLEGROUP = test_desc + KV_ATTRIBUTES ='{"Hbase": {"TimeToLive": 7200, "MaxVersions": 3}}' + PARTITION BY RANGE COLUMNS(K) ( PARTITION p1 VALUES LESS THAN ('c'), PARTITION p2 VALUES LESS THAN ('e'), PARTITION p3 VALUES LESS THAN ('g'), @@ -116,11 +122,30 @@ public void testGetTableDescriptor() throws Exception { OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient(tableNameStr); hTable2.init(); try { - HTableDescriptor descriptor = hTable2.getTableDescriptor(); - Assert.assertNotNull(descriptor); - Assert.assertTrue(descriptor.hasFamily("family1".getBytes())); - Assert.assertTrue(descriptor.hasFamily("family2".getBytes())); - Assert.assertFalse(descriptor.hasFamily("family".getBytes())); + { + HTableDescriptor descriptor1 = hTable2.getTableDescriptor(); + Assert.assertNotNull(descriptor1); + Assert.assertEquals(2, descriptor1.getColumnFamilyCount()); + Assert.assertTrue(descriptor1.hasFamily("family1".getBytes())); + Assert.assertTrue(descriptor1.hasFamily("family2".getBytes())); + Assert.assertFalse(descriptor1.hasFamily("family".getBytes())); + ColumnFamilyDescriptor family1 = descriptor1.getColumnFamily("family1".getBytes()); + ColumnFamilyDescriptor family2 = descriptor1.getColumnFamily("family2".getBytes()); + Assert.assertEquals(3600, family1.getTimeToLive()); + Assert.assertEquals(7200, family2.getTimeToLive()); + } + { + TableDescriptor descriptor2 = hTable2.getDescriptor(); + Assert.assertNotNull(descriptor2); + Assert.assertEquals(2, descriptor2.getColumnFamilyCount()); + Assert.assertTrue(descriptor2.hasColumnFamily("family1".getBytes())); + Assert.assertTrue(descriptor2.hasColumnFamily("family2".getBytes())); + Assert.assertFalse(descriptor2.hasColumnFamily("family".getBytes())); + ColumnFamilyDescriptor family1 = descriptor2.getColumnFamily("family1".getBytes()); + ColumnFamilyDescriptor family2 = descriptor2.getColumnFamily("family2".getBytes()); + Assert.assertEquals(3600, family1.getTimeToLive()); + Assert.assertEquals(7200, family2.getTimeToLive()); + } } finally { hTable2.close(); } diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index f6b1068f..cc7ddfec 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -298,14 +298,15 @@ CREATE TABLE `test_region_locator$family_region_locator` ( ); CREATE TABLEGROUP test_desc SHARDING = 'ADAPTIVE'; - CREATE TABLE `test_desc$family1` ( `K` varbinary(1024) NOT NULL, `Q` varbinary(256) NOT NULL, `T` bigint(20) NOT NULL, `V` varbinary(1024) DEFAULT NULL, PRIMARY KEY (`K`, `Q`, `T`) -) TABLEGROUP = test_desc PARTITION BY RANGE COLUMNS(K) ( +) TABLEGROUP = test_desc +KV_ATTRIBUTES ='{"Hbase": {"TimeToLive": 3600, "MaxVersions": 3}}' +PARTITION BY RANGE COLUMNS(K) ( PARTITION p1 VALUES LESS THAN ('c'), PARTITION p2 VALUES LESS THAN ('e'), PARTITION p3 VALUES LESS THAN ('g'), @@ -324,7 +325,9 @@ CREATE TABLE `test_desc$family2` ( `T` bigint(20) NOT NULL, `V` varbinary(1024) DEFAULT NULL, PRIMARY KEY (`K`, `Q`, `T`) -) TABLEGROUP = test_desc PARTITION BY RANGE COLUMNS(K) ( +) TABLEGROUP = test_desc +KV_ATTRIBUTES ='{"Hbase": {"TimeToLive": 7200, "MaxVersions": 3}}' +PARTITION BY RANGE COLUMNS(K) ( PARTITION p1 VALUES LESS THAN ('c'), PARTITION p2 VALUES LESS THAN ('e'), PARTITION p3 VALUES LESS THAN ('g'), From 692c7261c47189ccf70bff2fb8344175d8dc9b44 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Thu, 12 Jun 2025 14:14:43 +0800 Subject: [PATCH 20/36] change fastjson to jackson --- pom.xml | 5 ++ .../execute/AbstractObTableMetaExecutor.java | 17 ++++ .../hbase/execute/ObTableMetaExecutor.java | 17 ++++ .../hbase/util/OHCreateTableExecutor.java | 7 +- .../hbase/util/OHDeleteTableExecutor.java | 24 +++++- .../oceanbase/hbase/util/OHRegionLocator.java | 34 ++++++-- .../hbase/util/OHRegionLocatorExecutor.java | 44 +++++++--- .../oceanbase/hbase/util/OHRegionMetrics.java | 24 +++++- .../hbase/util/OHRegionMetricsExecutor.java | 52 +++++++++--- .../util/OHTableAccessControlExecutor.java | 27 +++++- .../hbase/util/OHTableDescriptorExecutor.java | 85 +++++++++++-------- .../hbase/util/OHTableExistsExecutor.java | 31 +++++-- 12 files changed, 280 insertions(+), 87 deletions(-) diff --git a/pom.xml b/pom.xml index 3b8fbf94..0a9a6022 100644 --- a/pom.xml +++ b/pom.xml @@ -198,6 +198,11 @@ 1.2.12 test + + com.fasterxml.jackson.core + jackson-databind + 2.19.0 + diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java index d6d99da1..1c5d51d4 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java @@ -1,3 +1,20 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.execute; import com.alipay.oceanbase.rpc.ObTableClient; diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java index 4e8a0ffb..d55ff275 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/ObTableMetaExecutor.java @@ -1,3 +1,20 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.execute; import com.alipay.oceanbase.rpc.ObTableClient; diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java index a56e8097..4bab8e3b 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java @@ -17,7 +17,7 @@ package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -53,7 +53,7 @@ public void createTable(TableDescriptor tableDescriptor, byte[][] splitKeys) thr final ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); - requestData.put("htable_name", tableDescriptor.getTableName().getName()); + requestData.put("htable_name", tableDescriptor.getTableName().getNameAsString()); Map> columnFamilies = new HashMap<>(); for (ColumnFamilyDescriptor columnDescriptor : tableDescriptor.getColumnFamilies()) { Map columnFamily = new HashMap<>(); @@ -62,7 +62,8 @@ public void createTable(TableDescriptor tableDescriptor, byte[][] splitKeys) thr columnFamilies.put(columnDescriptor.getNameAsString(), columnFamily); } requestData.put("column_families", columnFamilies); - String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); execute(client, request); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java index 47280128..1f689596 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHDeleteTableExecutor.java @@ -1,7 +1,23 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.annotation.JSONField; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -24,7 +40,6 @@ public ObTableRpcMetaType getMetaType() { return ObTableRpcMetaType.HTABLE_DELETE_TABLE; } - @Override public Void parse(ObTableMetaResponse response) throws IOException { // do nothing, error will be thrown from table @@ -36,7 +51,8 @@ public Void deleteTable(String tableName) throws IOException { request.setMetaType(getMetaType()); Map requestDataMap = new HashMap<>(); requestDataMap.put("table_name", tableName); - String jsonData = JSON.toJSONString(requestDataMap); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestDataMap); request.setData(jsonData); return execute(tableClient, request); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java index ab99bc22..4894501c 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java @@ -1,3 +1,20 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.rpc.ObTableClient; @@ -13,16 +30,16 @@ import java.util.List; public class OHRegionLocator implements RegionLocator { - private byte[][] startKeys; - private byte[][] endKeys; - private ObTableClient tableClient; - private TableName tableName; + private byte[][] startKeys; + private byte[][] endKeys; + private ObTableClient tableClient; + private TableName tableName; private List regionLocations; public OHRegionLocator(byte[][] startKeys, byte[][] endKeys, - List regionLocations, - TableName tableName, ObTableClient tableClient) { + List regionLocations, TableName tableName, + ObTableClient tableClient) { this.startKeys = startKeys; this.endKeys = endKeys; this.regionLocations = regionLocations; @@ -44,7 +61,8 @@ public HRegionLocation getRegionLocation(byte[] bytes) throws IOException { @Override public HRegionLocation getRegionLocation(byte[] bytes, boolean b) throws IOException { if (b || regionLocations.isEmpty()) { - OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), tableClient); + OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), + tableClient); try { RegionLocator location = executor.getRegionLocator(tableName.toString()); this.startKeys = location.getStartKeys(); @@ -52,7 +70,7 @@ public HRegionLocation getRegionLocation(byte[] bytes, boolean b) throws IOExcep this.regionLocations = location.getAllRegionLocations(); } catch (IOException e) { if (e.getCause() instanceof ObTableTransportException - && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { throw new TimeoutIOException(e.getCause()); } else { throw e; diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java index 435cd6f4..c901d4ad 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java @@ -1,13 +1,29 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.constant.Constants; -import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; -import com.alipay.oceanbase.rpc.location.model.TableEntry; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; @@ -19,7 +35,7 @@ import java.util.stream.IntStream; public class OHRegionLocatorExecutor extends AbstractObTableMetaExecutor { - private final String tableName; + private final String tableName; private final ObTableClient client; OHRegionLocatorExecutor(String tableName, ObTableClient client) { @@ -42,7 +58,8 @@ public ObTableRpcMetaType getMetaType() { public OHRegionLocator parse(ObTableMetaResponse response) throws IOException { try { final String jsonData = response.getData(); - final JSONObject jsonMap = Optional.ofNullable(JSON.parseObject(jsonData)) + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(jsonData)) .orElseThrow(() -> new IOException("jsonMap is null")); /* { @@ -77,14 +94,17 @@ public OHRegionLocator parse(ObTableMetaResponse response) throws IOException { ] } */ - - final List partitions = Optional.>ofNullable(jsonMap.getJSONArray("partitions")) + JsonNode partitionsNode = Optional.ofNullable(jsonMap.get("partitions")) .orElseThrow(() -> new IOException("partitions is null")); + List partitions = objectMapper.convertValue(partitionsNode, new TypeReference>(){}); - final List tableIdDict = Optional.>ofNullable(jsonMap.getJSONArray("table_id_dict")) + JsonNode tableIdDictNode = Optional.ofNullable(jsonMap.get("table_id_dict")) .orElseThrow(() -> new IOException("tableIdDict is null")); - final List replicaDict = Optional.>ofNullable(jsonMap.getJSONArray("replica_dict")) + List tableIdDict = objectMapper.convertValue(tableIdDictNode, new TypeReference>(){}); + + JsonNode replicaDictNode = Optional.ofNullable(jsonMap.get("replica_dict")) .orElseThrow(() -> new IOException("replicaDict is null")); + List replicaDict = objectMapper.convertValue(replicaDictNode, new TypeReference>(){}); final boolean isHashLikePartition = partitions.stream() .map(obj -> (List) obj) @@ -213,8 +233,8 @@ public OHRegionLocator getRegionLocator(final String tableName) throws IOExcepti request.setMetaType(getMetaType()); final Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - - final String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + final String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); return execute(client, request); diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java index 8f161575..0e1e4732 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetrics.java @@ -1,17 +1,33 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import org.apache.hadoop.hbase.RegionMetrics; import org.apache.hadoop.hbase.Size; -import java.util.Collections; import java.util.Map; public class OHRegionMetrics implements RegionMetrics { private final String tablegroup; - private final byte[] name; // tablet_name, id in String - private final Size storeFileSize; // tablet storage used in ssTable - private final Size memStoreSize; // tablet storage used in memTable + private final byte[] name; // tablet_name, id in String + private final Size storeFileSize; // tablet storage used in ssTable + private final Size memStoreSize; // tablet storage used in memTable OHRegionMetrics(String tablegroup, byte[] name, Size storeFileSize, Size memStoreSize) { this.tablegroup = tablegroup; diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java index 430f568f..40235ef3 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionMetricsExecutor.java @@ -1,7 +1,25 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -11,16 +29,15 @@ import org.apache.hadoop.hbase.Size; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class OHRegionMetricsExecutor extends AbstractObTableMetaExecutor> { private final ObTableClient tableClient; + OHRegionMetricsExecutor(ObTableClient tableClient) { this.tableClient = tableClient; } + @Override public ObTableRpcMetaType getMetaType() throws IOException { return ObTableRpcMetaType.HTABLE_REGION_METRICS; @@ -38,13 +55,21 @@ public ObTableRpcMetaType getMetaType() throws IOException { * */ @Override public List parse(ObTableMetaResponse response) throws IOException { + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(response.getData())) + .orElseThrow(() -> new IOException("jsonMap is null")); + JsonNode tableGroupNameNode = Optional.ofNullable(jsonMap.get("tableName")) + .orElseThrow(() -> new IOException("tableName is null")); + String tableGroupName = tableGroupNameNode.asText(); + JsonNode regionListNode = Optional.ofNullable(jsonMap.get("regionList")) + .orElseThrow(() -> new IOException("regionList is null")); + List regions = Optional.>ofNullable(objectMapper.convertValue(regionListNode.get("regions"), new TypeReference>() {})) + .orElseThrow(() -> new IOException("regions is null")); + List memTableSizeList = Optional.>ofNullable(objectMapper.convertValue(regionListNode.get("memTableSize"), new TypeReference>() {})) + .orElseThrow(() -> new IOException("memTableSize is null")); + List ssTableSizeList = Optional.>ofNullable(objectMapper.convertValue(regionListNode.get("ssTableSize"), new TypeReference>() {})) + .orElseThrow(() -> new IOException("ssTableSize is null")); List metricsList = new ArrayList<>(); - JSONObject metrcisJSONObject = JSON.parseObject(response.getData()); - String tableGroupName = metrcisJSONObject.getString("tableName"); - JSONObject regionList = metrcisJSONObject.getJSONObject("regionList"); - List regions = regionList.getJSONArray("regions").toJavaList(Integer.class); - List memTableSizeList = regionList.getJSONArray("memTableSize").toJavaList(Integer.class); - List ssTableSizeList = regionList.getJSONArray("ssTableSize").toJavaList(Integer.class); if (regions.isEmpty() || regions.size() != memTableSizeList.size() || memTableSizeList.size() != ssTableSizeList.size()) { throw new IOException("size length has to be the same"); } @@ -64,7 +89,8 @@ public List getRegionMetrics(String tableName) throws IOException request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); return execute(tableClient, request); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java index 090da7e4..9bf0075b 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableAccessControlExecutor.java @@ -1,6 +1,23 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -15,7 +32,7 @@ import java.util.Map; public class OHTableAccessControlExecutor extends AbstractObTableMetaExecutor { - private final ObTableClient tableClient; + private final ObTableClient tableClient; private final ObTableRpcMetaType type; OHTableAccessControlExecutor(ObTableClient tableClient, ObTableRpcMetaType type) { @@ -38,7 +55,8 @@ public void enableTable(String tableName) throws IOException, TableNotFoundExcep request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); execute(tableClient, request); } @@ -48,7 +66,8 @@ public void disableTable(String tableName) throws IOException, TableNotFoundExce request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); execute(tableClient, request); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java index dae981f0..de0ee103 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java @@ -1,7 +1,25 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -18,7 +36,7 @@ import java.util.Optional; public class OHTableDescriptorExecutor extends AbstractObTableMetaExecutor { - private final String tableName; + private final String tableName; private final ObTableClient client; public OHTableDescriptorExecutor(String tableName, ObTableClient client) { @@ -30,35 +48,35 @@ public OHTableDescriptorExecutor(String tableName, ObTableClient client) { public HTableDescriptor parse(ObTableMetaResponse response) throws IOException { try { final String jsonData = response.getData(); - final JSONObject jsonMap = Optional.ofNullable(JSON.parseObject(jsonData)) + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(jsonData)) .orElseThrow(() -> new IOException("jsonMap is null")); /* { - "cfDesc": { + "cfDescs": { "cf1": { - "TTL":604800 + "TTL":604800 }, "cf2": { - "TTL":259200 + "TTL":259200 } }, "tbDesc": { - "name":"test" + "name":"test", + "state":"disable" ("enable") } } */ HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName)); - JSONObject cfDesc = jsonMap.getJSONObject("cfDescs"); - if (cfDesc != null) { - for (Map.Entry entry : cfDesc.entrySet()) { - String cfName = entry.getKey(); - JSONObject attributes = (JSONObject) entry.getValue(); - HColumnDescriptor cf = new HColumnDescriptor(cfName); - cf.setTimeToLive(attributes.getIntValue("TTL")); - tableDescriptor.addFamily(cf); - } - } else { - throw new IOException("cfDesc is null"); + JsonNode cfDescsNode = Optional.ofNullable(jsonMap.get("cfDescs")) + .orElseThrow(() -> new IOException("cfDesc is null")); + Map cfDescsMap = objectMapper.convertValue(cfDescsNode, new TypeReference>(){}); + for (Map.Entry entry : cfDescsMap.entrySet()) { + String cfName = entry.getKey(); + JsonNode attributes = (JsonNode) entry.getValue(); + HColumnDescriptor cf = new HColumnDescriptor(cfName); + cf.setTimeToLive(attributes.get("TTL").asInt()); + tableDescriptor.addFamily(cf); } return tableDescriptor; } catch (IllegalArgumentException e) { @@ -71,14 +89,13 @@ public ObTableRpcMetaType getMetaType() throws IOException { return ObTableRpcMetaType.HTABLE_GET_DESC; } - public HTableDescriptor getTableDescriptor() throws IOException { final ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); final Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - - final String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + final String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); return execute(client, request); @@ -90,22 +107,21 @@ public boolean isDisable() throws IOException { request.setMetaType(getMetaType()); final Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - - final String jsonData = JSON.toJSONString(requestData); + final ObjectMapper objectMapper = new ObjectMapper(); + final String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); try { ObTableMetaResponse response = innerExecute(client, request); final String responseData = response.getData(); - final JSONObject jsonMap = Optional.ofNullable(JSON.parseObject(responseData)) + final JsonNode jsonMap = Optional.ofNullable(objectMapper.readTree(responseData)) .orElseThrow(() -> new IOException("jsonMap is null")); - JSONObject tbDesc = jsonMap.getJSONObject("tableDesc"); - if (tbDesc != null) { - String state = tbDesc.getString("state"); - if (state.compareToIgnoreCase("disable") == 0) { - isDisable = true; - } else { - isDisable = false; - } + JsonNode tbDesc = Optional.ofNullable(jsonMap.get("tableDesc")) + .orElseThrow(() -> new IOException("tableDesc is null")); + String state = tbDesc.get("state").asText(); + if (state.compareToIgnoreCase("disable") == 0) { + isDisable = true; + } else { + isDisable = false; } } catch (IOException e) { throw e; @@ -113,7 +129,8 @@ public boolean isDisable() throws IOException { return isDisable; } - private ObTableMetaResponse innerExecute(ObTableClient client, ObTableMetaRequest request) throws IOException { + private ObTableMetaResponse innerExecute(ObTableClient client, ObTableMetaRequest request) + throws IOException { if (request.getMetaType() != getMetaType()) { throw new IOException("Invalid meta type, expected " + getMetaType()); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java index 7b827226..ffaaca18 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableExistsExecutor.java @@ -1,7 +1,24 @@ +/*- + * #%L + * com.oceanbase:obkv-hbase-client + * %% + * Copyright (C) 2022 - 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -11,6 +28,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Optional; public class OHTableExistsExecutor extends AbstractObTableMetaExecutor { private final ObTableClient tableClient; @@ -27,8 +45,10 @@ public ObTableRpcMetaType getMetaType() throws IOException { @Override public Boolean parse(ObTableMetaResponse response) throws IOException { String jsonData = response.getData(); - JSONObject object = JSONObject.parseObject(jsonData); - return object.getBoolean("exists"); + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode jsonNode = Optional.ofNullable(objectMapper.readTree(jsonData)) + .orElseThrow(() -> new IOException("jsonMap is null")); + return jsonNode.get("exists").asBoolean(); } public Boolean tableExists(String tableName) throws IOException { @@ -36,7 +56,8 @@ public Boolean tableExists(String tableName) throws IOException { request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); requestData.put("table_name", tableName); - String jsonData = JSON.toJSONString(requestData); + ObjectMapper objectMapper = new ObjectMapper(); + String jsonData = objectMapper.writeValueAsString(requestData); request.setData(jsonData); return execute(tableClient, request); } From 4ec96cdf42d2e0cb18c9528d045e063ff8e13185 Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Thu, 12 Jun 2025 11:30:15 +0800 Subject: [PATCH 21/36] [Test] add defensive test for htable ddl --- .../alipay/oceanbase/hbase/util/OHAdmin.java | 54 +++-- .../hbase/util/OHCreateTableExecutor.java | 2 +- .../hbase/OHTableAdminInterfaceTest.java | 201 +++++++++++++++++- 3 files changed, 225 insertions(+), 32 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index add7434c..b91bfe6e 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -174,20 +174,19 @@ public TableDescriptor getDescriptor(TableName tableName) throws IOException { @Override public void createTable(TableDescriptor tableDescriptor) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); -// OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); -// ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableDescriptor.getTableName(), connectionConf); -// OHCreateTableExecutor executor = new OHCreateTableExecutor(tableClient); -// try { -// executor.createTable(tableDescriptor, null); -// } catch (IOException e) { -// if (e.getCause() instanceof ObTableTransportException -// && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { -// throw new TimeoutIOException(e.getCause()); -// } else { -// throw e; -// } -// } + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableDescriptor.getTableName(), connectionConf); + OHCreateTableExecutor executor = new OHCreateTableExecutor(tableClient); + try { + executor.createTable(tableDescriptor, null); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } } @Override @@ -207,20 +206,19 @@ public Future createTableAsync(TableDescriptor tableDescriptor, byte[][] b @Override public void deleteTable(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); -// OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); -// ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); -// OHDeleteTableExecutor executor = new OHDeleteTableExecutor(tableClient); -// try { -// executor.deleteTable(tableName.getNameAsString()); -// } catch (IOException e) { -// if (e.getCause() instanceof ObTableTransportException -// && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { -// throw new TimeoutIOException(e.getCause()); -// } else { -// throw e; -// } -// } + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHDeleteTableExecutor executor = new OHDeleteTableExecutor(tableClient); + try { + executor.deleteTable(tableName.getNameAsString()); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } } @Override diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java index a56e8097..c7f87a04 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHCreateTableExecutor.java @@ -53,7 +53,7 @@ public void createTable(TableDescriptor tableDescriptor, byte[][] splitKeys) thr final ObTableMetaRequest request = new ObTableMetaRequest(); request.setMetaType(getMetaType()); Map requestData = new HashMap<>(); - requestData.put("htable_name", tableDescriptor.getTableName().getName()); + requestData.put("htable_name", tableDescriptor.getTableName().getNameAsString()); Map> columnFamilies = new HashMap<>(); for (ColumnFamilyDescriptor columnDescriptor : tableDescriptor.getColumnFamilies()) { Map columnFamily = new HashMap<>(); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 748d21c8..aa056738 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -35,6 +35,7 @@ import java.io.IOException; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.LinkedList; @@ -780,17 +781,17 @@ public void testCreateDeleteTable() throws Exception { assertTrue(admin.tableExists(tableName)); // TODO: show create table, need to be replace by getDescriptor java.sql.Connection conn = ObHTableTestUtil.getConnection(); - String selectSql = "show create table " + tableName.getNameAsString() + "$" + cf1; + String selectSql = "show create table " + tableName.getNameAsString() + "$" + Bytes.toString(cf1); System.out.println("execute sql: " + selectSql); java.sql.ResultSet resultSet = conn.createStatement().executeQuery(selectSql); ResultSetPrinter.print(resultSet); - selectSql = "show create table " + tableName.getNameAsString() + "$" + cf2; + selectSql = "show create table " + tableName.getNameAsString() + "$" + Bytes.toString(cf2); System.out.println("execute sql: " + selectSql); resultSet = conn.createStatement().executeQuery(selectSql); ResultSetPrinter.print(resultSet); - selectSql = "show create table " + tableName.getNameAsString() + "$" + cf3; + selectSql = "show create table " + tableName.getNameAsString() + "$" + Bytes.toString(cf3); System.out.println("execute sql: " + selectSql); resultSet = conn.createStatement().executeQuery(selectSql); ResultSetPrinter.print(resultSet); @@ -1033,4 +1034,198 @@ public void testConcurCreateDelPerf() throws Exception { duration = System.currentTimeMillis() - start; System.out.println("delete " + tableNums + " tables cost " + duration + " ms."); } + + @Test + public void testHTableDDLDefense() throws Exception { + TableName tableName = TableName.valueOf("testHTableDefense"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + // 1. construct htable desc and column family desc + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + hcd1.setMaxVersions(2); + hcd1.setTimeToLive(172800); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + hcd1.setMaxVersions(1); + hcd1.setTimeToLive(86400); + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + + // 2. execute create table and check exists + try { + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + admin.createTable(htd); + assertTrue(admin.tableExists(tableName)); + + /// execute the following ddl stmt in created by admin table, should be prohibited + // 4. alter table add constraint + try { + String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT cons1 CHECK(T < 0)"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 5. alter table add index + try { + String sql = "alter table testHTableDefense$cf1 ADD INDEX idx_1(T)"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 5. alter table add fk + try { + String sql = "alter table testHTableDefense$cf1 MODIFY COLUMN V LONGTEXT"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 6. alter table modify column to lob + try { + String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(K) REFERENCES testHTableDefense$cf2(K)"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 7. create trigger + try { + String sql = " CREATE TRIGGER hbase_trigger_1" + + " AFTER INSERT ON testHTableDefense$cf1 FOR EACH ROW" + + " BEGIN END"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 8. create view + try { + String sql = " CREATE VIEW hbase_view_1 as select * from testHTableDefense$cf1"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 9. alter view + try { + String sql = "ALTER VIEW hbase_view_1 as select * from testHTableDefense$cf1"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 10. create index + try { + String sql = " CREATE INDEX testHTableDefense$cf1_idx_T on testHTableDefense$cf1(T)"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + + // 11. explicit create table and specify created_by:admin, should be prohibited + try { + String sql = "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CREATED_BY\": \"ADMIN\"}}'"; + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + } + + // 12. alter table to created_by:admin, should be prohibited + try { + String sql1 = "CREATE TABLE testHTableDefense$cf3(a int primary key)"; + System.out.println("execute sql: " + sql1); + conn.createStatement().execute(sql1); + String sql2 = "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CREATED_BY\": \"ADMIN\"}}'"; + System.out.println("execute sql: " + sql2); + conn.createStatement().execute(sql2); + } catch (SQLException e) { + e.printStackTrace(); + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("alter table kv attributes to created by admin not supported", e.getMessage()); + // clean table + String sql3 = "drop table if exists testHTableDefense$cf3"; + System.out.println("execute sql: " + sql3); + conn.createStatement().execute(sql3); + } + + // 13. disable a htable did not created by admin is not suppported + try { + String sql1 = "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2"; + System.out.println("execute sql: " + sql1); + conn.createStatement().execute(sql1); + String sql2 = "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf4(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2"; + System.out.println("execute sql: " + sql2); + conn.createStatement().execute(sql2); + admin.disableTable(TableName.valueOf("testHTableDefense2")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + + } + + // 14. delete a htable did not created by admin is not suppported + try { + String sql1 = "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2"; + System.out.println("execute sql: " + sql1); + conn.createStatement().execute(sql1); + String sql2 = "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf5(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2"; + System.out.println("execute sql: " + sql2); + conn.createStatement().execute(sql2); + admin.deleteTable(TableName.valueOf("testHTableDefense2")); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + } + + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } finally { + admin.disableTable(tableName); + admin.deleteTable(tableName); + String sql1 = "DROP TABLE IF EXISTS testHTableDefense2$cf4"; + System.out.println("execute sql: " + sql1); + conn.createStatement().execute(sql1); + String sql2 = "DROP TABLE IF EXISTS testHTableDefense2$cf5"; + System.out.println("execute sql: " + sql2); + conn.createStatement().execute(sql2); + String sql3 = "DROP TABLEGROUP IF EXISTS testHTableDefense2"; + System.out.println("execute sql: " + sql3); + conn.createStatement().execute(sql3); + } + } } From ac04d6ae37140293cb9667a1b49f2bc19d779def Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Thu, 12 Jun 2025 11:53:39 +0800 Subject: [PATCH 22/36] [Chore] open the limiation for other htable admin operation --- .../alipay/oceanbase/hbase/util/OHAdmin.java | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index b91bfe6e..99f43267 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -248,20 +248,19 @@ public Future truncateTableAsync(TableName tableName, boolean b) throws IO @Override public void enableTable(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); -// OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); -// ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); -// OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_ENABLE_TABLE); -// try { -// executor.enableTable(tableName.getNameAsString()); -// } catch (IOException e) { -// if (e.getCause() instanceof ObTableTransportException -// && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { -// throw new TimeoutIOException(e.getCause()); -// } else { -// throw e; -// } -// } + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableAccessControlExecutor executor = new OHTableAccessControlExecutor(tableClient, ObTableRpcMetaType.HTABLE_ENABLE_TABLE); + try { + executor.enableTable(tableName.getNameAsString()); + } catch (IOException e) { + if (e.getCause() instanceof ObTableTransportException + && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { + throw new TimeoutIOException(e.getCause()); + } else { + throw e; + } + } } @Override @@ -313,14 +312,12 @@ public HTableDescriptor[] disableTables(Pattern pattern) throws IOException { @Override public boolean isTableEnabled(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); -// return isDisabled(tableName) == false; + return isDisabled(tableName) == false; } @Override public boolean isTableDisabled(TableName tableName) throws IOException { - throw new FeatureNotSupportedException("does not support yet"); -// return isDisabled(tableName) == true; + return isDisabled(tableName) == true; } private boolean isDisabled(TableName tableName) throws IOException { From 91c9e5442fac2d2afebfb2fbb2ec49e39da9009a Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Thu, 12 Jun 2025 14:28:57 +0800 Subject: [PATCH 23/36] [Test] modify test cases after observer changed --- .../hbase/OHTableAdminInterfaceTest.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index aa056738..f083a392 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -1067,10 +1067,10 @@ public void testHTableDDLDefense() throws Exception { String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT cons1 CHECK(T < 0)"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 5. alter table add index @@ -1078,10 +1078,10 @@ public void testHTableDDLDefense() throws Exception { String sql = "alter table testHTableDefense$cf1 ADD INDEX idx_1(T)"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 5. alter table add fk @@ -1089,10 +1089,10 @@ public void testHTableDDLDefense() throws Exception { String sql = "alter table testHTableDefense$cf1 MODIFY COLUMN V LONGTEXT"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 6. alter table modify column to lob @@ -1100,10 +1100,10 @@ public void testHTableDDLDefense() throws Exception { String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(K) REFERENCES testHTableDefense$cf2(K)"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 7. create trigger @@ -1113,10 +1113,10 @@ public void testHTableDDLDefense() throws Exception { " BEGIN END"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 8. create view @@ -1124,10 +1124,10 @@ public void testHTableDDLDefense() throws Exception { String sql = " CREATE VIEW hbase_view_1 as select * from testHTableDefense$cf1"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 9. alter view @@ -1135,10 +1135,10 @@ public void testHTableDDLDefense() throws Exception { String sql = "ALTER VIEW hbase_view_1 as select * from testHTableDefense$cf1"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 10. create index @@ -1146,22 +1146,22 @@ public void testHTableDDLDefense() throws Exception { String sql = " CREATE INDEX testHTableDefense$cf1_idx_T on testHTableDefense$cf1(T)"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 11. explicit create table and specify created_by:admin, should be prohibited try { - String sql = "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CREATED_BY\": \"ADMIN\"}}'"; + String sql = "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'"; System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); - Assert.assertEquals("user ddl with created_by attribute not supported", e.getMessage()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } // 12. alter table to created_by:admin, should be prohibited @@ -1169,11 +1169,11 @@ public void testHTableDDLDefense() throws Exception { String sql1 = "CREATE TABLE testHTableDefense$cf3(a int primary key)"; System.out.println("execute sql: " + sql1); conn.createStatement().execute(sql1); - String sql2 = "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CREATED_BY\": \"ADMIN\"}}'"; + String sql2 = "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'"; System.out.println("execute sql: " + sql2); conn.createStatement().execute(sql2); + fail(); } catch (SQLException e) { - e.printStackTrace(); Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("alter table kv attributes to created by admin not supported", e.getMessage()); // clean table @@ -1191,8 +1191,8 @@ public void testHTableDDLDefense() throws Exception { System.out.println("execute sql: " + sql2); conn.createStatement().execute(sql2); admin.disableTable(TableName.valueOf("testHTableDefense2")); + fail(); } catch (Exception e) { - e.printStackTrace(); Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); } @@ -1206,8 +1206,8 @@ public void testHTableDDLDefense() throws Exception { System.out.println("execute sql: " + sql2); conn.createStatement().execute(sql2); admin.deleteTable(TableName.valueOf("testHTableDefense2")); + fail(); } catch (Exception e) { - e.printStackTrace(); Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); } From ab6c9d16c6c80b7441062e5179be8ae56477a5e7 Mon Sep 17 00:00:00 2001 From: GroundWu <1175416256@qq.com> Date: Thu, 12 Jun 2025 17:03:11 +0800 Subject: [PATCH 24/36] add htable ddl concurrent test case (#241) * add htable ddl concurrent test case * create table by admin * [Test] add defensive test for htable ddl * [Chore] open the limiation for other htable admin operation * [Test] modify test cases after observer changed * add htable ddl concurrent test case --------- Co-authored-by: shenyunlong.syl --- pom.xml | 2 +- .../hbase/OHTableAdminInterfaceTest.java | 297 +++++++----------- 2 files changed, 120 insertions(+), 179 deletions(-) diff --git a/pom.xml b/pom.xml index 3b8fbf94..233148c1 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ ${project.encoding} UTF-8 1.7.21 - 2.0.0 + 2.0.1-SNAPSHOT diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index f083a392..27d1dc24 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -37,10 +37,9 @@ import java.io.IOException; import java.sql.SQLException; import java.sql.Statement; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; +import java.util.*; import java.util.concurrent.*; +import java.util.stream.Collectors; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; import static org.apache.hadoop.hbase.util.Bytes.toBytes; @@ -270,59 +269,31 @@ public void testGetStartEndKeysOHTablePoolLoadNon() throws Exception { Assert.assertEquals(0, startEndKeys.getSecond()[0].length); } + public static void createTable(Admin admin, TableName tableName, String... columnFamilies) throws IOException { + HTableDescriptor htd = new HTableDescriptor(tableName); + // Add column families + for (String cf : columnFamilies) { + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(cf)); + htd.addFamily(hcd); + } + // Create the table + admin.createTable(htd); + } + @Test public void testAdminEnDisableTable() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); - st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + - "\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "\n" + - "CREATE DATABASE IF NOT EXISTS `n1`;\n" + - "use `n1`;\n" + - "CREATE TABLEGROUP IF NOT EXISTS `n1:test` SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `n1:test$family_group` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test`;" + - "\n" + - "CREATE TABLE IF NOT EXISTS `n1:test$family1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test`;"); + st.execute("CREATE DATABASE IF NOT EXISTS `en_dis`"); + st.close(); + conn.close(); Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); - assertTrue(admin.tableExists(TableName.valueOf("n1", "test"))); - assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); + createTable(admin, TableName.valueOf("test_en_dis_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("en_dis", "test"), "cf1", "cf2", "cf3"); + assertTrue(admin.tableExists(TableName.valueOf("en_dis", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); // 1. disable a non-existed table { IOException thrown = assertThrows(IOException.class, @@ -334,28 +305,28 @@ public void testAdminEnDisableTable() throws Exception { } // 2. write an enabled table, should succeed { - if (admin.isTableDisabled(TableName.valueOf("test_multi_cf"))) { - admin.enableTable(TableName.valueOf("test_multi_cf")); + if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { + admin.enableTable(TableName.valueOf("test_en_dis_tb")); } - batchInsert(10, "test_multi_cf"); - batchGet(10, "test_multi_cf"); + batchInsert(10, "test_en_dis_tb"); + batchGet(10, "test_en_dis_tb"); } // 3. disable a enable table { - if (admin.isTableEnabled(TableName.valueOf("test_multi_cf"))) { - admin.disableTable(TableName.valueOf("test_multi_cf")); + if (admin.isTableEnabled(TableName.valueOf("test_en_dis_tb"))) { + admin.disableTable(TableName.valueOf("test_en_dis_tb")); } // write and read disable table, should fail try { - batchInsert(10, "test_multi_cf"); + batchInsert(10, "test_en_dis_tb"); Assert.fail(); } catch (IOException ex) { Assert.assertTrue(ex.getCause() instanceof ObTableException); System.out.println(ex.getCause().getMessage()); } try { - batchGet(10, "test_multi_cf"); + batchGet(10, "test_en_dis_tb"); Assert.fail(); } catch (IOException ex) { Assert.assertTrue(ex.getCause() instanceof ObTableException); @@ -367,48 +338,48 @@ public void testAdminEnDisableTable() throws Exception { // 4. enable a disabled table { - if (admin.isTableDisabled(TableName.valueOf("test_multi_cf"))) { - admin.enableTable(TableName.valueOf("test_multi_cf")); + if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { + admin.enableTable(TableName.valueOf("test_en_dis_tb")); } // write an enabled table, should succeed - batchInsert(10, "test_multi_cf"); - batchGet(10, "test_multi_cf"); + batchInsert(10, "test_en_dis_tb"); + batchGet(10, "test_en_dis_tb"); } // 5. enable an enabled table { - if (admin.isTableDisabled(TableName.valueOf("n1", "test"))) { - admin.enableTable(TableName.valueOf("n1", "test")); + if (admin.isTableDisabled(TableName.valueOf("en_dis", "test"))) { + admin.enableTable(TableName.valueOf("en_dis", "test")); } try { - admin.enableTable(TableName.valueOf("n1", "test")); + admin.enableTable(TableName.valueOf("en_dis", "test")); Assert.fail(); } catch (IOException ex) { Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, ((ObTableException) ex.getCause()).getErrorCode()); } } // 6. disable a disabled table { - if (admin.isTableEnabled(TableName.valueOf("n1", "test"))) { - admin.disableTable(TableName.valueOf("n1", "test")); + if (admin.isTableEnabled(TableName.valueOf("en_dis", "test"))) { + admin.disableTable(TableName.valueOf("en_dis", "test")); } try { - admin.disableTable(TableName.valueOf("n1", "test")); + admin.disableTable(TableName.valueOf("en_dis", "test")); Assert.fail(); } catch (IOException ex) { Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, ((ObTableException) ex.getCause()).getErrorCode()); } } - admin.deleteTable(TableName.valueOf("test_multi_cf")); - assertFalse(admin.tableExists(TableName.valueOf("test_multi_cf"))); - admin.deleteTable(TableName.valueOf("n1", "test")); - assertFalse(admin.tableExists(TableName.valueOf("n1", "test"))); + admin.deleteTable(TableName.valueOf("test_en_dis_tb")); + assertFalse(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); + admin.deleteTable(TableName.valueOf("en_dis", "test")); + assertFalse(admin.tableExists(TableName.valueOf("en_dis", "test"))); } private void batchGet(int rows, String tablegroup) throws Exception { @@ -433,78 +404,60 @@ private void batchGet(int rows, String tablegroup) throws Exception { public void testAdminGetRegionMetrics() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); - st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE TABLEGROUP IF NOT EXISTS test_no_part SHARDING = 'ADAPTIVE';\n"+ - "CREATE TABLE IF NOT EXISTS `test_no_part$family_with_group1` (\n" + + st.execute("CREATE TABLEGROUP IF NOT EXISTS test_get_region_metrics SHARDING = 'ADAPTIVE';\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_get_region_metrics$family_with_group1` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_no_part;\n" + - "CREATE TABLE IF NOT EXISTS `test_no_part$family_with_group2` (\n" + + ") TABLEGROUP = test_get_region_metrics PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_get_region_metrics$family_with_group2` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_no_part;\n" + - "CREATE TABLE IF NOT EXISTS `test_no_part$family_with_group3` (\n" + + ") TABLEGROUP = test_get_region_metrics PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE TABLE IF NOT EXISTS `test_get_region_metrics$family_with_group3` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_no_part;\n" + - "CREATE DATABASE IF NOT EXISTS `n1`;\n" + - "use `n1`;\n" + - "CREATE TABLEGROUP IF NOT EXISTS `n1:test_multi_cf` SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group1` (\n" + + ") TABLEGROUP = test_get_region_metrics PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "\n" + + "CREATE DATABASE IF NOT EXISTS `get_region`;\n" + + "use `get_region`;\n" + + "CREATE TABLEGROUP IF NOT EXISTS `get_region:test_multi_cf` SHARDING = 'ADAPTIVE';\n" + + "CREATE TABLE IF NOT EXISTS `get_region:test_multi_cf$family_with_group1` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group2` (\n" + + ") TABLEGROUP = `get_region:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "CREATE TABLE IF NOT EXISTS `get_region:test_multi_cf$family_with_group2` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group3` (\n" + + ") TABLEGROUP = `get_region:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + + "CREATE TABLE IF NOT EXISTS `get_region:test_multi_cf$family_with_group3` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;"); + ") TABLEGROUP = `get_region:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;"); st.close(); conn.close(); - String tablegroup1 = "test_multi_cf"; - String tablegroup2 = "n1:test_multi_cf"; + String tablegroup1 = "test_get_region_metrics"; + String tablegroup2 = "get_region:test_multi_cf"; Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); @@ -613,79 +566,33 @@ public void testAdminGetRegionMetrics() throws Exception { public void testAdminDeleteTable() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); - st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group2` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group3` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE DATABASE IF NOT EXISTS `n1`;\n" + - "use `n1`;\n" + - "CREATE TABLEGROUP IF NOT EXISTS `n1:test_multi_cf` SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;\n" + - "CREATE TABLE IF NOT EXISTS `n1:test_multi_cf$family_with_group2` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = `n1:test_multi_cf` PARTITION BY KEY(`K`) PARTITIONS 3;"); + st.execute("CREATE DATABASE IF NOT EXISTS `del_tb`"); st.close(); conn.close(); Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); - assertTrue(admin.tableExists(TableName.valueOf("n1", "test_multi_cf"))); - assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); + createTable(admin, TableName.valueOf("test_del_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("del_tb", "test"), "cf1", "cf2", "cf3"); + assertTrue(admin.tableExists(TableName.valueOf("del_tb", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("test_del_tb"))); IOException thrown = assertThrows(IOException.class, () -> { admin.deleteTable(TableName.valueOf("tablegroup_not_exists")); }); Assert.assertTrue(thrown.getCause() instanceof ObTableException); Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); - admin.deleteTable(TableName.valueOf("n1", "test_multi_cf")); - admin.deleteTable(TableName.valueOf("test_multi_cf")); - assertFalse(admin.tableExists(TableName.valueOf("n1", "test_multi_cf"))); - assertFalse(admin.tableExists(TableName.valueOf("test_multi_cf"))); + admin.deleteTable(TableName.valueOf("del_tb", "test")); + admin.deleteTable(TableName.valueOf("test_del_tb")); + assertFalse(admin.tableExists(TableName.valueOf("del_tb", "test"))); + assertFalse(admin.tableExists(TableName.valueOf("test_del_tb"))); } @Test public void testAdminTableExists() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); Statement st = conn.createStatement(); - st.execute("CREATE TABLEGROUP IF NOT EXISTS test_multi_cf SHARDING = 'ADAPTIVE';\n" + - "CREATE TABLE IF NOT EXISTS `test_multi_cf$family_with_group1` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = test_multi_cf PARTITION BY KEY(`K`) PARTITIONS 10;\n" + - "CREATE DATABASE IF NOT EXISTS `n1`;\n" + - "use `n1`;\n" + - "CREATE TABLEGROUP IF NOT EXISTS `n1:test_multi_cf` SHARDING = 'ADAPTIVE';"); + st.execute("CREATE DATABASE IF NOT EXISTS `exist_tb`"); st.close(); conn.close(); Configuration conf = ObHTableTestUtil.newConfiguration(); @@ -697,14 +604,16 @@ public void testAdminTableExists() throws Exception { TableName.valueOf("random_string$"); }); Assert.assertFalse(admin.tableExists(TableName.valueOf("tablegroup_not_exists"))); - Assert.assertTrue(admin.tableExists(TableName.valueOf("n1", "test_multi_cf"))); - Assert.assertTrue(admin.tableExists(TableName.valueOf("test_multi_cf"))); + createTable(admin, TableName.valueOf("test_exist_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("exist_tb", "test"), "cf1", "cf2", "cf3");å + Assert.assertTrue(admin.tableExists(TableName.valueOf("test_exist_tb"))); + Assert.assertTrue(admin.tableExists(TableName.valueOf("exist_tb", "test"))); } private void batchInsert(int rows, String tablegroup) throws Exception { - byte[] family1 = Bytes.toBytes("family_with_group1"); - byte[] family2 = Bytes.toBytes("family_with_group2"); - byte[] family3 = Bytes.toBytes("family_with_group3"); + byte[] family1 = Bytes.toBytes("cf1"); + byte[] family2 = Bytes.toBytes("cf2"); + byte[] family3 = Bytes.toBytes("cf3"); byte[] family1_column1 = "family1_column1".getBytes(); byte[] family1_column2 = "family1_column2".getBytes(); byte[] family1_column3 = "family1_column3".getBytes(); @@ -840,6 +749,7 @@ void testConcurCreateDelTablesHelper(List tableNames, Boolean ignoreE hcd1.setMaxVersions(1); hcd1.setTimeToLive(86400); HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + List originalColumnDescriptors = Arrays.asList(hcd1, hcd2, hcd3); // 1. generate create table task, one task per table List> tasks = new ArrayList<>(); @@ -847,11 +757,19 @@ void testConcurCreateDelTablesHelper(List tableNames, Boolean ignoreE int finalI = i; tasks.add(()->{ HTableDescriptor htd = new HTableDescriptor(tableNames.get(finalI)); - htd.addFamily(hcd1); - htd.addFamily(hcd2); - htd.addFamily(hcd3); + List columnDescriptors = new ArrayList<>(originalColumnDescriptors); + // 随机打乱列族顺序 + Collections.shuffle(columnDescriptors); + String shuffledCfNames = columnDescriptors.stream() + .map(hcd -> Bytes.toString(hcd.getName())) + .collect(Collectors.joining(", ")); + System.out.println("Table " + tableNames.get(finalI) + " shuffled column families: " + shuffledCfNames); + for (HColumnDescriptor hcd : columnDescriptors) { + htd.addFamily(hcd); + } try { admin.createTable(htd); + System.out.println("success to create table:" + tableNames.get(finalI)); } catch (Exception e) { System.out.println(e); if (!ignoreException) { @@ -892,9 +810,31 @@ void testConcurCreateDelTablesHelper(List tableNames, Boolean ignoreE } // 4. disable all tables; + List> disableTasks = new ArrayList<>(); + for (int i = 0; i < tableNums; i++) { + int finalI = i; + disableTasks.add(()->{ + try { + admin.disableTable(tableNames.get(finalI)); + System.out.println("success to disable table:" + tableNames.get(finalI)); + } catch (Exception e) { + System.out.println(e); + if (!ignoreException) { + throw e; + } + } + return null; + }); + } + ExecutorService disExecutorService = Executors.newFixedThreadPool(tableNums); + disExecutorService.invokeAll(disableTasks); + disExecutorService.shutdown(); + disExecutorService.awaitTermination(1, TimeUnit.MINUTES); + + assertTrue(admin.isTableDisabled(tableNames.get(0))); for (int i = 0; i < tableNames.size(); i++) { TableName tableName = tableNames.get(i); - admin.disableTable(tableName); + assertTrue(admin.isTableDisabled(tableName)); } // 5. generate delete table task @@ -904,6 +844,7 @@ void testConcurCreateDelTablesHelper(List tableNames, Boolean ignoreE delTasks.add(()->{ try { admin.deleteTable(tableNames.get(finalI)); + System.out.println("success to drop table:" + tableNames.get(finalI)); } catch (Exception e) { System.out.println(e); if (!ignoreException) { From 7fdf24d736f9bfda5815d0714733611f25ece120 Mon Sep 17 00:00:00 2001 From: GroundWu <1175416256@qq.com> Date: Thu, 12 Jun 2025 17:09:27 +0800 Subject: [PATCH 25/36] fix for compile --- .../com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 27d1dc24..cd8d64bd 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -605,7 +605,7 @@ public void testAdminTableExists() throws Exception { }); Assert.assertFalse(admin.tableExists(TableName.valueOf("tablegroup_not_exists"))); createTable(admin, TableName.valueOf("test_exist_tb"), "cf1", "cf2", "cf3"); - createTable(admin, TableName.valueOf("exist_tb", "test"), "cf1", "cf2", "cf3");å + createTable(admin, TableName.valueOf("exist_tb", "test"), "cf1", "cf2", "cf3"); Assert.assertTrue(admin.tableExists(TableName.valueOf("test_exist_tb"))); Assert.assertTrue(admin.tableExists(TableName.valueOf("exist_tb", "test"))); } From 0ff5726a5b7fc126aaae77865afe664b84325bba Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Fri, 13 Jun 2025 11:18:57 +0800 Subject: [PATCH 26/36] [Test] add testcases for pk and kv_attributes --- .../hbase/OHTableAdminInterfaceTest.java | 330 ++++++++++-------- .../hbase/util/ObHTableTestUtil.java | 5 + 2 files changed, 190 insertions(+), 145 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index cd8d64bd..fb5ada0a 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -17,7 +17,6 @@ package com.alipay.oceanbase.hbase; -import com.alipay.oceanbase.hbase.util.OHRegionMetrics; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; import com.alipay.oceanbase.hbase.util.ResultSetPrinter; @@ -42,6 +41,7 @@ import java.util.stream.Collectors; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.executeSQL; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; import static org.junit.Assert.assertFalse; @@ -290,96 +290,125 @@ public void testAdminEnDisableTable() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); - createTable(admin, TableName.valueOf("test_en_dis_tb"), "cf1", "cf2", "cf3"); - createTable(admin, TableName.valueOf("en_dis", "test"), "cf1", "cf2", "cf3"); - assertTrue(admin.tableExists(TableName.valueOf("en_dis", "test"))); - assertTrue(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); - // 1. disable a non-existed table - { - IOException thrown = assertThrows(IOException.class, - () -> { - admin.disableTable(TableName.valueOf("tablegroup_not_exists")); - }); - assertTrue(thrown.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); - } - // 2. write an enabled table, should succeed - { - if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { - admin.enableTable(TableName.valueOf("test_en_dis_tb")); + try { + createTable(admin, TableName.valueOf("test_en_dis_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("en_dis", "test"), "cf1", "cf2", "cf3"); + assertTrue(admin.tableExists(TableName.valueOf("en_dis", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); + String kvAttrStrDefault = "{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\"}}"; + String kvAttributeDisable = "{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"disable\"}}"; + String kvAttributeEnable = "{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"enable\"}}"; + + checkKVAttributes("test_en_dis_tb$cf1", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf2", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf3", kvAttrStrDefault); + // 1. disable a non-existed table + { + IOException thrown = assertThrows(IOException.class, + () -> { + admin.disableTable(TableName.valueOf("tablegroup_not_exists")); + }); + assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); } - batchInsert(10, "test_en_dis_tb"); - batchGet(10, "test_en_dis_tb"); - } + // 2. write an enabled table, should succeed + { + if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { + admin.enableTable(TableName.valueOf("test_en_dis_tb")); + } + checkKVAttributes("test_en_dis_tb$cf1", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf2", kvAttrStrDefault); + checkKVAttributes("test_en_dis_tb$cf3", kvAttrStrDefault); - // 3. disable a enable table - { - if (admin.isTableEnabled(TableName.valueOf("test_en_dis_tb"))) { - admin.disableTable(TableName.valueOf("test_en_dis_tb")); - } - // write and read disable table, should fail - try { batchInsert(10, "test_en_dis_tb"); - Assert.fail(); - } catch (IOException ex) { - Assert.assertTrue(ex.getCause() instanceof ObTableException); - System.out.println(ex.getCause().getMessage()); - } - try { batchGet(10, "test_en_dis_tb"); - Assert.fail(); - } catch (IOException ex) { - Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, - ((ObTableException) ex.getCause()).getErrorCode()); } - } + // 3. disable a enable table + { + if (admin.isTableEnabled(TableName.valueOf("test_en_dis_tb"))) { + admin.disableTable(TableName.valueOf("test_en_dis_tb")); + } + checkKVAttributes("test_en_dis_tb$cf1", kvAttributeDisable); + checkKVAttributes("test_en_dis_tb$cf2", kvAttributeDisable); + checkKVAttributes("test_en_dis_tb$cf3", kvAttributeDisable); + // write and read disable table, should fail + try { + batchInsert(10, "test_en_dis_tb"); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + System.out.println(ex.getCause().getMessage()); + } + try { + batchGet(10, "test_en_dis_tb"); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, + ((ObTableException) ex.getCause()).getErrorCode()); + } - // 4. enable a disabled table - { - if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { - admin.enableTable(TableName.valueOf("test_en_dis_tb")); } - // write an enabled table, should succeed - batchInsert(10, "test_en_dis_tb"); - batchGet(10, "test_en_dis_tb"); - } - // 5. enable an enabled table - { - if (admin.isTableDisabled(TableName.valueOf("en_dis", "test"))) { - admin.enableTable(TableName.valueOf("en_dis", "test")); - } - try { - admin.enableTable(TableName.valueOf("en_dis", "test")); - Assert.fail(); - } catch (IOException ex) { - Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, - ((ObTableException) ex.getCause()).getErrorCode()); + // 4. enable a disabled table + { + if (admin.isTableDisabled(TableName.valueOf("test_en_dis_tb"))) { + admin.enableTable(TableName.valueOf("test_en_dis_tb")); + } + checkKVAttributes("test_en_dis_tb$cf1", kvAttributeEnable); + checkKVAttributes("test_en_dis_tb$cf2", kvAttributeEnable); + checkKVAttributes("test_en_dis_tb$cf3", kvAttributeEnable); + // write an enabled table, should succeed + batchInsert(10, "test_en_dis_tb"); + batchGet(10, "test_en_dis_tb"); } - } - // 6. disable a disabled table - { - if (admin.isTableEnabled(TableName.valueOf("en_dis", "test"))) { - admin.disableTable(TableName.valueOf("en_dis", "test")); + // 5. enable an enabled table + { + if (admin.isTableDisabled(TableName.valueOf("en_dis", "test"))) { + admin.enableTable(TableName.valueOf("en_dis", "test")); + } + checkKVAttributes("en_dis:test$cf1", kvAttrStrDefault); + checkKVAttributes("en_dis:test$cf2", kvAttrStrDefault); + checkKVAttributes("en_dis:test$cf3", kvAttrStrDefault); + try { + admin.enableTable(TableName.valueOf("en_dis", "test")); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, + ((ObTableException) ex.getCause()).getErrorCode()); + } } - try { - admin.disableTable(TableName.valueOf("en_dis", "test")); - Assert.fail(); - } catch (IOException ex) { - Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, - ((ObTableException) ex.getCause()).getErrorCode()); + + // 6. disable a disabled table + { + if (admin.isTableEnabled(TableName.valueOf("en_dis", "test"))) { + admin.disableTable(TableName.valueOf("en_dis", "test")); + } + checkKVAttributes("en_dis:test$cf1", kvAttributeDisable); + checkKVAttributes("en_dis:test$cf1", kvAttributeDisable); + checkKVAttributes("en_dis:test$cf1", kvAttributeDisable); + try { + admin.disableTable(TableName.valueOf("en_dis", "test")); + Assert.fail(); + } catch (IOException ex) { + Assert.assertTrue(ex.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, + ((ObTableException) ex.getCause()).getErrorCode()); + } } - } - admin.deleteTable(TableName.valueOf("test_en_dis_tb")); - assertFalse(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); - admin.deleteTable(TableName.valueOf("en_dis", "test")); - assertFalse(admin.tableExists(TableName.valueOf("en_dis", "test"))); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } finally { + admin.deleteTable(TableName.valueOf("test_en_dis_tb")); + assertFalse(admin.tableExists(TableName.valueOf("test_en_dis_tb"))); + admin.deleteTable(TableName.valueOf("en_dis", "test")); + assertFalse(admin.tableExists(TableName.valueOf("en_dis", "test"))); + } } private void batchGet(int rows, String tablegroup) throws Exception { @@ -1003,90 +1032,109 @@ public void testHTableDDLDefense() throws Exception { assertTrue(admin.tableExists(tableName)); /// execute the following ddl stmt in created by admin table, should be prohibited - // 4. alter table add constraint + // 3. alter table add constraint try { - String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT cons1 CHECK(T < 0)"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "alter table testHTableDefense$cf1 ADD CONSTRAINT cons1 CHECK(T < 0)", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 5. alter table add index + // 4. alter table add index try { - String sql = "alter table testHTableDefense$cf1 ADD INDEX idx_1(T)"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "alter table testHTableDefense$cf1 ADD INDEX idx_1(T)", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 5. alter table add fk + // 5. alter table modify column to lob try { - String sql = "alter table testHTableDefense$cf1 MODIFY COLUMN V LONGTEXT"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "alter table testHTableDefense$cf1 MODIFY COLUMN V LONGTEXT", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 6. alter table modify column to lob + // 6. alter hbase admin table add fk try { - String sql = "alter table testHTableDefense$cf1 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(K) REFERENCES testHTableDefense$cf2(K)"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "alter table testHTableDefense$cf1 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(K) REFERENCES testHTableDefense$cf2(K)", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 7. create trigger + // 7. create a normal table to refer to hbase admin table try { - String sql = " CREATE TRIGGER hbase_trigger_1" + + executeSQL(conn, "create table testHTableDefense_t1(a varbinary(1024) primary key, FOREIGN KEY(a) REFERENCES testHTableDefense$cf1(K));" , true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + + // 8. alter a normal table to refer to hbase admin table + try { + executeSQL(conn, "create table testHTableDefense_t2(a varbinary(1024) primary key)", true); + executeSQL(conn, "alter table testHTableDefense_t2 ADD CONSTRAINT hbase_fk_1 FOREIGN KEY(a) REFERENCES testHTableDefense$cf1(K);", true); + fail(); + } catch (SQLException e) { + Assert.assertEquals(1235, e.getErrorCode()); + Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); + } + // 9. create a normal table A to refer to a table mock parent table B, and create table B using hbase admin + try { + executeSQL(conn, "SET foreign_key_checks = 0", true); + + executeSQL(conn, "create table testHTableDefense_t3(a varbinary(1024) primary key, FOREIGN KEY(a) REFERENCES testHTableDefense2$cf1(K));", true); + HTableDescriptor htd2 = new HTableDescriptor(TableName.valueOf("testHTableDefense2")); + HColumnDescriptor hcd4 = new HColumnDescriptor("cf1".getBytes()); + hcd4.setMaxVersions(2); + hcd4.setTimeToLive(172800); + htd2.addFamily(hcd4); + admin.createTable(htd2); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); + } + + + // 10. create trigger + try { + executeSQL(conn, " CREATE TRIGGER hbase_trigger_1" + " AFTER INSERT ON testHTableDefense$cf1 FOR EACH ROW" + - " BEGIN END"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + " BEGIN END", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 8. create view + // 11. create view try { - String sql = " CREATE VIEW hbase_view_1 as select * from testHTableDefense$cf1"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, " CREATE VIEW hbase_view_1 as select * from testHTableDefense$cf1", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 9. alter view + // 12. alter view try { - String sql = "ALTER VIEW hbase_view_1 as select * from testHTableDefense$cf1"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "ALTER VIEW hbase_view_1 as select * from testHTableDefense$cf1", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 10. create index + // 13. create index try { - String sql = " CREATE INDEX testHTableDefense$cf1_idx_T on testHTableDefense$cf1(T)"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, " CREATE INDEX testHTableDefense$cf1_idx_T on testHTableDefense$cf1(T)", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); @@ -1094,25 +1142,19 @@ public void testHTableDDLDefense() throws Exception { } - // 11. explicit create table and specify created_by:admin, should be prohibited + // 14. explicit create table and specify created_by:admin, should be prohibited try { - String sql = "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'"; - System.out.println("execute sql: " + sql); - conn.createStatement().execute(sql); + executeSQL(conn, "CREATE TABLE testHTableDefense$cf3(a int primary key) kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); Assert.assertEquals("table kv_attribute with '\"CreateBy\": \"Admin\"' not supported", e.getMessage()); } - // 12. alter table to created_by:admin, should be prohibited + // 15. alter table to created_by:admin, should be prohibited try { - String sql1 = "CREATE TABLE testHTableDefense$cf3(a int primary key)"; - System.out.println("execute sql: " + sql1); - conn.createStatement().execute(sql1); - String sql2 = "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'"; - System.out.println("execute sql: " + sql2); - conn.createStatement().execute(sql2); + executeSQL(conn, "CREATE TABLE testHTableDefense$cf3(a int primary key)", true); + executeSQL(conn, "alter table testHTableDefense$cf3 kv_attributes ='{\"Hbase\": {\"CreatedBy\": \"Admin\"}}'", true); fail(); } catch (SQLException e) { Assert.assertEquals(1235, e.getErrorCode()); @@ -1123,29 +1165,21 @@ public void testHTableDDLDefense() throws Exception { conn.createStatement().execute(sql3); } - // 13. disable a htable did not created by admin is not suppported + // 16. disable a htable did not created by admin is not suppported try { - String sql1 = "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2"; - System.out.println("execute sql: " + sql1); - conn.createStatement().execute(sql1); - String sql2 = "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf4(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2"; - System.out.println("execute sql: " + sql2); - conn.createStatement().execute(sql2); + executeSQL(conn, "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2", true); + executeSQL(conn, "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf4(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2", true); admin.disableTable(TableName.valueOf("testHTableDefense2")); fail(); } catch (Exception e) { Assert.assertEquals(-4007, ((ObTableException)e.getCause()).getErrorCode()); - } - // 14. delete a htable did not created by admin is not suppported + // 17. delete a htable did not created by admin is not suppported try { - String sql1 = "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2"; - System.out.println("execute sql: " + sql1); - conn.createStatement().execute(sql1); - String sql2 = "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf5(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2"; - System.out.println("execute sql: " + sql2); - conn.createStatement().execute(sql2); + executeSQL(conn, "CREATE TABLEGROUP IF NOT EXISTS testHTableDefense2", true); + executeSQL(conn, + "CREATE TABLE IF NOT EXISTS testHTableDefense2$cf5(a int primary key) kv_attributes ='{\"Hbase\": {}}' TABLEGROUP=testHTableDefense2", true); admin.deleteTable(TableName.valueOf("testHTableDefense2")); fail(); } catch (Exception e) { @@ -1158,15 +1192,21 @@ public void testHTableDDLDefense() throws Exception { } finally { admin.disableTable(tableName); admin.deleteTable(tableName); - String sql1 = "DROP TABLE IF EXISTS testHTableDefense2$cf4"; - System.out.println("execute sql: " + sql1); - conn.createStatement().execute(sql1); - String sql2 = "DROP TABLE IF EXISTS testHTableDefense2$cf5"; - System.out.println("execute sql: " + sql2); - conn.createStatement().execute(sql2); - String sql3 = "DROP TABLEGROUP IF EXISTS testHTableDefense2"; - System.out.println("execute sql: " + sql3); - conn.createStatement().execute(sql3); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense2$cf4", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense2$cf5", true); + executeSQL(conn, "DROP TABLEGROUP IF EXISTS testHTableDefense2", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense_t1", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense_t2", true); + executeSQL(conn, "DROP TABLE IF EXISTS testHTableDefense_t3", true); } } + + void checkKVAttributes(String tableName, String kvAttributes) throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery("select kv_attributes from oceanbase.__all_table where table_name = '" + tableName + "'"); + resultSet.next(); + String value = resultSet.getString(1); + Assert.assertEquals(kvAttributes, value); + Assert.assertFalse(resultSet.next()); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index b2c94e6c..b0310567 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -229,4 +229,9 @@ public static boolean secureCompare(byte[] a, byte[] b) { } return diff == 0; } + + public static void executeSQL(Connection conn, String sql, boolean printSQL) throws SQLException { + System.out.println("execute sql: " + sql); + conn.createStatement().execute(sql); + } } \ No newline at end of file From 3c9e9fc2464b247981b9fbf9a26bc826c32f3014 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Fri, 13 Jun 2025 17:53:47 +0800 Subject: [PATCH 27/36] fix namespace not exsit --- .../alipay/oceanbase/hbase/util/OHAdmin.java | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java index 99f43267..5cb7d92c 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHAdmin.java @@ -3,8 +3,10 @@ import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.bolt.transport.TransportCodes; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; +import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.exception.ObTableTransportException; import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.*; @@ -63,10 +65,27 @@ public Connection getConnection() { @Override public boolean tableExists(TableName tableName) throws IOException { - OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); - ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); - OHTableExistsExecutor executor = new OHTableExistsExecutor(tableClient); - return executor.tableExists(tableName.getNameAsString()); + try { + OHConnectionConfiguration connectionConf = new OHConnectionConfiguration(conf); + ObTableClient tableClient = ObTableClientManager.getOrCreateObTableClientByTableName(tableName, connectionConf); + OHTableExistsExecutor executor = new OHTableExistsExecutor(tableClient); + return executor.tableExists(tableName.getNameAsString()); + } catch (Exception e) { + // try to get the original cause + Throwable cause = e.getCause(); + while(cause != null && cause.getCause() != null) { + cause = cause.getCause(); + } + if (cause instanceof ObTableException) { + int errCode = ((ObTableException) cause).getErrorCode(); + // if the original cause is database_not_exist, means namespace in tableName does not exist + // for HBase, namespace not exist will not throw exceptions but will return false + if (errCode == ResultCodes.OB_ERR_BAD_DATABASE.errorCode) { + return false; + } + } + throw e; + } } @Override From 5fcfbcc2f0d67da4821cde5b9d5c5250013128fa Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 16 Jun 2025 19:43:56 +0800 Subject: [PATCH 28/36] patch bufferedMutator deals exception --- .../hbase/util/OHBufferedMutatorImpl.java | 21 +++++-------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java index d6e63d40..76ddf9f7 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBufferedMutatorImpl.java @@ -275,15 +275,11 @@ private void execute(boolean flushAll) throws IOException { // if commit all successfully, clean execBuffer execBuffer.clear(); } catch (Exception ex) { - LOGGER.error(LCD.convert("01-00026"), ex); - if (ex.getCause() instanceof RetriesExhaustedWithDetailsException) { - LOGGER.error(tableName + ": One or more of the operations have failed after retries."); - RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex.getCause(); - // recollect failed mutations - execBuffer.clear(); - for (int i = 0; i < retryException.getNumExceptions(); ++i) { - execBuffer.add((Mutation) retryException.getRow(i)); - } + // do not recollect error operations, notify outside + LOGGER.error("error happens, table name: {}", tableName.getNameAsString(), ex); + if (ex instanceof RetriesExhaustedWithDetailsException) { + LOGGER.error("TableName: {}, One or more of the operations have failed after retries.", tableName.getNameAsString(), ex); + RetriesExhaustedWithDetailsException retryException = (RetriesExhaustedWithDetailsException) ex; if (listener != null) { listener.onException(retryException, this); } else { @@ -293,13 +289,6 @@ private void execute(boolean flushAll) throws IOException { LOGGER.error("Errors unrelated to operations occur during mutation operation", ex); throw ex; } - } finally { - for (Mutation mutation : execBuffer) { - long size = mutation.heapSize(); - currentAsyncBufferSize.addAndGet(size); - asyncWriteBuffer.add(mutation); - undealtMutationCount.incrementAndGet(); - } } } From 41795b1ad0ce84cc2017505caf268132da42a9a5 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Mon, 16 Jun 2025 19:37:22 +0800 Subject: [PATCH 29/36] fix jackson bug --- .../hbase/util/OHTableDescriptorExecutor.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java index de0ee103..dcf42681 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHTableDescriptorExecutor.java @@ -17,7 +17,6 @@ package com.alipay.oceanbase.hbase.util; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.alipay.oceanbase.hbase.execute.AbstractObTableMetaExecutor; @@ -32,8 +31,10 @@ import java.io.IOException; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.Optional; +import java.util.stream.Stream; public class OHTableDescriptorExecutor extends AbstractObTableMetaExecutor { private final String tableName; @@ -56,9 +57,11 @@ public HTableDescriptor parse(ObTableMetaResponse response) throws IOException { "cfDescs": { "cf1": { "TTL":604800 + "maxVersions": 3 }, "cf2": { "TTL":259200 + "maxVersions": 2 } }, "tbDesc": { @@ -70,14 +73,17 @@ public HTableDescriptor parse(ObTableMetaResponse response) throws IOException { HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName)); JsonNode cfDescsNode = Optional.ofNullable(jsonMap.get("cfDescs")) .orElseThrow(() -> new IOException("cfDesc is null")); - Map cfDescsMap = objectMapper.convertValue(cfDescsNode, new TypeReference>(){}); - for (Map.Entry entry : cfDescsMap.entrySet()) { + Stream> stream = cfDescsNode.propertyStream(); + stream.forEach(entry -> { String cfName = entry.getKey(); - JsonNode attributes = (JsonNode) entry.getValue(); + JsonNode value = entry.getValue(); + int ttl = value.path("TTL").asInt(); + int maxVersions = value.path("maxVersions").asInt(); HColumnDescriptor cf = new HColumnDescriptor(cfName); - cf.setTimeToLive(attributes.get("TTL").asInt()); + cf.setTimeToLive(ttl); + cf.setMaxVersions(maxVersions); tableDescriptor.addFamily(cf); - } + }); return tableDescriptor; } catch (IllegalArgumentException e) { throw new IOException("Failed to parse response", e); From cdadb4a2c3def86766364e9d0e89286c0d41d41f Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 11 Jun 2025 17:03:18 +0800 Subject: [PATCH 30/36] add test case for admin interface error injection --- .../hbase/OHTableAdminInterfaceTest.java | 120 ++++++++++++++++++ .../hbase/util/ObHTableTestUtil.java | 56 +++++++- 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index fb5ada0a..df8a66d3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -67,6 +67,58 @@ public OHTablePool setUpPool() throws IOException { return ohTablePool; } + enum ErrSimPoint { + EN_CREATE_HTABLE_TG_FINISH_ERR(2621), + EN_CREATE_HTABLE_CF_FINISH_ERR(2622), + EN_DISABLE_HTABLE_CF_FINISH_ERR(2623), + EN_DELETE_HTABLE_CF_FINISH_ERR(2624); + + private final int errCode; + + ErrSimPoint(int errCode) { + this.errCode = errCode; + } + + public int getErrCode() { + return errCode; + } + } + + private void setErrSimPoint(ErrSimPoint errSimPoint, boolean enable) { + java.sql.Connection connection = null; + java.sql.Statement statement = null; + + try { + connection = ObHTableTestUtil.getSysTenantConnection(); + statement = connection.createStatement(); + + String sql = String.format( + "alter system set_tp tp_no = %d, error_code = 4016, frequency = %d", + errSimPoint.getErrCode(), + enable ? 1 : 0 + ); + + statement.execute(sql); + } catch (Exception e) { + throw new RuntimeException("Error injection setup failed", e); + } finally { + if (statement != null) { + try { + statement.close(); + } catch (Exception e) { + // ignore + } + } + if (connection != null) { + try { + connection.close(); + } catch (Exception e) { + // ignore + } + } + } + } + @Test public void testGetStartEndKeysOHTableClientRange() throws Exception { // Init OHTableClient @@ -1209,4 +1261,72 @@ void checkKVAttributes(String tableName, String kvAttributes) throws Exception { Assert.assertEquals(kvAttributes, value); Assert.assertFalse(resultSet.next()); } + + // NOTE: observer should build with `-DOB_ERRSIM=ON` option, otherwise the test will fail + // This test verifies error injection scenarios for table operations + @Test + public void testCreateTableInjectError() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + byte[] tableName = Bytes.toBytes("test_create_table_inject_error"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + byte[] cf3 = Bytes.toBytes("cf3"); + + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + htd.addFamily(hcd3); + + // 1. open err EN_CREATE_HTABLE_TG_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_TG_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.createTable(htd)); + assertFalse("Table should not exist after TG error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_TG_FINISH_ERR, false); + + // 2. open err EN_CREATE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_CF_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.createTable(htd)); + assertFalse("Table should not exist after CF error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_CF_FINISH_ERR, false); + + // 3. create table without error + admin.createTable(htd); + assertTrue("Table should exist after normal creation", + admin.tableExists(TableName.valueOf(tableName))); + assertEquals("Table should have 3 column families", 3, + admin.getTableDescriptor(TableName.valueOf(tableName)).getFamilies().size()); + + // 4. open err EN_DISABLE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_DISABLE_HTABLE_CF_FINISH_ERR, true); + admin.disableTable(TableName.valueOf(tableName)); + assertFalse("Table should not be disabled after disable error injection", + admin.isTableDisabled(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_DISABLE_HTABLE_CF_FINISH_ERR, false); + + // 5. disable table without error + admin.disableTable(TableName.valueOf(tableName)); + assertTrue("Table should be disabled after normal disable", + admin.isTableDisabled(TableName.valueOf(tableName))); + + // 6. open err EN_DELETE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, true); + admin.deleteTable(TableName.valueOf(tableName)); + assertTrue("Table should still exist after delete error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, false); + + // 7. delete table without error + admin.deleteTable(TableName.valueOf(tableName)); + assertFalse("Table should not exist after normal delete", + admin.tableExists(TableName.valueOf(tableName))); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index b0310567..ef6cf8e3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -55,7 +55,13 @@ public class ObHTableTestUtil { + "oceanbase?" + "useUnicode=TRUE&" + "characterEncoding=utf-8&" + "socketTimeout=3000000&" + "connectTimeout=60000"; + public static String SYS_TENANT_JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + + "oceanbase?" + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + "connectTimeout=60000"; + public static String SYS_TENANT_USER_NAME = "root@sys"; + public static String SYS_TENANT_PASSWORD = ""; public static String SQL_FORMAT = "truncate %s"; public static List tableNameList = new LinkedList(); public static Connection conn; @@ -172,6 +178,17 @@ static public Connection getSysConnection() { } } + static public Connection getSysTenantConnection() { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + Connection conn = DriverManager + .getConnection(SYS_TENANT_JDBC_URL, SYS_TENANT_USER_NAME, SYS_TENANT_PASSWORD); + return conn; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @FunctionalInterface public interface CheckedConsumer { void accept(T t) throws Throwable; @@ -234,4 +251,41 @@ public static void executeSQL(Connection conn, String sql, boolean printSQL) thr System.out.println("execute sql: " + sql); conn.createStatement().execute(sql); } -} \ No newline at end of file + + @FunctionalInterface + public interface CheckedRunnable { + void run() throws Exception; + } + + public static void executeIgnoreUnexpectedError(CheckedRunnable operation) throws Exception { + executeIgnoreExpectedErrors(operation, "OB_ERR_UNEXPECTED"); + } + + public static void executeIgnoreExpectedErrors(CheckedRunnable operation, String... expectedErrorMessages) throws Exception { + try { + operation.run(); + } catch (Exception e) { + boolean shouldIgnore = false; + String[] messagesToCheck = { + e.getMessage(), + e.getCause() != null ? e.getCause().getMessage() : null + }; + + for (String expectedMessage : expectedErrorMessages) { + for (String actualMessage : messagesToCheck) { + if (actualMessage != null && actualMessage.contains(expectedMessage)) { + shouldIgnore = true; + break; + } + } + if (shouldIgnore) { + break; + } + } + + if (!shouldIgnore) { + throw e; + } + } + } +} From bf3161a6a67f8db5881bba09f33c56e1c543e007 Mon Sep 17 00:00:00 2001 From: JackShi148 Date: Wed, 18 Jun 2025 11:07:42 +0800 Subject: [PATCH 31/36] use table fastxml:jackson, exclude hbase-client fastxml:jackson --- pom.xml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 7cce5f23..3e2e7667 100644 --- a/pom.xml +++ b/pom.xml @@ -167,6 +167,10 @@ jersey-json com.sun.jersey + + jackson-databind + com.fasterxml.jackson.core + @@ -198,11 +202,6 @@ 1.2.12 test - - com.fasterxml.jackson.core - jackson-databind - 2.19.0 - From 414972a38f9bd3869d81214c61167ff1ee93cda4 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 18 Jun 2025 16:52:21 +0800 Subject: [PATCH 32/36] fix hregion location build error --- .../hbase/util/OHRegionLocatorExecutor.java | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java index c901d4ad..33f65b46 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java @@ -141,33 +141,43 @@ private OHRegionLocator createRangePartitionLocator( throw new ObTableUnexpectedException( "The number of partitions should be an integer multiple of the number of tables"); } - // the size of partitions the multiple of the number of zones, the number of tablets and the number of tables - final int regionCount = partitions.size() / tableIdDict.size(); // get tablet boundaries of leaders and followers final List startKeysList = new ArrayList<>(); final List endKeysList = new ArrayList<>(); - - for (int i = 0; i < regionCount; ++i) { + // Currently based on SHARDING = 'ADAPTIVE' Table Group implementation for multi-CF, + // where one constraint is that Tables within the same Table Group have the same partitioning method, + // thus their associated partition boundaries are consistent. + // + // In native HBase, a Region can contain multiple CFs. + // Similarly, for OBKV-HBase, multiple CFs corresponding to related tablets also reside on the same machine. + // Therefore, here we maintain the same behavior by returning all partitions from just one table. + final int regionCountPerTable = partitions.size() / tableIdDict.size(); + + List oneTableLeaders = new ArrayList<>(); + for (int i = 0; i < regionCountPerTable; ++i) { boolean isLeader = ((int) ((List) partitions.get(i)).get(4) == 1); - if (!isLeader) { // only record leader's boundary - continue; + if (isLeader) { + oneTableLeaders.add(partitions.get(i)); } + } + // Note that the number of leaders per single table != (regionCountPerTable / replicaDict.size()). + for (int i = 0; i < oneTableLeaders.size(); ++i) { if (i == 0) { startKeysList.add(HConstants.EMPTY_BYTE_ARRAY); - endKeysList.add(((List) partitions.get(i)).get(2).toString().getBytes()); - } else if (i == regionCount - 1) { - startKeysList.add(((List) partitions.get(i - 1)).get(2).toString().getBytes()); + endKeysList.add(((List) oneTableLeaders.get(i)).get(2).toString().getBytes()); + } else if (i == oneTableLeaders.size() - 1) { + startKeysList.add(((List) oneTableLeaders.get(i - 1)).get(2).toString().getBytes()); endKeysList.add(HConstants.EMPTY_BYTE_ARRAY); } else { - startKeysList.add(((List) partitions.get(i - 1)).get(2).toString().getBytes()); - endKeysList.add(((List) partitions.get(i)).get(2).toString().getBytes()); + startKeysList.add(((List) oneTableLeaders.get(i - 1)).get(2).toString().getBytes()); + endKeysList.add(((List) oneTableLeaders.get(i)).get(2).toString().getBytes()); } } final byte[][] startKeys = startKeysList.toArray(new byte[0][]); final byte[][] endKeys = endKeysList.toArray(new byte[0][]); - // Create region locations for all regions - final List regionLocations = IntStream.range(0, regionCount) + // Create region locations for all regions in one table + final List regionLocations = IntStream.range(0, regionCountPerTable) .mapToObj(i -> { - final List partition = (List) partitions.get(Math.min(i, regionCount - 1)); + final List partition = (List) partitions.get(Math.min(i, regionCountPerTable - 1)); final int replicationIdx = (int) partition.get(3); final List hostInfo = (List) replicaDict.get(replicationIdx); @@ -176,10 +186,11 @@ private OHRegionLocator createRangePartitionLocator( (int) hostInfo.get(1), i ); + int boundIndex = i / replicaDict.size(); final HRegionInfo regionInfo = new HRegionInfo( TableName.valueOf(tableName), - startKeys[i], - endKeys[i] + startKeys[boundIndex], + endKeys[boundIndex] ); return new HRegionLocation(regionInfo, serverName, i); }) From b23214a7d4b85da5566756e679d620e8f7796497 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Thu, 19 Jun 2025 11:43:34 +0800 Subject: [PATCH 33/36] fix getRegion not return leader host --- .../oceanbase/hbase/util/OHRegionLocator.java | 29 +++++++------ .../hbase/util/OHRegionLocatorExecutor.java | 41 +++++++++++-------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java index 4894501c..42ea4179 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocator.java @@ -28,17 +28,18 @@ import java.io.IOException; import java.util.List; +import java.util.stream.Collectors; public class OHRegionLocator implements RegionLocator { - private byte[][] startKeys; - private byte[][] endKeys; - private ObTableClient tableClient; - private TableName tableName; + private byte[][] startKeys; + private byte[][] endKeys; + private final ObTableClient tableClient; + private final TableName tableName; - private List regionLocations; + private List> regionLocations; public OHRegionLocator(byte[][] startKeys, byte[][] endKeys, - List regionLocations, TableName tableName, + List> regionLocations, TableName tableName, ObTableClient tableClient) { this.startKeys = startKeys; this.endKeys = endKeys; @@ -50,24 +51,28 @@ public OHRegionLocator(byte[][] startKeys, byte[][] endKeys, @Override public HRegionLocation getRegionLocation(byte[] bytes) throws IOException { // check if bytes is in the range of startKeys and endKeys - for (HRegionLocation regionLocation : regionLocations) { - if (regionLocation.getRegionInfo().containsRow(bytes)) { - return regionLocation; + for (Pair pair : regionLocations) { + if (pair.getSecond() && pair.getFirst().getRegionInfo().containsRow(bytes)) { + return pair.getFirst(); } } return null; } + public List> getRegionLocationPair() { + return regionLocations; + } + @Override public HRegionLocation getRegionLocation(byte[] bytes, boolean b) throws IOException { if (b || regionLocations.isEmpty()) { OHRegionLocatorExecutor executor = new OHRegionLocatorExecutor(tableName.toString(), tableClient); try { - RegionLocator location = executor.getRegionLocator(tableName.toString()); + OHRegionLocator location = executor.getRegionLocator(tableName.toString()); this.startKeys = location.getStartKeys(); this.endKeys = location.getEndKeys(); - this.regionLocations = location.getAllRegionLocations(); + this.regionLocations = location.getRegionLocationPair(); } catch (IOException e) { if (e.getCause() instanceof ObTableTransportException && ((ObTableTransportException) e.getCause()).getErrorCode() == TransportCodes.BOLT_TIMEOUT) { @@ -82,7 +87,7 @@ public HRegionLocation getRegionLocation(byte[] bytes, boolean b) throws IOExcep @Override public List getAllRegionLocations() throws IOException { - return regionLocations; + return regionLocations.stream().map(Pair::getFirst).collect(Collectors.toList()); } /** diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java index 33f65b46..63978d6c 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHRegionLocatorExecutor.java @@ -28,6 +28,7 @@ import com.alipay.oceanbase.rpc.meta.ObTableMetaResponse; import com.alipay.oceanbase.rpc.meta.ObTableRpcMetaType; import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.util.Pair; import java.io.IOException; import java.util.*; @@ -72,25 +73,25 @@ public OHRegionLocator parse(ObTableMetaResponse response) throws IOException { "partitions": [ // 表1001的3个分区，每个分区3副本 [0, 50001, "rowkey_1", 0, 1], // leader - [0, 50001, "rowkey_1", 1, 0], // follower - [0, 50001, "rowkey_1", 2, 0], // follower + [0, 50001, "rowkey_1", 1, 2], // follower + [0, 50001, "rowkey_1", 2, 2], // follower [0, 50002, "rowkey_2", 0, 1], - [0, 50002, "rowkey_2", 1, 0], - [0, 50002, "rowkey_2", 2, 0], + [0, 50002, "rowkey_2", 1, 2], + [0, 50002, "rowkey_2", 2, 2], [0, 50003, "rowkey_3", 0, 1], - [0, 50003, "rowkey_3", 1, 0], - [0, 50003, "rowkey_3", 2, 0], + [0, 50003, "rowkey_3", 1, 2], + [0, 50003, "rowkey_3", 2, 2], // 表1002的3个分区，每个分区3副本 [1, 50004, "rowkey_1", 0, 1], - [1, 50004, "rowkey_1", 1, 0], - [1, 50004, "rowkey_1", 2, 0], + [1, 50004, "rowkey_1", 1, 2], + [1, 50004, "rowkey_1", 2, 2], [1, 50005, "rowkey_2", 0, 1], - [1, 50005, "rowkey_2", 1, 0], - [1, 50005, "rowkey_2", 2, 0], + [1, 50005, "rowkey_2", 1, 2], + [1, 50005, "rowkey_2", 2, 2], [1, 50006, "rowkey_3", 0, 1], - [1, 50006, "rowkey_3", 1, 0], - [1, 50006, "rowkey_3", 2, 0] + [1, 50006, "rowkey_3", 1, 2], + [1, 50006, "rowkey_3", 2, 2] ] } */ @@ -175,7 +176,7 @@ private OHRegionLocator createRangePartitionLocator( final byte[][] startKeys = startKeysList.toArray(new byte[0][]); final byte[][] endKeys = endKeysList.toArray(new byte[0][]); // Create region locations for all regions in one table - final List regionLocations = IntStream.range(0, regionCountPerTable) + final List regionLocations = IntStream.range(0, regionCountPerTable) .mapToObj(i -> { final List partition = (List) partitions.get(Math.min(i, regionCountPerTable - 1)); final int replicationIdx = (int) partition.get(3); @@ -192,7 +193,9 @@ private OHRegionLocator createRangePartitionLocator( startKeys[boundIndex], endKeys[boundIndex] ); - return new HRegionLocation(regionInfo, serverName, i); + HRegionLocation location = new HRegionLocation(regionInfo, serverName, i); + Boolean role = (int) partition.get(4) == 1; + return new Pair(location, role); }) .collect(Collectors.toList()); @@ -215,8 +218,8 @@ private OHRegionLocator createHashPartitionLocator( final byte[][] endKeys = new byte[1][]; startKeys[0] = HConstants.EMPTY_BYTE_ARRAY; endKeys[0] = HConstants.EMPTY_BYTE_ARRAY; - - final List regionLocations = IntStream.range(0, partitions.size()) + final int regionCountPerTable = partitions.size() / tableIdDict.size(); + final List regionLocations = IntStream.range(0, regionCountPerTable) .mapToObj(i -> { final List partition = (List) partitions.get(i); final int replicationIdx = (int) partition.get(3); @@ -232,10 +235,12 @@ private OHRegionLocator createHashPartitionLocator( startKeys[0], endKeys[0] ); - return new HRegionLocation(regionInfo, serverName, i); + HRegionLocation location = new HRegionLocation(regionInfo, serverName, i); + Boolean role = (int) partition.get(4) == 1; + return new Pair(location, role); }) .collect(Collectors.toList()); - + return new OHRegionLocator(startKeys, endKeys, regionLocations, TableName.valueOf(tableName), client); } From c27889e64a35c84dae643c5b04c26b639b3d1acf Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Thu, 19 Jun 2025 14:10:04 +0800 Subject: [PATCH 34/36] [Chore] add _enable_kv_hbase_admin_ddl test and fix admin ddl exception --- .../execute/AbstractObTableMetaExecutor.java | 3 +- .../hbase/util/OHBaseExceptionUtil.java | 27 +++ .../hbase/util/ObTableClientManager.java | 2 +- .../hbase/OHTableAdminInterfaceTest.java | 213 +++++++++++++++++- 4 files changed, 231 insertions(+), 14 deletions(-) create mode 100644 src/main/java/com/alipay/oceanbase/hbase/util/OHBaseExceptionUtil.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java index 1c5d51d4..05b48b4a 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java +++ b/src/main/java/com/alipay/oceanbase/hbase/execute/AbstractObTableMetaExecutor.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.hbase.execute; +import com.alipay.oceanbase.hbase.util.OHBaseExceptionUtil; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.exception.ObTableException; import com.alipay.oceanbase.rpc.meta.ObTableMetaRequest; @@ -41,7 +42,7 @@ public T execute(ObTableClient client, ObTableMetaRequest request) throws IOExce null /*tableName*/ ); } catch (Exception e) { - throw new IOException("Failed to execute request", e); + throw OHBaseExceptionUtil.convertTableException(e); } return parse(response); } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/OHBaseExceptionUtil.java b/src/main/java/com/alipay/oceanbase/hbase/util/OHBaseExceptionUtil.java new file mode 100644 index 00000000..7223ca90 --- /dev/null +++ b/src/main/java/com/alipay/oceanbase/hbase/util/OHBaseExceptionUtil.java @@ -0,0 +1,27 @@ +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.rpc.exception.ObTableException; +import com.alipay.oceanbase.rpc.protocol.payload.ResultCodes; +import org.apache.hadoop.hbase.*; + +import java.io.IOException; + +public class OHBaseExceptionUtil { + public static IOException convertTableException(Exception e) { + if (e instanceof ObTableException) { + final int errCode = ((ObTableException) e).getErrorCode(); + if (errCode == ResultCodes.OB_KV_HBASE_TABLE_NOT_EXISTS.errorCode) { + return (TableNotFoundException) new TableNotFoundException(e.getMessage()).initCause(e); + } else if (errCode == ResultCodes.OB_KV_HBASE_TABLE_EXISTS.errorCode) { + return (TableExistsException) new TableExistsException(e.getMessage()).initCause(e); + } else if (errCode == ResultCodes.OB_KV_HBASE_NAMESPACE_NOT_FOUND.errorCode) { + return (NamespaceNotFoundException) new NamespaceNotFoundException(e.getMessage()).initCause(e); + } else if (errCode == ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode) { + return (TableNotEnabledException) new TableNotEnabledException(e.getMessage()).initCause(e); + } else if (errCode == ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode) { + return (TableNotDisabledException) new TableNotDisabledException(e.getMessage()).initCause(e); + } + } + return new IOException("Failed to execute request", e); + } +} diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java index e0d50cea..9a3a56f2 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java @@ -120,7 +120,7 @@ public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableCli OB_TABLE_CLIENT_INSTANCE.put(obTableClientKey, obTableClient); } } catch (Exception e) { - throw new IOException(e); + throw OHBaseExceptionUtil.convertTableException(e); } finally { lock.unlock(); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index df8a66d3..660c8c12 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -29,10 +29,11 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; - import java.io.IOException; import java.sql.SQLException; import java.sql.Statement; @@ -45,7 +46,7 @@ import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; import static org.junit.Assert.assertFalse; -import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; public class OHTableAdminInterfaceTest { public OHTablePool setUpLoadPool() throws IOException { @@ -58,6 +59,28 @@ public OHTablePool setUpLoadPool() throws IOException { return ohTablePool; } + public static void openHbaseAdminDDL() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + String stmt = "ALTER SYSTEM SET _enable_kv_hbase_admin_ddl = true;"; + conn.createStatement().execute(stmt); + } + + public static void closeHbaseAdminDDL() throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + String stmt = "ALTER SYSTEM SET _enable_kv_hbase_admin_ddl = false;"; + conn.createStatement().execute(stmt); + } + + @BeforeClass + public static void before() throws Exception { + openHbaseAdminDDL(); + } + + @AfterClass + public static void finish() throws Exception { + closeHbaseAdminDDL(); + } + public OHTablePool setUpPool() throws IOException { Configuration c = ObHTableTestUtil.newConfiguration(); OHTablePool ohTablePool = new OHTablePool(c, 10); @@ -321,7 +344,8 @@ public void testGetStartEndKeysOHTablePoolLoadNon() throws Exception { Assert.assertEquals(0, startEndKeys.getSecond()[0].length); } - public static void createTable(Admin admin, TableName tableName, String... columnFamilies) throws IOException { + public static void createTable(Admin admin, TableName tableName, String... columnFamilies) + throws IOException { HTableDescriptor htd = new HTableDescriptor(tableName); // Add column families for (String cf : columnFamilies) { @@ -427,10 +451,8 @@ public void testAdminEnDisableTable() throws Exception { try { admin.enableTable(TableName.valueOf("en_dis", "test")); Assert.fail(); - } catch (IOException ex) { - Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_ENABLED.errorCode, - ((ObTableException) ex.getCause()).getErrorCode()); + } catch (Exception ex) { + Assert.assertEquals(TableNotDisabledException.class, ex.getClass()); } } @@ -445,10 +467,8 @@ public void testAdminEnDisableTable() throws Exception { try { admin.disableTable(TableName.valueOf("en_dis", "test")); Assert.fail(); - } catch (IOException ex) { - Assert.assertTrue(ex.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_KV_TABLE_NOT_DISABLED.errorCode, - ((ObTableException) ex.getCause()).getErrorCode()); + } catch (Exception ex) { + Assert.assertEquals(TableNotEnabledException.class, ex.getClass()); } } @@ -1255,7 +1275,9 @@ public void testHTableDDLDefense() throws Exception { void checkKVAttributes(String tableName, String kvAttributes) throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); - java.sql.ResultSet resultSet = conn.createStatement().executeQuery("select kv_attributes from oceanbase.__all_table where table_name = '" + tableName + "'"); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery( + "select kv_attributes from oceanbase.__all_table where table_name = '" + tableName + + "'"); resultSet.next(); String value = resultSet.getString(1); Assert.assertEquals(kvAttributes, value); @@ -1329,4 +1351,171 @@ public void testCreateTableInjectError() throws Exception { assertFalse("Table should not exist after normal delete", admin.tableExists(TableName.valueOf(tableName))); } + + @Test + public void testHbaseAdminDDLKnob() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + try { + closeHbaseAdminDDL(); + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException) e.getCause()).getErrorCode()); + } + + try { + admin.disableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException) e.getCause()).getErrorCode()); + } + + try { + admin.enableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException) e.getCause()).getErrorCode()); + } + + try { + admin.deleteTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(-4007, ((ObTableException) e.getCause()).getErrorCode()); + } + } catch (Exception e) { + e.printStackTrace(); + throw e; + } finally { + openHbaseAdminDDL(); + } + } + + @Test + public void testHbaseDDLException() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + // 1. create a created table + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableExistsException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 2. delete a non-exist table + try { + admin.deleteTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotFoundException.class); + } + + // 3. enable a enabled table + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + admin.enableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotDisabledException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 4. disable a disabled table + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + admin.disableTable(TableName.valueOf("t1")); + admin.disableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotEnabledException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 5. get htable descriptor from an uncreated table + try { + HTableDescriptor descriptor = admin.getTableDescriptor(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotFoundException.class); + } + + // 6. get region metrics from an uncreated table + try { + admin.getRegionMetrics(null, TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertEquals(e.getClass(), TableNotFoundException.class); + } + + // 6. create a table in an uncreated namespace + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableExistsException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 7. delete a non-exist table in an uncreated namespace + try { + admin.deleteTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotFoundException.class); + } + + // 8. enable a enabled table in an uncreated namespace + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + admin.enableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotDisabledException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 9. disable a disabled table in an uncreated namespace + try { + createTable(admin, TableName.valueOf("t1"), "cf1", "cf2", "cf3"); + admin.disableTable(TableName.valueOf("t1")); + admin.disableTable(TableName.valueOf("t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), TableNotEnabledException.class); + } finally { + admin.deleteTable(TableName.valueOf("t1")); + } + + // 10. get a table metrics from an uncreated namespace + try { + admin.getRegionMetrics(null, TableName.valueOf("n1:t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), NamespaceNotFoundException.class); + } + + // 11. check table exists from an uncreated namespace + try { + admin.tableExists(TableName.valueOf("n1:t1")); + fail(); + } catch (Exception e) { + Assert.assertEquals(e.getClass(), NamespaceNotFoundException.class); + } + + } } From beb162eba5f69f55aae754ae64211db29d7ee5e2 Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Mon, 23 Jun 2025 21:01:15 +0800 Subject: [PATCH 35/36] [Test] add test for create/drp tablegroup and ddl stmt str --- .../hbase/OHTableAdminInterfaceTest.java | 302 ++++++++++++++---- 1 file changed, 241 insertions(+), 61 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 660c8c12..0d13e35a 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -94,8 +94,9 @@ enum ErrSimPoint { EN_CREATE_HTABLE_TG_FINISH_ERR(2621), EN_CREATE_HTABLE_CF_FINISH_ERR(2622), EN_DISABLE_HTABLE_CF_FINISH_ERR(2623), - EN_DELETE_HTABLE_CF_FINISH_ERR(2624); - + EN_DELETE_HTABLE_CF_FINISH_ERR(2624), + EN_DELETE_HTABLE_SKIP_CF_ERR(2625); + private final int errCode; ErrSimPoint(int errCode) { @@ -568,7 +569,7 @@ public void testAdminGetRegionMetrics() throws Exception { admin.getRegionMetrics(null, TableName.valueOf("tablegroup_not_exists")); }); Assert.assertTrue(thrown.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + Assert.assertEquals(ResultCodes.OB_KV_HBASE_TABLE_NOT_EXISTS.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); // test use serverName without tableName to get region metrics assertThrows(FeatureNotSupportedException.class, @@ -663,6 +664,19 @@ public void testAdminGetRegionMetrics() throws Exception { assertEquals(3, metrics.size()); } + private void deleteTableIfExists(Admin admin, TableName tableName) throws Exception { + if (admin.tableExists(tableName)) { + if (admin.isTableEnabled(tableName)) { + admin.disableTable(tableName); + } + admin.deleteTable(tableName); + } + } + + private void deleteTableIfExists(Admin admin, String tableName) throws Exception { + deleteTableIfExists(admin, TableName.valueOf(tableName)); + } + @Test public void testAdminDeleteTable() throws Exception { java.sql.Connection conn = ObHTableTestUtil.getConnection(); @@ -673,20 +687,28 @@ public void testAdminDeleteTable() throws Exception { Configuration conf = ObHTableTestUtil.newConfiguration(); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); - createTable(admin, TableName.valueOf("test_del_tb"), "cf1", "cf2", "cf3"); - createTable(admin, TableName.valueOf("del_tb", "test"), "cf1", "cf2", "cf3"); - assertTrue(admin.tableExists(TableName.valueOf("del_tb", "test"))); - assertTrue(admin.tableExists(TableName.valueOf("test_del_tb"))); - IOException thrown = assertThrows(IOException.class, - () -> { - admin.deleteTable(TableName.valueOf("tablegroup_not_exists")); - }); - Assert.assertTrue(thrown.getCause() instanceof ObTableException); - Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); - admin.deleteTable(TableName.valueOf("del_tb", "test")); - admin.deleteTable(TableName.valueOf("test_del_tb")); - assertFalse(admin.tableExists(TableName.valueOf("del_tb", "test"))); - assertFalse(admin.tableExists(TableName.valueOf("test_del_tb"))); + try { + createTable(admin, TableName.valueOf("test_del_tb"), "cf1", "cf2", "cf3"); + createTable(admin, TableName.valueOf("del_tb", "test"), "cf1", "cf2", "cf3"); + assertTrue(admin.tableExists(TableName.valueOf("del_tb", "test"))); + assertTrue(admin.tableExists(TableName.valueOf("test_del_tb"))); + IOException thrown = assertThrows(IOException.class, + () -> { + admin.deleteTable(TableName.valueOf("tablegroup_not_exists")); + }); + Assert.assertTrue(thrown.getCause() instanceof ObTableException); + Assert.assertEquals(ResultCodes.OB_TABLEGROUP_NOT_EXIST.errorCode, ((ObTableException) thrown.getCause()).getErrorCode()); + admin.deleteTable(TableName.valueOf("del_tb", "test")); + admin.deleteTable(TableName.valueOf("test_del_tb")); + assertFalse(admin.tableExists(TableName.valueOf("del_tb", "test"))); + assertFalse(admin.tableExists(TableName.valueOf("test_del_tb"))); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } finally { + deleteTableIfExists(admin, "test_del_tb"); + deleteTableIfExists(admin, "del_tb"); + } } @Test @@ -1305,51 +1327,62 @@ public void testCreateTableInjectError() throws Exception { htd.addFamily(hcd1); htd.addFamily(hcd2); htd.addFamily(hcd3); - - // 1. open err EN_CREATE_HTABLE_TG_FINISH_ERR - setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_TG_FINISH_ERR, true); - ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.createTable(htd)); - assertFalse("Table should not exist after TG error injection", - admin.tableExists(TableName.valueOf(tableName))); - setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_TG_FINISH_ERR, false); - - // 2. open err EN_CREATE_HTABLE_CF_FINISH_ERR - setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_CF_FINISH_ERR, true); - ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.createTable(htd)); - assertFalse("Table should not exist after CF error injection", - admin.tableExists(TableName.valueOf(tableName))); - setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_CF_FINISH_ERR, false); - - // 3. create table without error - admin.createTable(htd); - assertTrue("Table should exist after normal creation", - admin.tableExists(TableName.valueOf(tableName))); - assertEquals("Table should have 3 column families", 3, + + try { + // 1. open err EN_CREATE_HTABLE_TG_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_TG_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.createTable(htd)); + assertFalse("Table should not exist after TG error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_TG_FINISH_ERR, false); + + // 2. open err EN_CREATE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_CF_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.createTable(htd)); + assertFalse("Table should not exist after CF error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_CREATE_HTABLE_CF_FINISH_ERR, false); + + // 3. create table without error + admin.createTable(htd); + assertTrue("Table should exist after normal creation", + admin.tableExists(TableName.valueOf(tableName))); + assertEquals("Table should have 3 column families", 3, admin.getTableDescriptor(TableName.valueOf(tableName)).getFamilies().size()); - - // 4. open err EN_DISABLE_HTABLE_CF_FINISH_ERR - setErrSimPoint(ErrSimPoint.EN_DISABLE_HTABLE_CF_FINISH_ERR, true); - admin.disableTable(TableName.valueOf(tableName)); - assertFalse("Table should not be disabled after disable error injection", - admin.isTableDisabled(TableName.valueOf(tableName))); - setErrSimPoint(ErrSimPoint.EN_DISABLE_HTABLE_CF_FINISH_ERR, false); - - // 5. disable table without error - admin.disableTable(TableName.valueOf(tableName)); - assertTrue("Table should be disabled after normal disable", - admin.isTableDisabled(TableName.valueOf(tableName))); - - // 6. open err EN_DELETE_HTABLE_CF_FINISH_ERR - setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, true); - admin.deleteTable(TableName.valueOf(tableName)); - assertTrue("Table should still exist after delete error injection", - admin.tableExists(TableName.valueOf(tableName))); - setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, false); - - // 7. delete table without error - admin.deleteTable(TableName.valueOf(tableName)); - assertFalse("Table should not exist after normal delete", - admin.tableExists(TableName.valueOf(tableName))); + + // 4. open err EN_DISABLE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_DISABLE_HTABLE_CF_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.disableTable(TableName.valueOf(tableName))); + assertFalse("Table should not be disabled after disable error injection", + admin.isTableDisabled(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_DISABLE_HTABLE_CF_FINISH_ERR, false); + + // 5. disable table without error + admin.disableTable(TableName.valueOf(tableName)); + assertTrue("Table should be disabled after normal disable", + admin.isTableDisabled(TableName.valueOf(tableName))); + + // 6. open err EN_DELETE_HTABLE_CF_FINISH_ERR + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, true); + ObHTableTestUtil.executeIgnoreUnexpectedError(() -> admin.deleteTable(TableName.valueOf(tableName))); + assertTrue("Table should still exist after delete error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, false); + + // 7. delete table without error + admin.deleteTable(TableName.valueOf(tableName)); + assertFalse("Table should not exist after normal delete", + admin.tableExists(TableName.valueOf(tableName))); + } catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } finally { + if (admin.tableExists(TableName.valueOf(tableName))) { + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_CF_FINISH_ERR, false); + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_SKIP_CF_ERR, false); + admin.deleteTable(TableName.valueOf(tableName)); + } + } } @Test @@ -1518,4 +1551,151 @@ public void testHbaseDDLException() throws Exception { } } + + // Test cases for abnormal scene in CreateTableGroupHelper/DropTableGroupHelper + @Test + public void testCreateDropTableGroup() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + java.sql.Connection sysConn = ObHTableTestUtil.getSysConnection(); + String tenantName = "mysql"; + + byte[] tableName = Bytes.toBytes("test_create_drop_tg_helper"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + byte[] cf3 = Bytes.toBytes("cf3"); + + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + HColumnDescriptor hcd3 = new HColumnDescriptor(cf3); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + htd.addFamily(hcd3); + + try { + admin.createTable(htd); + + // 1. open err EN_DELETE_HTABLE_SKIP_CF_ERR, will skip delete cf table when delete hbase table + // and the subsequent delete htable operations will return OB_TABLEGROUP_NOT_EMPTY + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_SKIP_CF_ERR, true); + ObHTableTestUtil.executeIgnoreExpectedErrors(() -> admin.deleteTable(TableName.valueOf(tableName)), "OB_TABLEGROUP_NOT_EMPTY"); + assertTrue("Table should still exist after delete error injection", + admin.tableExists(TableName.valueOf(tableName))); + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_SKIP_CF_ERR, false); + + // 2. create a database and set default tablegroup to test_create_drop_tg_helper, + // and the subsequent delete htable operation will return OB_TABLEGROUP_NOT_EMPTY + executeSQL(conn, "create database db_test_create_drop_tg_helper default tablegroup test_create_drop_tg_helper", true); + ObHTableTestUtil.executeIgnoreExpectedErrors(() -> admin.deleteTable(TableName.valueOf(tableName)), "OB_TABLEGROUP_NOT_EMPTY"); + executeSQL(conn, "drop database db_test_create_drop_tg_helper", true); + + // 3. set tenant's default tablegroup to test_create_drop_tg_helper, + // and the subsequent delete htable operation will return OB_TABLEGROUP_NOT_EMPTY + executeSQL(sysConn, String.format("alter tenant %s set default tablegroup = test_create_drop_tg_helper", tenantName), true); + ObHTableTestUtil.executeIgnoreExpectedErrors(() -> admin.deleteTable(TableName.valueOf(tableName)), "OB_TABLEGROUP_NOT_EMPTY"); + executeSQL(sysConn, String.format("alter tenant %s set default tablegroup = null", tenantName), true); + + } catch (Exception e) { + e.printStackTrace(); + throw e; + } finally { + executeSQL(conn, "drop database if exists db_test_create_drop_tg_helper", true); + executeSQL(sysConn, String.format("alter tenant %s set default tablegroup = null", tenantName), true); + if (admin.tableExists(TableName.valueOf(tableName))) { + setErrSimPoint(ErrSimPoint.EN_DELETE_HTABLE_SKIP_CF_ERR, false); + admin.deleteTable(TableName.valueOf(tableName)); + } + } + } + + private void checkDDLStmtStr(List> ddlStmts) throws Exception { + java.sql.Connection conn = ObHTableTestUtil.getConnection(); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery( + String.format("select operation_type, ddl_stmt_str from oceanbase.__all_ddl_operation order by gmt_modified desc limit %d", ddlStmts.size() + 2)); + Assert.assertTrue(resultSet.next()); + int operationType = resultSet.getInt("operation_type"); + Assert.assertEquals(1503, operationType); + String ddlStmtStr = resultSet.getString("ddl_stmt_str"); + Assert.assertEquals("", ddlStmtStr); + + for (int i = 0; i < ddlStmts.size(); i++) { + Assert.assertTrue(resultSet.next()); + operationType = resultSet.getInt("operation_type"); + Assert.assertEquals(ddlStmts.get(i).getKey().intValue(), operationType); + ddlStmtStr = resultSet.getString("ddl_stmt_str"); + Assert.assertEquals(ddlStmts.get(i).getValue(), ddlStmtStr); + } + Assert.assertTrue(resultSet.next()); + operationType = resultSet.getInt("operation_type"); + Assert.assertEquals(1503, operationType); + ddlStmtStr = resultSet.getString("ddl_stmt_str"); + Assert.assertEquals("", ddlStmtStr); + Assert.assertFalse(resultSet.next()); + } + + @Test + public void testDDLStmtStr() throws Exception { + Configuration conf = ObHTableTestUtil.newConfiguration(); + Connection connection = ConnectionFactory.createConnection(conf); + Admin admin = connection.getAdmin(); + + byte[] tableName = Bytes.toBytes("test_ddl_stmt_str"); + byte[] cf1 = Bytes.toBytes("cf1"); + byte[] cf2 = Bytes.toBytes("cf2"); + + HColumnDescriptor hcd1 = new HColumnDescriptor(cf1); + HColumnDescriptor hcd2 = new HColumnDescriptor(cf2); + + HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); + htd.addFamily(hcd1); + htd.addFamily(hcd2); + + try { + // 1. create hbase table + admin.createTable(htd); + List> expStmtStrs = new ArrayList<>(); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(4, "CREATE TABLE `test_ddl_stmt_str$cf1` (K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, T bigint NOT NULL, V varbinary(1048576) NOT NULL, " + + "PRIMARY KEY (K, Q, T)) TABLEGROUP = `test_ddl_stmt_str` kv_attributes = '{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\"}}'")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(4, "CREATE TABLE `test_ddl_stmt_str$cf2` (K varbinary(1024) NOT NULL, Q varbinary(256) NOT NULL, T bigint NOT NULL, V varbinary(1048576) NOT NULL, " + + "PRIMARY KEY (K, Q, T)) TABLEGROUP = `test_ddl_stmt_str` kv_attributes = '{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\"}}'")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(302, "CREATE TABLEGROUP `test_ddl_stmt_str`")); + checkDDLStmtStr(expStmtStrs); + + // 2. disable hbase table + admin.disableTable(TableName.valueOf(tableName)); + expStmtStrs.clear(); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(3, "ALTER TABLE test_ddl_stmt_str$cf1 KV_ATTRIBUTES='{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"disable\"}}'")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(3, "ALTER TABLE test_ddl_stmt_str$cf2 KV_ATTRIBUTES='{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"disable\"}}'")); + checkDDLStmtStr(expStmtStrs); + + // 3. enable hbase table + admin.enableTable(TableName.valueOf(tableName)); + expStmtStrs.clear(); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(3, "ALTER TABLE test_ddl_stmt_str$cf1 KV_ATTRIBUTES='{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"enable\"}}'")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(3, "ALTER TABLE test_ddl_stmt_str$cf2 KV_ATTRIBUTES='{\"Hbase\": {\"MaxVersions\": 1, \"CreatedBy\": \"Admin\", \"State\": \"enable\"}}'")); + checkDDLStmtStr(expStmtStrs); + + // 4. delete hbase table + admin.disableTable(TableName.valueOf(tableName)); + admin.deleteTable(TableName.valueOf(tableName)); + expStmtStrs.clear(); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(303, "DROP TABLEGROUP `test_ddl_stmt_str`")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(2, "DROP TABLE `test`.`test_ddl_stmt_str$cf1`")); + expStmtStrs.add(new AbstractMap.SimpleEntry<>(2, "DROP TABLE `test`.`test_ddl_stmt_str$cf2`")); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } finally { + if (admin.tableExists(TableName.valueOf(tableName))) { + if (admin.isTableEnabled(TableName.valueOf(tableName))) { + admin.disableTable(TableName.valueOf(tableName)); + } + admin.deleteTable(TableName.valueOf(tableName)); + } + } + } } From fd0931daf88a57b794ce3b3d80203837856a7f99 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Tue, 24 Jun 2025 11:01:00 +0800 Subject: [PATCH 36/36] fix add case for secondary partition region locator --- .../oceanbase/hbase/OHConnectionTest.java | 33 ++++++++++++++ src/test/java/unit_test_db.sql | 43 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index 3e4046ce..9473efd3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -1050,6 +1050,39 @@ public void testKeyPartitionWithRegionLocator() throws IOException { Assert.assertEquals(startKeys[0], endKeys[0]); Assert.assertEquals(startKeys[0], HConstants.EMPTY_BYTE_ARRAY); } + + @Test + public void testRangeKeySecondaryPartitionWithRegionLocator() throws IOException { + final String tableNameStr = "test_secondary_range_key"; + final TableName tableName = TableName.valueOf(tableNameStr); + final Configuration conf = ObHTableTestUtil.newConfiguration(); + connection = ConnectionFactory.createConnection(conf); + hTable = connection.getTable(tableName); + RegionLocator locator = connection.getRegionLocator(tableName); + byte[][] startKeys = locator.getStartKeys(); + byte[][] endKeys = locator.getEndKeys(); + Assert.assertEquals("Should have 1 region", 1, startKeys.length); + Assert.assertEquals("Should have 1 region", 1, endKeys.length); + Assert.assertEquals(startKeys[0], endKeys[0]); + Assert.assertEquals(startKeys[0], HConstants.EMPTY_BYTE_ARRAY); + } + + @Test + public void testKeyRangeSecondaryPartitionWithRegionLocator() throws IOException { + final String tableNameStr = "test_secondary_key_range"; + final TableName tableName = TableName.valueOf(tableNameStr); + final Configuration conf = ObHTableTestUtil.newConfiguration(); + connection = ConnectionFactory.createConnection(conf); + hTable = connection.getTable(tableName); + RegionLocator locator = connection.getRegionLocator(tableName); + byte[][] startKeys = locator.getStartKeys(); + byte[][] endKeys = locator.getEndKeys(); + Assert.assertEquals("Should have 1 region", 1, startKeys.length); + Assert.assertEquals("Should have 1 region", 1, endKeys.length); + Assert.assertEquals(startKeys[0], endKeys[0]); + Assert.assertEquals(startKeys[0], HConstants.EMPTY_BYTE_ARRAY); + } + @Test public void testBufferedMutatorPeriodicFlush() throws Exception { diff --git a/src/test/java/unit_test_db.sql b/src/test/java/unit_test_db.sql index cc7ddfec..3fd00d75 100644 --- a/src/test/java/unit_test_db.sql +++ b/src/test/java/unit_test_db.sql @@ -339,3 +339,46 @@ PARTITION BY RANGE COLUMNS(K) ( PARTITION p9 VALUES LESS THAN ('v'), PARTITION p10 VALUES LESS THAN (MAXVALUE) ); + +CREATE TABLEGROUP test_secondary_key_range SHARDING = 'ADAPTIVE'; +CREATE TABLE IF NOT EXISTS `test_secondary_key_range$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + `G` bigint(20) GENERATED ALWAYS AS (ABS(T)), + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_secondary_key_range PARTITION BY KEY(`K`) PARTITIONS 3 +SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE ( + SUBPARTITION `p1` VALUES LESS THAN (100), + SUBPARTITION `p2` VALUES LESS THAN (200), + SUBPARTITION `p3` VALUES LESS THAN MAXVALUE +); +CREATE TABLE IF NOT EXISTS `test_secondary_key_range$family2` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + `G` bigint(20) GENERATED ALWAYS AS (ABS(T)), + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_secondary_key_range PARTITION BY KEY(`K`) PARTITIONS 3 +SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE ( + SUBPARTITION `p1` VALUES LESS THAN (100), + SUBPARTITION `p2` VALUES LESS THAN (200), + SUBPARTITION `p3` VALUES LESS THAN MAXVALUE +); + +CREATE TABLEGROUP test_secondary_range_key SHARDING = 'ADAPTIVE'; +CREATE TABLE IF NOT EXISTS `test_secondary_range_key$family1` ( + `K` varbinary(1024) NOT NULL, + `Q` varbinary(256) NOT NULL, + `T` bigint(20) NOT NULL, + `V` varbinary(1024) DEFAULT NULL, + `G` bigint(20) GENERATED ALWAYS AS (ABS(T)), + PRIMARY KEY (`K`, `Q`, `T`) +) TABLEGROUP = test_secondary_range_key PARTITION BY RANGE COLUMNS(`G`) +SUBPARTITION BY KEY(`K`) SUBPARTITIONS 3 +( PARTITION `p1` VALUES LESS THAN (100), + PARTITION `p2` VALUES LESS THAN (200), + PARTITION `p3` VALUES LESS THAN MAXVALUE +);