From 6e6d0f712e9531b4a32e820647bca7d3c1033526 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Sat, 8 Mar 2025 18:08:44 +0800 Subject: [PATCH 1/3] refactor secondart test --- .../hbase/OHTableSecondaryPartMcfTest.java | 314 +++-------- .../hbase/OHTableSecondaryPartTest.java | 494 +++++++++--------- .../hbase/util/ObHTableTestUtil.java | 24 + .../hbase/util/TableTemplateManager.java | 259 +++++++++ .../oceanbase/hbase/util/TimeGenerator.java | 32 ++ 5 files changed, 622 insertions(+), 501 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java index 235b06a6..71cda1c1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java @@ -18,204 +18,67 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import com.alipay.oceanbase.hbase.util.TimeGenerator; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.util.Bytes; import org.junit.*; import java.sql.Connection; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; import static org.apache.hadoop.hbase.util.Bytes.toBytes; public class OHTableSecondaryPartMcfTest { - private static String tableGroupNames[] = { - "test_table_group1", - "test_table_group2", - "test_table_group3", - "test_table_group4", - }; - private static String tableNames[][] = new String[][] { - {tableGroupNames[0] + "$cf1", tableGroupNames[0] + "$cf2"}, - {tableGroupNames[1] + "$cf1", tableGroupNames[1] + "$cf2"}, - {tableGroupNames[2] + "$cf1", tableGroupNames[2] + "$cf2"}, - {tableGroupNames[3] + "$cf1", tableGroupNames[3] + "$cf2"} - }; - private static String createTableStmts[][] = { - {"CREATE TABLE IF NOT EXISTS `" + tableNames[0][0] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[0] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");", - "CREATE TABLE IF NOT EXISTS `" + tableNames[0][1] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[0] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");" - }, - {"CREATE TABLE IF NOT EXISTS `" + tableNames[1][0] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[1] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");", - "CREATE TABLE IF NOT EXISTS `" + tableNames[1][1] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[1] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");" - - }, - {"CREATE TABLE IF NOT EXISTS `" + tableNames[2][0] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[2] +" PARTITION BY KEY(`K`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;", - "CREATE TABLE IF NOT EXISTS `" + tableNames[2][1] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[2] +" PARTITION BY KEY(`K`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;" - - }, - {"CREATE TABLE IF NOT EXISTS `" + tableNames[3][0] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[3] +" PARTITION BY KEY(`K_PREFIX`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;", - "CREATE TABLE IF NOT EXISTS `" + tableNames[3][1] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[3] +" PARTITION BY KEY(`K_PREFIX`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;" - - } - }; - - public static void createTableGroups() throws Exception { - System.out.print("create table group start"); - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableGroupNames.length; i++) { - System.out.print("."); - String stmt = "CREATE TABLEGROUP IF NOT EXISTS `"+ tableGroupNames[i] +"` SHARDING = 'ADAPTIVE';"; - conn.createStatement().execute(stmt); - } - System.out.println("end"); - } - - public static void dropTableGroups() throws Exception { - System.out.print("drop table group start"); - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableGroupNames.length; i++) { - System.out.print("."); - String stmt = "DROP TABLEGROUP IF EXISTS " + tableGroupNames[i] + ";"; - conn.createStatement().execute(stmt); - } - System.out.println("end"); - } + private static Map> tableNames = new LinkedHashMap<>(); public static void dropTables() throws Exception { - System.out.print("drop table start"); Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableNames.length; i++) { - for (int j = 0; j < tableNames[i].length; j++) { - System.out.print("."); - String stmt = "DROP TABLE IF EXISTS " + tableNames[i][j] + ";"; + for (Map.Entry> entry : tableNames.entrySet()) { + for (String tableName : entry.getValue()) { + String stmt = "DROP TABLE IF EXISTS " + tableName + ";"; conn.createStatement().execute(stmt); + System.out.println("============= drop table " + tableName + " done ============="); } + String stmt = "DROP TABLEGROUP IF EXISTS " + entry.getKey() + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= drop tablegroup " + entry.getKey() + " done ============="); } - System.out.println("end"); } - - public static void createTables() throws Exception { - System.out.print("create table start"); + public static void createMultiCFTables(TableTemplateManager.TableType type, boolean printSql) throws Exception { Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < createTableStmts.length; i++) { - for (int j = 0; j < createTableStmts[i].length; j++) { - System.out.print("."); - conn.createStatement().execute(createTableStmts[i][j]); - } + TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); + String tableGroup = TableTemplateManager.getTableGroupName(type); + String tableGroupSql = TableTemplateManager.generateTableGroupSQL(tableGroup); + conn.createStatement().execute(tableGroupSql); + tableNames.put(tableGroup, new LinkedList<>()); + for (int i = 1; i <= 3; ++i) { + String tableName = TableTemplateManager.generateTableName(tableGroup, true, i); + String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); + conn.createStatement().execute(sql); + tableNames.get(tableGroup).add(tableName); + System.out.println("============= create table: " + tableName + + " table_group: " + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + " \n============= done =============\n"); } - System.out.println("end"); } + public static void truncateTables() throws Exception { - System.out.print("truncate table start"); Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableNames.length; i++) { - for (int j = 0; j < tableNames[i].length; j++) { - System.out.print("."); - String stmt = "TRUNCATE TABLE " + tableNames[i][j] + ";"; + for (Map.Entry> entry : tableNames.entrySet()) { + for (String tableName : entry.getValue()) { + String stmt = "TRUNCATE TABLE " + tableName + ";"; conn.createStatement().execute(stmt); + System.out.println("============= truncate table " + tableName + " done ============="); } } - System.out.println("end"); } public static String getTableName(String input) throws Exception { @@ -248,14 +111,14 @@ public static String getColumnFamilyName(String input) throws Exception { @BeforeClass public static void before() throws Exception { - createTableGroups(); - createTables(); + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createMultiCFTables(type, true); + } } @AfterClass public static void finish() throws Exception { dropTables(); - dropTableGroups(); } @Before @@ -263,93 +126,54 @@ public void prepareCase() throws Exception { truncateTables(); } - @Test - public void testPut() throws Exception { + public static void testMultiCFPut(Map.Entry> entry) throws Exception { String key = "putKey"; String value = "value"; String column = "putColumn"; long timestamp = System.currentTimeMillis(); - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); - hTable.init(); - Put put = new Put(toBytes(key)); - for (int j = 0; j < tableNames[i].length; j++) { - String family = getColumnFamilyName(tableNames[i][j]); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - } - hTable.put(put); - hTable.close(); - } - } - - @Test - public void testIncrement() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); - hTable.init(); - - String key = "Key"; - String column = "Column"; - Increment increment = new Increment(key.getBytes()); - for (int j = 0; j < tableNames[i].length; j++) { - String family = getColumnFamilyName(tableNames[i][j]); - increment.addColumn(family.getBytes(), column.getBytes(), 1L); - } - hTable.increment(increment); - hTable.close(); - } - } - - @Test - public void testAppend() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); - hTable.init(); - - String key = "Key"; - String column = "Column"; - String value = "app"; - Append append = new Append(key.getBytes()); - for (int j = 0; j < tableNames[i].length; j++) { - String family = getColumnFamilyName(tableNames[i][j]); - KeyValue kv1 = new KeyValue(key.getBytes(), family.getBytes(), column.getBytes(), value.getBytes()); - append.add(kv1); - } - hTable.append(append); - - hTable.close(); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); } + hTable.put(put); + hTable.close(); } + + public static void testMultiCFGet(Map.Entry> entry) throws Exception { + String key = "putKey"; + String column = "putColumn"; + + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + String value = family + "_value"; + long timestamp = System.currentTimeMillis(); - @Test - public void testGet() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); - - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - long timestamp = System.currentTimeMillis(); Put put = new Put(toBytes(key)); - for (int j = 0; j < tableNames[i].length; j++) { - String family = getColumnFamilyName(tableNames[i][j]); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - } + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); hTable.put(put); Get get = new Get(key.getBytes()); - get.setMaxVersions(Integer.MAX_VALUE); - for (int j = 0; j < tableNames[i].length; j++) { - String family = getColumnFamilyName(tableNames[i][j]); - get.addColumn(family.getBytes(), column.getBytes()); - } + get.addFamily(family.getBytes()); Result r = hTable.get(get); Assert.assertEquals(1, r.raw().length); hTable.close(); } } - + + @Test + public void testPut() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartMcfTest::testMultiCFPut); + } + + @Test + public void testGet() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartMcfTest::testMultiCFGet); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java index 235e4375..be39b71b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java @@ -18,6 +18,8 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import com.alipay.oceanbase.hbase.util.TimeGenerator; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; @@ -31,92 +33,45 @@ import java.util.LinkedList; import java.util.List; -import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.getConnection; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.*; import static org.apache.hadoop.hbase.util.Bytes.toBytes; -import static org.junit.Assert.*; import static org.junit.Assert.assertEquals; public class OHTableSecondaryPartTest { - private static String tableNames[] = new String[] {"test$cf1", "test$cf2", "test$cf3", "test$cf4"}; - private static String createTableStmt1 = "CREATE TABLE IF NOT EXISTS `" + tableNames[0] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");"; - private static String createTableStmt2 = "CREATE TABLE IF NOT EXISTS `" + tableNames[1] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");"; - private static String createTableStmt3 = "CREATE TABLE IF NOT EXISTS `" + tableNames[2] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") PARTITION BY KEY(`K`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;"; - private static String createTableStmt4 = "CREATE TABLE IF NOT EXISTS `" + tableNames[3] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") PARTITION BY KEY(`K_PREFIX`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;"; - private static String createTableStmts[] = new String[] {createTableStmt1, createTableStmt2, createTableStmt3, createTableStmt4}; + private static List tableNames = new LinkedList(); public static void dropTables() throws Exception { Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableNames.length; i++) { - String stmt = "DROP TABLE IF EXISTS " + tableNames[i] + ";"; + for (int i = 0; i < tableNames.size(); i++) { + String stmt = "DROP TABLE IF EXISTS " + tableNames.get(i) + ";"; conn.createStatement().execute(stmt); - System.out.println("drop table " + tableNames[i] + " done"); + System.out.println("============= drop table " + tableNames.get(i) + " done ============="); + } + for (int i = 0; i < tableNames.size(); i++) { + String stmt = "DROP TABLEGROUP IF EXISTS " + getTableName(tableNames.get(i)) + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= drop tablegroup " + tableNames.get(i) + " done ============="); } } - - public static void createTables() throws Exception { + public static void createTables(TableTemplateManager.TableType type, boolean printSql) throws Exception { Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < createTableStmts.length; i++) { - conn.createStatement().execute(createTableStmts[i]); - System.out.println("create table " + tableNames[i] + " done"); - } + TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); + String tableGroup = TableTemplateManager.getTableGroupName(type); + String tableGroupSql = TableTemplateManager.generateTableGroupSQL(tableGroup); + conn.createStatement().execute(tableGroupSql); + String tableName = TableTemplateManager.generateTableName(tableGroup, false, 1); + String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); + conn.createStatement().execute(sql); + tableNames.add(tableName); + System.out.println("============= create table: " + tableName + " table_group: " + getTableName(tableName) + " =============\n" + (printSql ? sql : "") + " \n============= done =============\n"); } public static void truncateTables() throws Exception { Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableNames.length; i++) { - String stmt = "TRUNCATE TABLE " + tableNames[i] + ";"; + for (int i = 0; i < tableNames.size(); i++) { + String stmt = "TRUNCATE TABLE " + tableNames.get(i) + ";"; conn.createStatement().execute(stmt); - System.out.println("truncate table " + tableNames[i] + " done"); + System.out.println("============= truncate table " + tableNames.get(i) + " done ============="); } } @@ -148,7 +103,9 @@ public static String getColumnFamilyName(String input) throws Exception { @BeforeClass public static void before() throws Exception { - createTables(); + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, true); + } } @AfterClass @@ -161,211 +118,236 @@ public void prepareCase() throws Exception { truncateTables(); } + public static void testPut(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + + hTable.close(); + } + + public static void testGet(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), toBytes(value)); + hTable.put(put); + + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(1, cells.length); + assertEquals(column, Bytes.toString(CellUtil.cloneQualifier(cells[0]))); + assertEquals("value", Bytes.toString(CellUtil.cloneValue(cells[0]))); + System.out.println("get table " + tableName + " done"); + + hTable.close(); + } + + public static void testIncrement(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); + hTable.put(put); + + Increment increment = new Increment(key.getBytes()); + increment.addColumn(family.getBytes(), column1.getBytes(), 1L); + increment.addColumn(family.getBytes(), column2.getBytes(), 1L); + hTable.increment(increment); + + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(2, cells.length); + for (Cell cell : cells) { + if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column1)) { + assertEquals("2", Bytes.toString(CellUtil.cloneValue(cell))); + } else if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column2)) { + assertEquals("2", Bytes.toString(CellUtil.cloneValue(cell))); + } + } + hTable.close(); + } + + public static void testAppend(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + String value = "value"; + Append append = new Append(key.getBytes()); + KeyValue kv1 = new KeyValue(key.getBytes(), family.getBytes(), column1.getBytes(), value.getBytes()); + KeyValue kv2 = new KeyValue(key.getBytes(), family.getBytes(), column2.getBytes(), value.getBytes()); + append.add(kv1); + append.add(kv2); + hTable.append(append); + + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(2, cells.length); + + for (Cell cell : cells) { + if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column1)) { + assertEquals("valuevalue", Bytes.toString(CellUtil.cloneValue(cell))); + } else if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column2)) { + assertEquals("valuevalue", Bytes.toString(CellUtil.cloneValue(cell))); + } + } + hTable.close(); + } + + public static void testCheckAndMutate(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + String newValue = "newValue"; + RowMutations mutations = new RowMutations(key.getBytes()); + Put put = new Put(key.getBytes()); + long timestamp = System.currentTimeMillis(); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + + Put newPut = new Put(key.getBytes()); + newPut.add(family.getBytes(), column.getBytes(), timestamp, toBytes(newValue)); + mutations.add(newPut); + hTable.checkAndMutate(key.getBytes(), family.getBytes(), + column.getBytes(), CompareFilter.CompareOp.EQUAL, value.getBytes(), + mutations); + + hTable.close(); + } + + public static void testBatchGet(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); + hTable.put(put); + + List gets = new ArrayList<>(); + Get get1 = new Get(key.getBytes()); + get1.addFamily(family.getBytes()); + gets.add(get1); + + Get get2 = new Get(key.getBytes()); + get2.addColumn(family.getBytes(), column1.getBytes()); + gets.add(get2); + + Get get3 = new Get(key.getBytes()); + get3.addColumn(family.getBytes(), column2.getBytes()); + gets.add(get3); + + + Result[] results = hTable.get(gets); + for (Result result : results) { + for (Cell cell : result.listCells()) { + String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); + String V = Bytes.toString(CellUtil.cloneValue(cell)); + System.out.println("Column: " + Q + ", Value: " + V); + } + } + } + + public static void testScan(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); + hTable.put(put); + + Scan scan = new Scan(key.getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + assertEquals(column1, Bytes.toString(keyValue.getQualifier())); + assertEquals("1", Bytes.toString(keyValue.getValue())); + count++; + } + } + assertEquals(1, count); + } + + @Test public void testPut() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("put table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - System.out.println("put table " + tableNames[i] + " done"); - - hTable.close(); - } + FOR_EACH(tableNames, OHTableSecondaryPartTest::testPut); } @Test - public void testIncrement() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("increment table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String key = "Key"; - String column1 = "Column1"; - String column2 = "Column2"; - Increment increment = new Increment(key.getBytes()); - increment.addColumn(family.getBytes(), column1.getBytes(), 1L); - increment.addColumn(family.getBytes(), column2.getBytes(), 1L); - hTable.increment(increment); - System.out.println("increment table " + tableNames[i] + " done"); - - hTable.close(); - } + public void testGet() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartTest::testGet); } @Test - public void testAppend() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("append table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String key = "Key"; - String column1 = "Column1"; - String column2 = "Column2"; - String value = "app"; - Append append = new Append(key.getBytes()); - KeyValue kv1 = new KeyValue(key.getBytes(), family.getBytes(), column1.getBytes(), value.getBytes()); - KeyValue kv2 = new KeyValue(key.getBytes(), family.getBytes(), column2.getBytes(), value.getBytes()); - append.add(kv1); - append.add(kv2); - hTable.append(append); - System.out.println("append table " + tableNames[i] + " done"); - - hTable.close(); - } + public void testIncrement() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartTest::testIncrement); } @Test - public void testCheckAndMutate() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("checkAndMutate table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - String newValue = "newValue"; - RowMutations mutations = new RowMutations(key.getBytes()); - Put put = new Put(key.getBytes()); - long timestamp = System.currentTimeMillis(); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - - Put newPut = new Put(key.getBytes()); - newPut.add(family.getBytes(), column.getBytes(), timestamp, toBytes(newValue)); - mutations.add(newPut); - hTable.checkAndMutate(key.getBytes(), family.getBytes(), - column.getBytes(), CompareFilter.CompareOp.EQUAL, value.getBytes(), - mutations); - System.out.println("checkAndMutate table " + tableNames[i] + " done"); - - hTable.close(); - } + public void testAppend() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartTest::testAppend); } @Test - public void testGet() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("put table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - System.out.println("put table " + tableNames[i] + " done"); - - System.out.println("get table " + tableNames[i] + " begin"); - Get get = new Get(key.getBytes()); - get.setMaxVersions(Integer.MAX_VALUE); - get.addColumn(family.getBytes(), column.getBytes()); - Result r = hTable.get(get); - Assert.assertEquals(1, r.raw().length); - System.out.println("get table " + tableNames[i] + " begin"); - - hTable.close(); - } + public void testCheckAndMutate() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartTest::testCheckAndMutate); } - + @Test public void testBatchGet() throws Exception { - long batchSize = 10; - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("put table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String column = "putColumn"; - String value = "value"; - long timestamp = System.currentTimeMillis(); - for (int j = 0; j < batchSize; j++) { - String key = "putKey" + j; - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - } - System.out.println("put table " + tableNames[i] + " done"); - - System.out.println("get table " + tableNames[i] + " begin"); - List gets = new ArrayList<>(); - for (int j = 0; j < batchSize; j++) { - String key = "putKey" + j; - Get get = new Get(key.getBytes()); - get.setMaxVersions(Integer.MAX_VALUE); - get.addColumn(family.getBytes(), column.getBytes()); - gets.add(get); - } - Result[] results = hTable.get(gets); - for (Result result : results) { - for (Cell cell : result.listCells()) { - String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); - String V = Bytes.toString(CellUtil.cloneValue(cell)); - System.out.println("Column: " + Q + ", Value: " + V); - } - } - System.out.println("get table " + tableNames[i] + " begin"); - - hTable.close(); - } + FOR_EACH(tableNames, OHTableSecondaryPartTest::testBatchGet); } @Test public void testScan() throws Exception { - long rowSize = 10; - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("put table " + tableNames[i] + " begin"); - String key = "putKey"; - String family = getColumnFamilyName(tableNames[i]); - String column = "Column"; - String value = "value"; - long timestamp = System.currentTimeMillis(); - for (int j = 0; j < rowSize; j++) { - Put put = new Put(toBytes(key)); - String qualify = column + j; - put.add(family.getBytes(), qualify.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - } - System.out.println("put table " + tableNames[i] + " done"); - - System.out.println("scan table " + tableNames[i] + " begin"); - Scan scan = new Scan(toBytes(key)); - scan.addFamily(family.getBytes()); - ResultScanner scanner = hTable.getScanner(scan); - int count = 0; - for (Result result : scanner) { - for (KeyValue keyValue : result.raw()) { - assertEquals(column + count, Bytes.toString(keyValue.getQualifier())); - assertEquals(value, Bytes.toString(keyValue.getValue())); - count++; - } - } - assertEquals(rowSize, count); - System.out.println("scan table " + tableNames[i] + " end"); - - hTable.close(); - } + FOR_EACH(tableNames, OHTableSecondaryPartTest::testScan); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index 44583f74..584bdb39 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -156,4 +156,28 @@ static public Connection getConnection() { throw new RuntimeException(e); } } + + @FunctionalInterface + public interface CheckedConsumer { + void accept(T t) throws Exception; + default CheckedConsumer andThen(CheckedConsumer after) throws Exception { + if (after == null) { throw new NullPointerException();} + return (T t) -> { + accept(t); + after.accept(t); + }; + } + } + + public static void FOR_EACH(List tableNames, CheckedConsumer consumer) throws Exception { + for (String tableName : tableNames) { + consumer.accept(tableName); + } + } + + public static void FOR_EACH(Map> group2Tables, CheckedConsumer>> consumer) throws Exception { + for (Map.Entry> entry : group2Tables.entrySet()) { + consumer.accept(entry); + } + } } \ No newline at end of file diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java new file mode 100644 index 00000000..f0d90c20 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -0,0 +1,259 @@ +package com.alipay.oceanbase.hbase.util; + + +import java.util.EnumMap; +import java.util.Map; +public class TableTemplateManager { + public static final String TABLE_GROUP_PREFIX = "test_group_"; + public static final String COLUMN_FAMILY = "cf"; + public enum TableType { + NON_PARTITIONED_REGULAR, + NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_REGULAR, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_RANGE_KEY, // RANGE-KEY分区（使用K） + SECONDARY_PARTITIONED_RANGE_KEY_GEN, // RANGE-KEY分区（使用生成列） + SECONDARY_PARTITIONED_KEY_RANGE, // KEY-RANGE分区（使用K） + SECONDARY_PARTITIONED_KEY_RANGE_GEN, // KEY-RANGE分区（使用生成列） + SECONDARY_PARTITIONED_TIME_RANGE_KEY, // 时序表RANGE-KEY + SECONDARY_PARTITIONED_TIME_KEY_RANGE // 时序表KEY-RANGE + } + + private static final Map SQL_TEMPLATES = new EnumMap<>(TableType.class); + + static { + // 普通表非分区表模版 + SQL_TEMPLATES.put(TableType.NON_PARTITIONED_REGULAR, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s"); + // 时序表非分区表模版 + SQL_TEMPLATES.put(TableType.NON_PARTITIONED_TIME_SERIES, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s"); + // 普通表一级分区模板 + SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); + // 时序表一级分区模板 + SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_TIME_SERIES, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); + // 普通表RANGE-KEY分区（使用K） + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE)"); + + // 合并GEN类型的注释处理 + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 普通表KEY-RANGE分区（使用K） + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 普通表KEY-RANGE分区（使用生成列） + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 时序表RANGE-KEY分区 + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 时序表KEY-RANGE分区 + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_KEY_RANGE, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE)"); + } + + public static String getCreateTableSQL(TableType type, String tableName, + TimeGenerator.TimeRange timeRange) { + String template = SQL_TEMPLATES.get(type); + Object[] params; + String tableGroup = extractTableGroup(tableName); + + switch (type) { + case NON_PARTITIONED_REGULAR: + case NON_PARTITIONED_TIME_SERIES: + params = new Object[]{tableName, tableGroup}; + break; + case SINGLE_PARTITIONED_REGULAR: + case SINGLE_PARTITIONED_TIME_SERIES: // 合并相同处理逻辑 + params = new Object[]{tableName, tableGroup, 97}; + break; + case SECONDARY_PARTITIONED_RANGE_KEY: + case SECONDARY_PARTITIONED_RANGE_KEY_GEN: + case SECONDARY_PARTITIONED_KEY_RANGE: + case SECONDARY_PARTITIONED_KEY_RANGE_GEN: + boolean isGen = type.name().contains("GEN"); + params = new Object[]{ + tableName, + getGeneratedColumn(type), + tableGroup, + isGen ? "K_PREFIX" : "K", + 97, + timeRange.lowerBound1(), + timeRange.lowerBound1() + 86400000, + timeRange.lowerBound1() + 172800000 + }; + break; + case SECONDARY_PARTITIONED_TIME_RANGE_KEY: + case SECONDARY_PARTITIONED_TIME_KEY_RANGE: // 合并时序表处理 + params = new Object[]{ + tableName, + "", + tableGroup, + "K", + 97, + timeRange.lowerBound1(), + timeRange.lowerBound1() + 86400000, + timeRange.lowerBound1() + 172800000 + }; + break; + default: + throw new IllegalArgumentException("Unsupported table type"); + } + + return String.format(template, params); + } + + private static String getGeneratedColumn(TableType type) { + StringBuilder sb = new StringBuilder(); + boolean needsKPrefix = type.name().startsWith("SECONDARY_PARTITIONED") + && !type.name().contains("TIME") + && type.name().contains("GEN"); + + if (needsKPrefix) { + sb.append(",\n K_PREFIX varbinary(1024) GENERATED ALWAYS AS (substring(`K`, 1, 4))"); + } + return sb.toString(); + } + + private static String getPartitionStrategy(TableType type) { + if (type.name().contains("RANGE_KEY")) { + return type.name().contains("GEN") + ? "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 97" + : "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 97"; + } + if (type.name().contains("KEY_RANGE")) { + return type.name().contains("GEN") + ? "KEY(`K_PREFIX`) PARTITIONS 97 SUBPARTITION BY RANGE COLUMNS(`G`)" + : "KEY(`K`) PARTITIONS 97 SUBPARTITION BY RANGE COLUMNS(`G`)"; + } + return ""; + } + + public static String generateTableGroupSQL(String tableGroup) { + return String.format("CREATE TABLEGROUP IF NOT EXISTS %s SHARDING = 'ADAPTIVE'", tableGroup); + } + + public static String getTableGroupName(TableTemplateManager.TableType type) { + return TABLE_GROUP_PREFIX + type.name().toLowerCase(); + } + + public static String generateTableName(String tableGroup, boolean multiCf, int cfIndex) { + return String.format("%s$%s", + tableGroup, + multiCf ? COLUMN_FAMILY + cfIndex : COLUMN_FAMILY); + } + + public static String extractTableGroup(String tableName) { + int dollarIndex = tableName.indexOf('$'); + if (dollarIndex > 0) { + return tableName.substring(0, dollarIndex); + } + throw new IllegalArgumentException("Invalid table name: " + tableName); + } +} \ No newline at end of file diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java b/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java new file mode 100644 index 00000000..00c6aa46 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java @@ -0,0 +1,32 @@ +package com.alipay.oceanbase.hbase.util; + +public class TimeGenerator { + public static class TimeRange { + private final long lowerBound1; + private final long lowerBound2; + private final long upperBound1; + private final long upperBound2; + + public TimeRange(long lowerBound1, long lowerBound2, long upperBound1, long upperBound2) { + this.lowerBound1 = lowerBound1; + this.lowerBound2 = lowerBound2; + this.upperBound1 = upperBound1; + this.upperBound2 = upperBound2; + } + + public long lowerBound1() { return lowerBound1; } + public long lowerBound2() { return lowerBound2; } + public long upperBound1() { return upperBound1; } + public long upperBound2() { return upperBound2; } + } + + public static TimeRange generateTestTimeRange() { + long current = System.currentTimeMillis(); + return new TimeRange( + current - 86400000, // 24小时前 + current, + current + 86400000, // 24小时后 + current + 172800000 // 48小时后 + ); + } +} \ No newline at end of file From 1a39fed97d8450e927c12c32cf154cda570eb916 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 10 Mar 2025 16:53:45 +0800 Subject: [PATCH 2/3] fix --- .../hbase/util/TableTemplateManager.java | 19 ++++++++++++++++++- .../oceanbase/hbase/util/TimeGenerator.java | 19 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index f0d90c20..d797cac1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -1,3 +1,20 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; @@ -256,4 +273,4 @@ public static String extractTableGroup(String tableName) { } throw new IllegalArgumentException("Invalid table name: " + tableName); } -} \ No newline at end of file +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java b/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java index 00c6aa46..758dedb3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java @@ -1,3 +1,20 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; public class TimeGenerator { @@ -29,4 +46,4 @@ public static TimeRange generateTestTimeRange() { current + 172800000 // 48小时后 ); } -} \ No newline at end of file +} From 3033173d725c68e7dfea3dce95ccbce47b43db91 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 10 Mar 2025 17:58:18 +0800 Subject: [PATCH 3/3] refactor --- .../hbase/OHTableSecondaryPartMcfTest.java | 179 --------- .../hbase/OHTableSecondaryPartTest.java | 353 ------------------ .../OHTableSecondaryPartAppendTest.java | 103 +++++ .../OHTableSecondaryPartBatchGetTest.java | 106 ++++++ ...HTableSecondaryPartCheckAndMutateTest.java | 93 +++++ .../OHTableSecondaryPartGetTest.java | 123 ++++++ .../OHTableSecondaryPartIncrementTest.java | 106 ++++++ .../OHTableSecondaryPartPutTest.java | 104 ++++++ .../OHTableSecondaryPartScanTest.java | 105 ++++++ .../hbase/util/ObHTableSecondaryPartUtil.java | 164 ++++++++ .../hbase/util/TableTemplateManager.java | 4 +- 11 files changed, 906 insertions(+), 534 deletions(-) delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java deleted file mode 100644 index 71cda1c1..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java +++ /dev/null @@ -1,179 +0,0 @@ -/*- - * #%L - * OBKV HBase Client Framework - * %% - * Copyright (C) 2025 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.hbase; - -import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; -import com.alipay.oceanbase.hbase.util.TableTemplateManager; -import com.alipay.oceanbase.hbase.util.TimeGenerator; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.filter.CompareFilter; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.*; - -import java.sql.Connection; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; -import static org.apache.hadoop.hbase.util.Bytes.toBytes; - -public class OHTableSecondaryPartMcfTest { - private static Map> tableNames = new LinkedHashMap<>(); - - public static void dropTables() throws Exception { - Connection conn = ObHTableTestUtil.getConnection(); - for (Map.Entry> entry : tableNames.entrySet()) { - for (String tableName : entry.getValue()) { - String stmt = "DROP TABLE IF EXISTS " + tableName + ";"; - conn.createStatement().execute(stmt); - System.out.println("============= drop table " + tableName + " done ============="); - } - String stmt = "DROP TABLEGROUP IF EXISTS " + entry.getKey() + ";"; - conn.createStatement().execute(stmt); - System.out.println("============= drop tablegroup " + entry.getKey() + " done ============="); - } - } - public static void createMultiCFTables(TableTemplateManager.TableType type, boolean printSql) throws Exception { - Connection conn = ObHTableTestUtil.getConnection(); - TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); - String tableGroup = TableTemplateManager.getTableGroupName(type); - String tableGroupSql = TableTemplateManager.generateTableGroupSQL(tableGroup); - conn.createStatement().execute(tableGroupSql); - tableNames.put(tableGroup, new LinkedList<>()); - for (int i = 1; i <= 3; ++i) { - String tableName = TableTemplateManager.generateTableName(tableGroup, true, i); - String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); - conn.createStatement().execute(sql); - tableNames.get(tableGroup).add(tableName); - System.out.println("============= create table: " + tableName - + " table_group: " + getTableName(tableName) + " =============\n" - + (printSql ? sql : "") + " \n============= done =============\n"); - } - } - - - public static void truncateTables() throws Exception { - Connection conn = ObHTableTestUtil.getConnection(); - for (Map.Entry> entry : tableNames.entrySet()) { - for (String tableName : entry.getValue()) { - String stmt = "TRUNCATE TABLE " + tableName + ";"; - conn.createStatement().execute(stmt); - System.out.println("============= truncate table " + tableName + " done ============="); - } - } - } - - public static String getTableName(String input) throws Exception { - // 查找 '$' 的索引 - int index = input.indexOf('$'); - // 如果找到了 '$'，提取其前面的部分 - String result; - if (index != -1) { - result = input.substring(0, index); // 提取从开始到 '$' 的部分 - } else { - result = input; // 如果没有 '$' 则返回原字符串 - } - System.out.println("getTableName:" + result); - return result; - } - - public static String getColumnFamilyName(String input) throws Exception { - // 查找 '$' 的索引 - int index = input.indexOf('$'); - // 如果找到了 '$'，提取其后面的部分 - String result; - if (index != -1 && index + 1 < input.length()) { - result = input.substring(index + 1); // 提取从 '$' 后一个字符到结束的部分 - } else { - result = ""; // 如果没有 '$' 或 '$' 是最后一个字符，则返回空字符串 - } - System.out.println("getColumnFamilyName:" + result); - return result; - } - - @BeforeClass - public static void before() throws Exception { - for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { - createMultiCFTables(type, true); - } - } - - @AfterClass - public static void finish() throws Exception { - dropTables(); - } - - @Before - public void prepareCase() throws Exception { - truncateTables(); - } - - public static void testMultiCFPut(Map.Entry> entry) throws Exception { - String key = "putKey"; - String value = "value"; - String column = "putColumn"; - long timestamp = System.currentTimeMillis(); - - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); - hTable.init(); - Put put = new Put(toBytes(key)); - for (String tableName : entry.getValue()) { - String family = getColumnFamilyName(tableName); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - } - hTable.put(put); - hTable.close(); - } - - public static void testMultiCFGet(Map.Entry> entry) throws Exception { - String key = "putKey"; - String column = "putColumn"; - - for (String tableName : entry.getValue()) { - String family = getColumnFamilyName(tableName); - String value = family + "_value"; - long timestamp = System.currentTimeMillis(); - - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - - Get get = new Get(key.getBytes()); - get.addFamily(family.getBytes()); - Result r = hTable.get(get); - Assert.assertEquals(1, r.raw().length); - - hTable.close(); - } - } - - @Test - public void testPut() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartMcfTest::testMultiCFPut); - } - - @Test - public void testGet() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartMcfTest::testMultiCFGet); - } -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java deleted file mode 100644 index be39b71b..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java +++ /dev/null @@ -1,353 +0,0 @@ -/*- - * #%L - * OBKV HBase Client Framework - * %% - * Copyright (C) 2025 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.hbase; - -import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; -import com.alipay.oceanbase.hbase.util.TableTemplateManager; -import com.alipay.oceanbase.hbase.util.TimeGenerator; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.filter.CompareFilter; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.*; - -import java.sql.Connection; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - -import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.*; -import static org.apache.hadoop.hbase.util.Bytes.toBytes; -import static org.junit.Assert.assertEquals; - -public class OHTableSecondaryPartTest { - private static List tableNames = new LinkedList(); - - public static void dropTables() throws Exception { - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableNames.size(); i++) { - String stmt = "DROP TABLE IF EXISTS " + tableNames.get(i) + ";"; - conn.createStatement().execute(stmt); - System.out.println("============= drop table " + tableNames.get(i) + " done ============="); - } - for (int i = 0; i < tableNames.size(); i++) { - String stmt = "DROP TABLEGROUP IF EXISTS " + getTableName(tableNames.get(i)) + ";"; - conn.createStatement().execute(stmt); - System.out.println("============= drop tablegroup " + tableNames.get(i) + " done ============="); - } - } - public static void createTables(TableTemplateManager.TableType type, boolean printSql) throws Exception { - Connection conn = ObHTableTestUtil.getConnection(); - TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); - String tableGroup = TableTemplateManager.getTableGroupName(type); - String tableGroupSql = TableTemplateManager.generateTableGroupSQL(tableGroup); - conn.createStatement().execute(tableGroupSql); - String tableName = TableTemplateManager.generateTableName(tableGroup, false, 1); - String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); - conn.createStatement().execute(sql); - tableNames.add(tableName); - System.out.println("============= create table: " + tableName + " table_group: " + getTableName(tableName) + " =============\n" + (printSql ? sql : "") + " \n============= done =============\n"); - } - - public static void truncateTables() throws Exception { - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableNames.size(); i++) { - String stmt = "TRUNCATE TABLE " + tableNames.get(i) + ";"; - conn.createStatement().execute(stmt); - System.out.println("============= truncate table " + tableNames.get(i) + " done ============="); - } - } - - public static String getTableName(String input) throws Exception { - // 查找 '$' 的索引 - int index = input.indexOf('$'); - // 如果找到了 '$'，提取其前面的部分 - String result; - if (index != -1) { - result = input.substring(0, index); // 提取从开始到 '$' 的部分 - } else { - result = input; // 如果没有 '$' 则返回原字符串 - } - return result; - } - - public static String getColumnFamilyName(String input) throws Exception { - // 查找 '$' 的索引 - int index = input.indexOf('$'); - // 如果找到了 '$'，提取其后面的部分 - String result; - if (index != -1 && index + 1 < input.length()) { - result = input.substring(index + 1); // 提取从 '$' 后一个字符到结束的部分 - } else { - result = ""; // 如果没有 '$' 或 '$' 是最后一个字符，则返回空字符串 - } - return result; - } - - @BeforeClass - public static void before() throws Exception { - for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { - createTables(type, true); - } - } - - @AfterClass - public static void finish() throws Exception { - dropTables(); - } - - @Before - public void prepareCase() throws Exception { - truncateTables(); - } - - public static void testPut(String tableName) throws Exception { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - - String family = getColumnFamilyName(tableName); - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - - hTable.close(); - } - - public static void testGet(String tableName) throws Exception { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - - String family = getColumnFamilyName(tableName); - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), toBytes(value)); - hTable.put(put); - - Get get = new Get(key.getBytes()); - get.addFamily(family.getBytes()); - Result result = hTable.get(get); - Cell[] cells = result.rawCells(); - assertEquals(1, cells.length); - assertEquals(column, Bytes.toString(CellUtil.cloneQualifier(cells[0]))); - assertEquals("value", Bytes.toString(CellUtil.cloneValue(cells[0]))); - System.out.println("get table " + tableName + " done"); - - hTable.close(); - } - - public static void testIncrement(String tableName) throws Exception { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - - String family = getColumnFamilyName(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String column2 = "putColumn2"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); - hTable.put(put); - - Increment increment = new Increment(key.getBytes()); - increment.addColumn(family.getBytes(), column1.getBytes(), 1L); - increment.addColumn(family.getBytes(), column2.getBytes(), 1L); - hTable.increment(increment); - - Get get = new Get(key.getBytes()); - get.addFamily(family.getBytes()); - Result result = hTable.get(get); - Cell[] cells = result.rawCells(); - assertEquals(2, cells.length); - for (Cell cell : cells) { - if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column1)) { - assertEquals("2", Bytes.toString(CellUtil.cloneValue(cell))); - } else if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column2)) { - assertEquals("2", Bytes.toString(CellUtil.cloneValue(cell))); - } - } - hTable.close(); - } - - public static void testAppend(String tableName) throws Exception { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - - String family = getColumnFamilyName(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String column2 = "putColumn2"; - String value = "value"; - Append append = new Append(key.getBytes()); - KeyValue kv1 = new KeyValue(key.getBytes(), family.getBytes(), column1.getBytes(), value.getBytes()); - KeyValue kv2 = new KeyValue(key.getBytes(), family.getBytes(), column2.getBytes(), value.getBytes()); - append.add(kv1); - append.add(kv2); - hTable.append(append); - - Get get = new Get(key.getBytes()); - get.addFamily(family.getBytes()); - Result result = hTable.get(get); - Cell[] cells = result.rawCells(); - assertEquals(2, cells.length); - - for (Cell cell : cells) { - if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column1)) { - assertEquals("valuevalue", Bytes.toString(CellUtil.cloneValue(cell))); - } else if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column2)) { - assertEquals("valuevalue", Bytes.toString(CellUtil.cloneValue(cell))); - } - } - hTable.close(); - } - - public static void testCheckAndMutate(String tableName) throws Exception { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - - String family = getColumnFamilyName(tableName); - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - String newValue = "newValue"; - RowMutations mutations = new RowMutations(key.getBytes()); - Put put = new Put(key.getBytes()); - long timestamp = System.currentTimeMillis(); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - - Put newPut = new Put(key.getBytes()); - newPut.add(family.getBytes(), column.getBytes(), timestamp, toBytes(newValue)); - mutations.add(newPut); - hTable.checkAndMutate(key.getBytes(), family.getBytes(), - column.getBytes(), CompareFilter.CompareOp.EQUAL, value.getBytes(), - mutations); - - hTable.close(); - } - - public static void testBatchGet(String tableName) throws Exception { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - - String family = getColumnFamilyName(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String column2 = "putColumn2"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); - hTable.put(put); - - List gets = new ArrayList<>(); - Get get1 = new Get(key.getBytes()); - get1.addFamily(family.getBytes()); - gets.add(get1); - - Get get2 = new Get(key.getBytes()); - get2.addColumn(family.getBytes(), column1.getBytes()); - gets.add(get2); - - Get get3 = new Get(key.getBytes()); - get3.addColumn(family.getBytes(), column2.getBytes()); - gets.add(get3); - - - Result[] results = hTable.get(gets); - for (Result result : results) { - for (Cell cell : result.listCells()) { - String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); - String V = Bytes.toString(CellUtil.cloneValue(cell)); - System.out.println("Column: " + Q + ", Value: " + V); - } - } - } - - public static void testScan(String tableName) throws Exception { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - - String family = getColumnFamilyName(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String column2 = "putColumn2"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); - hTable.put(put); - - Scan scan = new Scan(key.getBytes()); - scan.addFamily(family.getBytes()); - ResultScanner scanner = hTable.getScanner(scan); - int count = 0; - for (Result result : scanner) { - for (KeyValue keyValue : result.raw()) { - assertEquals(column1, Bytes.toString(keyValue.getQualifier())); - assertEquals("1", Bytes.toString(keyValue.getValue())); - count++; - } - } - assertEquals(1, count); - } - - - @Test - public void testPut() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartTest::testPut); - } - - @Test - public void testGet() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartTest::testGet); - } - - @Test - public void testIncrement() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartTest::testIncrement); - } - - @Test - public void testAppend() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartTest::testAppend); - } - - @Test - public void testCheckAndMutate() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartTest::testCheckAndMutate); - } - - @Test - public void testBatchGet() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartTest::testBatchGet); - } - - @Test - public void testScan() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartTest::testScan); - } -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java new file mode 100644 index 00000000..d175a278 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java @@ -0,0 +1,103 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.junit.Assert.assertEquals; + +public class OHTableSecondaryPartAppendTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = null; + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testAppendImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + String value = "value"; + Append append = new Append(key.getBytes()); + KeyValue kv1 = new KeyValue(key.getBytes(), family.getBytes(), column1.getBytes(), value.getBytes()); + KeyValue kv2 = new KeyValue(key.getBytes(), family.getBytes(), column2.getBytes(), value.getBytes()); + append.add(kv1); + append.add(kv2); + hTable.append(append); + + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(2, cells.length); + + for (Cell cell : cells) { + if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column1)) { + assertEquals("valuevalue", Bytes.toString(CellUtil.cloneValue(cell))); + } else if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column2)) { + assertEquals("valuevalue", Bytes.toString(CellUtil.cloneValue(cell))); + } + } + hTable.close(); + } + + @Test + public void testAppend() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppendImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java new file mode 100644 index 00000000..f05772ab --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java @@ -0,0 +1,106 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableSecondaryPartBatchGetTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testBatchGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); + hTable.put(put); + + List gets = new ArrayList<>(); + Get get1 = new Get(key.getBytes()); + get1.addFamily(family.getBytes()); + gets.add(get1); + + Get get2 = new Get(key.getBytes()); + get2.addColumn(family.getBytes(), column1.getBytes()); + gets.add(get2); + + Get get3 = new Get(key.getBytes()); + get3.addColumn(family.getBytes(), column2.getBytes()); + gets.add(get3); + + + Result[] results = hTable.get(gets); + for (Result result : results) { + for (Cell cell : result.listCells()) { + String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); + String V = Bytes.toString(CellUtil.cloneValue(cell)); + System.out.println("Column: " + Q + ", Value: " + V); + } + } + } + + @Test + public void testBatchGet() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartBatchGetTest::testBatchGetImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java new file mode 100644 index 00000000..04c49fb1 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java @@ -0,0 +1,93 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableSecondaryPartCheckAndMutateTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testCheckAndMutateImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + String newValue = "newValue"; + RowMutations mutations = new RowMutations(key.getBytes()); + Put put = new Put(key.getBytes()); + long timestamp = System.currentTimeMillis(); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + + Put newPut = new Put(key.getBytes()); + newPut.add(family.getBytes(), column.getBytes(), timestamp, toBytes(newValue)); + mutations.add(newPut); + hTable.checkAndMutate(key.getBytes(), family.getBytes(), + column.getBytes(), CompareFilter.CompareOp.EQUAL, value.getBytes(), + mutations); + + hTable.close(); + } + + @Test + public void testCheckAndMutate() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateImpl); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java new file mode 100644 index 00000000..84a4ebf0 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -0,0 +1,123 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableSecondaryPartGetTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), toBytes(value)); + hTable.put(put); + + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(1, cells.length); + assertEquals(column, Bytes.toString(CellUtil.cloneQualifier(cells[0]))); + assertEquals("value", Bytes.toString(CellUtil.cloneValue(cells[0]))); + System.out.println("get table " + tableName + " done"); + + hTable.close(); + } + + public static void testMultiCFGetImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String column = "putColumn"; + + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + String value = family + "_value"; + long timestamp = System.currentTimeMillis(); + + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + hTable.close(); + } + } + + + @Test + public void testGet() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartGetTest::testGetImpl); + } + + @Test + public void testMultiCFGet() throws Exception { + FOR_EACH(group2tableNames, OHTableSecondaryPartGetTest::testMultiCFGetImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java new file mode 100644 index 00000000..d4d7a4cc --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java @@ -0,0 +1,106 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + +public class OHTableSecondaryPartIncrementTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = null; + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testIncrementImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); + hTable.put(put); + + Increment increment = new Increment(key.getBytes()); + increment.addColumn(family.getBytes(), column1.getBytes(), 1L); + increment.addColumn(family.getBytes(), column2.getBytes(), 1L); + hTable.increment(increment); + + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(2, cells.length); + for (Cell cell : cells) { + if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column1)) { + assertEquals("2", Bytes.toString(CellUtil.cloneValue(cell))); + } else if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column2)) { + assertEquals("2", Bytes.toString(CellUtil.cloneValue(cell))); + } + } + hTable.close(); + } + + @Test + public void testIncrement() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartIncrementTest::testIncrementImpl); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java new file mode 100644 index 00000000..add0d03a --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -0,0 +1,104 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.client.Put; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableSecondaryPartPutTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testPutImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + + hTable.close(); + } + + public static void testMultiCFPutImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + long timestamp = System.currentTimeMillis(); + + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + } + hTable.put(put); + hTable.close(); + } + + @Test + public void testPut() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartPutTest::testPutImpl); + } + + @Test + public void testMultiCFPut() throws Exception { + FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMultiCFPutImpl); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java new file mode 100644 index 00000000..036d9789 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java @@ -0,0 +1,105 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + +public class OHTableSecondaryPartScanTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + + public static void testScanImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); + hTable.put(put); + + Scan scan = new Scan(key.getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + assertEquals(column1, Bytes.toString(keyValue.getQualifier())); + assertEquals("1", Bytes.toString(keyValue.getValue())); + count++; + } + } + assertEquals(1, count); + } + + @Test + public void testScan() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartScanTest::testScanImpl); + } + + @Test + public void testMultiCFScan() throws Exception { + // implement later + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java new file mode 100644 index 00000000..48535e96 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -0,0 +1,164 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + + +public class ObHTableSecondaryPartUtil { + public static void createTables(TableTemplateManager.TableType type, List tableNames, Map> group2tableNames, boolean printSql) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // single cf table + if (tableNames != null) { + createTables(conn, type, tableNames, printSql); + } + // multi cf table + if (group2tableNames != null) { + createTables(conn, type, group2tableNames, printSql); + } + } + + public static void createTables(Connection conn, TableTemplateManager.TableType type, List tableNames, boolean printSql) throws Exception { + // create single cf table + if (tableNames != null) { + TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); + String tableGroup = TableTemplateManager.getTableGroupName(type, false); + String tableGroupSql = TableTemplateManager.generateTableGroupSQL(tableGroup); + conn.createStatement().execute(tableGroupSql); + String tableName = TableTemplateManager.generateTableName(tableGroup, false, 1); + String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); + conn.createStatement().execute(sql); + tableNames.add(tableName); + System.out.println("============= create table: " + tableName + " table_group: " + getTableName(tableName) + " =============\n" + (printSql ? sql : "") + " \n============= done =============\n"); + } + } + + public static void createTables(Connection conn, TableTemplateManager.TableType type, Map> group2tableNames, boolean printSql) throws Exception { + if (group2tableNames != null) { + TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); + String tableGroup = TableTemplateManager.getTableGroupName(type, true); + String tableGroupSql = TableTemplateManager.generateTableGroupSQL(tableGroup); + conn.createStatement().execute(tableGroupSql); + group2tableNames.put(tableGroup, new LinkedList<>()); + for (int i = 1; i <= 3; ++i) { + String tableName = TableTemplateManager.generateTableName(tableGroup, true, i); + String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); + conn.createStatement().execute(sql); + group2tableNames.get(tableGroup).add(tableName); + System.out.println("============= create table: " + tableName + + " table_group: " + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + " \n============= done =============\n"); + } + } + } + + public static void truncateTables(List tableNames, Map> group2tableNames) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // truncate single cf table + truncateTables(conn, tableNames); + // truncate multi cf table + truncateTables(conn, group2tableNames); + } + + public static void truncateTables(Connection conn, List tableNames) throws Exception { + if (tableNames != null) { + for (int i = 0; i < tableNames.size(); i++) { + String stmt = "TRUNCATE TABLE " + tableNames.get(i) + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= truncate table " + tableNames.get(i) + " done ============="); + } + } + } + + public static void truncateTables(Connection conn, Map> group2tableNames) throws Exception { + if (group2tableNames != null) { + for (Map.Entry> entry : group2tableNames.entrySet()) { + for (String tableName : entry.getValue()) { + String stmt = "TRUNCATE TABLE " + tableName + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= truncate table " + tableName + " done ============="); + } + } + } + } + + + public static void dropTables(List tableNames, Map> group2tableNames) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // drop single cf table + dropTables(conn, tableNames); + // drop multi cf table + dropTables(conn, group2tableNames); + } + + public static void dropTables(Connection conn, List tableNames) throws Exception { + if (tableNames != null) { + for (String tableName : tableNames) { + String stmt = "DROP TABLE IF EXISTS " + tableName + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= drop table " + tableName + " done ============="); + } + } + } + + public static void dropTables(Connection conn, Map> group2tableNames) throws Exception { + if (group2tableNames != null) { + for (Map.Entry> entry : group2tableNames.entrySet()) { + for (String tableName : entry.getValue()) { + String stmt = "DROP TABLE IF EXISTS " + tableName + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= drop table " + tableName + " done ============="); + } + String stmt = "DROP TABLEGROUP IF EXISTS " + entry.getKey() + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= drop tablegroup " + entry.getKey() + " done ============="); + } + } + } + + + public static String getTableName(String input) throws Exception { + // 查找 '$' 的索引 + int index = input.indexOf('$'); + // 如果找到了 '$'，提取其前面的部分 + String result; + if (index != -1) { + result = input.substring(0, index); // 提取从开始到 '$' 的部分 + } else { + result = input; // 如果没有 '$' 则返回原字符串 + } + return result; + } + + public static String getColumnFamilyName(String input) throws Exception { + // 查找 '$' 的索引 + int index = input.indexOf('$'); + // 如果找到了 '$'，提取其后面的部分 + String result; + if (index != -1 && index + 1 < input.length()) { + result = input.substring(index + 1); // 提取从 '$' 后一个字符到结束的部分 + } else { + result = ""; // 如果没有 '$' 或 '$' 是最后一个字符，则返回空字符串 + } + return result; + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index d797cac1..21c23081 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -256,8 +256,8 @@ public static String generateTableGroupSQL(String tableGroup) { return String.format("CREATE TABLEGROUP IF NOT EXISTS %s SHARDING = 'ADAPTIVE'", tableGroup); } - public static String getTableGroupName(TableTemplateManager.TableType type) { - return TABLE_GROUP_PREFIX + type.name().toLowerCase(); + public static String getTableGroupName(TableTemplateManager.TableType type, boolean multiCf) { + return TABLE_GROUP_PREFIX + type.name().toLowerCase() + (multiCf ? "_mcf" : ""); } public static String generateTableName(String tableGroup, boolean multiCf, int cfIndex) {