From aace87c3b968c4ec0d8a9469a45906d69c38c6bc Mon Sep 17 00:00:00 2001 From: vanson <43193589+WeiXinChan@users.noreply.github.com> Date: Fri, 21 Feb 2025 14:22:50 +0800 Subject: [PATCH 01/40] add single conlumn family put test (#158) (cherry picked from commit 3982cb9294c841e5446616fc3952e0dd42d43cb7) --- .../hbase/OHTableSecondaryPartTest.java | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java new file mode 100644 index 00000000..57f1344e --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java @@ -0,0 +1,176 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.util.LinkedList; +import java.util.List; + +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.getConnection; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.*; + +public class OHTableSecondaryPartTest { + private static String tableNames[] = new String[] {"test$cf1", "test$cf2", "test$cf3", "test$cf4"}; + private static String createTableStmt1 = "CREATE TABLE IF NOT EXISTS `" + tableNames[0] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 97 (" + + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + + " PARTITION `p3` VALUES LESS THAN MAXVALUE" + + ");"; + private static String createTableStmt2 = "CREATE TABLE IF NOT EXISTS `" + tableNames[1] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 97 (" + + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + + " PARTITION `p3` VALUES LESS THAN MAXVALUE" + + ");"; + private static String createTableStmt3 = "CREATE TABLE IF NOT EXISTS `" + tableNames[2] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") PARTITION BY KEY(`K`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + + " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + + " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + + ") PARTITIONS 97;"; + private static String createTableStmt4 = "CREATE TABLE IF NOT EXISTS `" + tableNames[3] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") PARTITION BY KEY(`K_PREFIX`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + + " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + + " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + + ") PARTITIONS 97;"; + private static String createTableStmts[] = new String[] {createTableStmt1, createTableStmt2, createTableStmt3, createTableStmt4}; + + public static void dropTables() throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + for (int i = 0; i < tableNames.length; i++) { + String stmt = "DROP TABLE IF EXISTS " + tableNames[i] + ";"; + conn.createStatement().execute(stmt); + } + } + + public static void createTables() throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + for (int i = 0; i < createTableStmts.length; i++) { + conn.createStatement().execute(createTableStmts[i]); + } + } + + public static void truncateTables() throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + for (int i = 0; i < tableNames.length; i++) { + String stmt = "TRUNCATE TABLE " + tableNames[i] + ";"; + conn.createStatement().execute(stmt); + } + } + + public static String getTableName(String input) throws Exception { + // 查找 '$' 的索引 + int index = input.indexOf('$'); + // 如果找到了 '$'，提取其前面的部分 + String result; + if (index != -1) { + result = input.substring(0, index); // 提取从开始到 '$' 的部分 + } else { + result = input; // 如果没有 '$' 则返回原字符串 + } + return result; + } + + public static String getColumnFamilyName(String input) throws Exception { + // 查找 '$' 的索引 + int index = input.indexOf('$'); + // 如果找到了 '$'，提取其后面的部分 + String result; + if (index != -1 && index + 1 < input.length()) { + result = input.substring(index + 1); // 提取从 '$' 后一个字符到结束的部分 + } else { + result = ""; // 如果没有 '$' 或 '$' 是最后一个字符，则返回空字符串 + } + return result; + } + + @BeforeClass + public static void before() throws Exception { + createTables(); + } + + @AfterClass + public static void finish() throws Exception { + dropTables(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(); + } + + @Test + public void testPut() throws Exception { + for (int i = 0; i < tableNames.length; i++) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); + hTable.init(); + + String family = getColumnFamilyName(tableNames[i]); + String key = "putKey"; + String column1 = "putColumn"; + String value = "value"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + + hTable.close(); + } + } +} From 23032372cd1f04c63bdd99572438a4a9920a6f24 Mon Sep 17 00:00:00 2001 From: vanson <43193589+WeiXinChan@users.noreply.github.com> Date: Tue, 25 Feb 2025 19:46:50 +0800 Subject: [PATCH 02/40] Secondary part tests (#159) * add single conlumn family put test * multi cf test (cherry picked from commit 01599e3c5b4da09d73cb2ee89ce1f0ee66e92162) --- .../hbase/OHTableSecondaryPartMcfTest.java | 286 ++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java new file mode 100644 index 00000000..c853beca --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java @@ -0,0 +1,286 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; + +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableSecondaryPartMcfTest { + private static String tableGroupNames[] = { + "test_table_group1", + "test_table_group2", + "test_table_group3", + "test_table_group4", + }; + private static String tableNames[][] = new String[][] { + {tableGroupNames[0] + "$cf1", tableGroupNames[0] + "$cf2"}, + {tableGroupNames[1] + "$cf1", tableGroupNames[1] + "$cf2"}, + {tableGroupNames[2] + "$cf1", tableGroupNames[2] + "$cf2"}, + {tableGroupNames[3] + "$cf1", tableGroupNames[3] + "$cf2"} + }; + private static String createTableStmts[][] = { + {"CREATE TABLE IF NOT EXISTS `" + tableNames[0][0] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") TABLEGROUP = "+ tableGroupNames[0] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 9 (" + + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + + " PARTITION `p3` VALUES LESS THAN MAXVALUE" + + ");", + "CREATE TABLE IF NOT EXISTS `" + tableNames[0][1] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") TABLEGROUP = "+ tableGroupNames[0] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 9 (" + + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + + " PARTITION `p3` VALUES LESS THAN MAXVALUE" + + ");" + }, + {"CREATE TABLE IF NOT EXISTS `" + tableNames[1][0] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") TABLEGROUP = "+ tableGroupNames[1] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 9 (" + + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + + " PARTITION `p3` VALUES LESS THAN MAXVALUE" + + ");", + "CREATE TABLE IF NOT EXISTS `" + tableNames[1][1] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") TABLEGROUP = "+ tableGroupNames[1] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 9 (" + + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + + " PARTITION `p3` VALUES LESS THAN MAXVALUE" + + ");" + + }, + {"CREATE TABLE IF NOT EXISTS `" + tableNames[2][0] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") TABLEGROUP = "+ tableGroupNames[2] +" PARTITION BY KEY(`K`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + + " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + + " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + + ") PARTITIONS 9;", + "CREATE TABLE IF NOT EXISTS `" + tableNames[2][1] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") TABLEGROUP = "+ tableGroupNames[2] +" PARTITION BY KEY(`K`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + + " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + + " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + + ") PARTITIONS 9;" + + }, + {"CREATE TABLE IF NOT EXISTS `" + tableNames[3][0] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") TABLEGROUP = "+ tableGroupNames[3] +" PARTITION BY KEY(`K_PREFIX`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + + " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + + " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + + ") PARTITIONS 9;", + "CREATE TABLE IF NOT EXISTS `" + tableNames[3][1] + "` (" + + " `K` varbinary(1024) NOT NULL," + + " `Q` varbinary(1024) NOT NULL," + + " `T` bigint(20) NOT NULL," + + " `V` varbinary(1024) DEFAULT NULL," + + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + + " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + + " PRIMARY KEY (`K`, `Q`, `T`)" + + ") TABLEGROUP = "+ tableGroupNames[3] +" PARTITION BY KEY(`K_PREFIX`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + + " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + + " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + + ") PARTITIONS 9;" + + } + }; + + public static void createTableGroups() throws Exception { + System.out.print("create table group start"); + Connection conn = ObHTableTestUtil.getConnection(); + for (int i = 0; i < tableGroupNames.length; i++) { + System.out.print("."); + String stmt = "CREATE TABLEGROUP IF NOT EXISTS `"+ tableGroupNames[i] +"` SHARDING = 'ADAPTIVE';"; + conn.createStatement().execute(stmt); + } + System.out.println("end"); + } + + public static void dropTableGroups() throws Exception { + System.out.print("drop table group start"); + Connection conn = ObHTableTestUtil.getConnection(); + for (int i = 0; i < tableGroupNames.length; i++) { + System.out.print("."); + String stmt = "DROP TABLEGROUP IF EXISTS " + tableGroupNames[i] + ";"; + conn.createStatement().execute(stmt); + } + System.out.println("end"); + } + + public static void dropTables() throws Exception { + System.out.print("drop table start"); + Connection conn = ObHTableTestUtil.getConnection(); + for (int i = 0; i < tableNames.length; i++) { + for (int j = 0; j < tableNames[i].length; j++) { + System.out.print("."); + String stmt = "DROP TABLE IF EXISTS " + tableNames[i][j] + ";"; + conn.createStatement().execute(stmt); + } + } + System.out.println("end"); + } + + public static void createTables() throws Exception { + System.out.print("create table start"); + Connection conn = ObHTableTestUtil.getConnection(); + for (int i = 0; i < createTableStmts.length; i++) { + for (int j = 0; j < createTableStmts[i].length; j++) { + System.out.print("."); + conn.createStatement().execute(createTableStmts[i][j]); + } + } + System.out.println("end"); + } + + public static void truncateTables() throws Exception { + System.out.print("truncate table start"); + Connection conn = ObHTableTestUtil.getConnection(); + for (int i = 0; i < tableNames.length; i++) { + for (int j = 0; j < tableNames[i].length; j++) { + System.out.print("."); + String stmt = "TRUNCATE TABLE " + tableNames[i][j] + ";"; + conn.createStatement().execute(stmt); + } + } + System.out.println("end"); + } + + public static String getTableName(String input) throws Exception { + // 查找 '$' 的索引 + int index = input.indexOf('$'); + // 如果找到了 '$'，提取其前面的部分 + String result; + if (index != -1) { + result = input.substring(0, index); // 提取从开始到 '$' 的部分 + } else { + result = input; // 如果没有 '$' 则返回原字符串 + } + System.out.println("getTableName:" + result); + return result; + } + + public static String getColumnFamilyName(String input) throws Exception { + // 查找 '$' 的索引 + int index = input.indexOf('$'); + // 如果找到了 '$'，提取其后面的部分 + String result; + if (index != -1 && index + 1 < input.length()) { + result = input.substring(index + 1); // 提取从 '$' 后一个字符到结束的部分 + } else { + result = ""; // 如果没有 '$' 或 '$' 是最后一个字符，则返回空字符串 + } + System.out.println("getColumnFamilyName:" + result); + return result; + } + + @BeforeClass + public static void before() throws Exception { + createTableGroups(); + createTables(); + } + + @AfterClass + public static void finish() throws Exception { + dropTables(); + dropTableGroups(); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(); + } + + @Test + public void testPut() throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + long timestamp = System.currentTimeMillis(); + for (int i = 0; i < tableNames.length; i++) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); + hTable.init(); + Put put = new Put(toBytes(key)); + for (int j = 0; j < tableNames[i].length; j++) { + String family = getColumnFamilyName(tableNames[i][j]); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + } + hTable.put(put); + hTable.close(); + } + } +} From 278b82c9f17f11d300ba7cea1d851d9cae9ce1e7 Mon Sep 17 00:00:00 2001 From: vanson <43193589+WeiXinChan@users.noreply.github.com> Date: Fri, 28 Feb 2025 15:11:18 +0800 Subject: [PATCH 03/40] update tests (#160) (cherry picked from commit 0fad7f747e9e47022ff8a9fe00feddc83501e1f6) --- .../hbase/OHTableSecondaryPartMcfTest.java | 16 +-- .../hbase/OHTableSecondaryPartTest.java | 97 +++++++++++++++++-- 2 files changed, 97 insertions(+), 16 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java index c853beca..642ac140 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java @@ -50,7 +50,7 @@ public class OHTableSecondaryPartMcfTest { " `V` varbinary(1024) DEFAULT NULL," + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[0] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 9 (" + + ") TABLEGROUP = "+ tableGroupNames[0] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 3 (" + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + @@ -63,7 +63,7 @@ public class OHTableSecondaryPartMcfTest { " `V` varbinary(1024) DEFAULT NULL," + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[0] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 9 (" + + ") TABLEGROUP = "+ tableGroupNames[0] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 3 (" + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + @@ -78,7 +78,7 @@ public class OHTableSecondaryPartMcfTest { " `G` bigint(20) GENERATED ALWAYS AS (-T)," + " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[1] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 9 (" + + ") TABLEGROUP = "+ tableGroupNames[1] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 3 (" + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + @@ -92,7 +92,7 @@ public class OHTableSecondaryPartMcfTest { " `G` bigint(20) GENERATED ALWAYS AS (-T)," + " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[1] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 9 (" + + ") TABLEGROUP = "+ tableGroupNames[1] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 3 (" + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + @@ -112,7 +112,7 @@ public class OHTableSecondaryPartMcfTest { " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 9;", + ") PARTITIONS 3;", "CREATE TABLE IF NOT EXISTS `" + tableNames[2][1] + "` (" + " `K` varbinary(1024) NOT NULL," + " `Q` varbinary(1024) NOT NULL," + @@ -125,7 +125,7 @@ public class OHTableSecondaryPartMcfTest { " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 9;" + ") PARTITIONS 3;" }, {"CREATE TABLE IF NOT EXISTS `" + tableNames[3][0] + "` (" + @@ -141,7 +141,7 @@ public class OHTableSecondaryPartMcfTest { " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 9;", + ") PARTITIONS 3;", "CREATE TABLE IF NOT EXISTS `" + tableNames[3][1] + "` (" + " `K` varbinary(1024) NOT NULL," + " `Q` varbinary(1024) NOT NULL," + @@ -155,7 +155,7 @@ public class OHTableSecondaryPartMcfTest { " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 9;" + ") PARTITIONS 3;" } }; diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java index 57f1344e..9b99bac4 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java @@ -18,8 +18,9 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.CompareFilter; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -42,7 +43,7 @@ public class OHTableSecondaryPartTest { " `V` varbinary(1024) DEFAULT NULL," + " `G` bigint(20) GENERATED ALWAYS AS (-T)," + " PRIMARY KEY (`K`, `Q`, `T`)" + - ") PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 97 (" + + ") PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 3 (" + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + @@ -56,7 +57,7 @@ public class OHTableSecondaryPartTest { " `G` bigint(20) GENERATED ALWAYS AS (-T)," + " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + " PRIMARY KEY (`K`, `Q`, `T`)" + - ") PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 97 (" + + ") PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 3 (" + " PARTITION `p0` VALUES LESS THAN (1728921600000)," + " PARTITION `p1` VALUES LESS THAN (1729008000000)," + " PARTITION `p2` VALUES LESS THAN (1729094400000)," + @@ -74,7 +75,7 @@ public class OHTableSecondaryPartTest { " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 97;"; + ") PARTITIONS 3;"; private static String createTableStmt4 = "CREATE TABLE IF NOT EXISTS `" + tableNames[3] + "` (" + " `K` varbinary(1024) NOT NULL," + " `Q` varbinary(1024) NOT NULL," + @@ -88,7 +89,7 @@ public class OHTableSecondaryPartTest { " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 97;"; + ") PARTITIONS 3;"; private static String createTableStmts[] = new String[] {createTableStmt1, createTableStmt2, createTableStmt3, createTableStmt4}; public static void dropTables() throws Exception { @@ -96,6 +97,7 @@ public static void dropTables() throws Exception { for (int i = 0; i < tableNames.length; i++) { String stmt = "DROP TABLE IF EXISTS " + tableNames[i] + ";"; conn.createStatement().execute(stmt); + System.out.println("drop table " + tableNames[i] + " done"); } } @@ -103,6 +105,7 @@ public static void createTables() throws Exception { Connection conn = ObHTableTestUtil.getConnection(); for (int i = 0; i < createTableStmts.length; i++) { conn.createStatement().execute(createTableStmts[i]); + System.out.println("create table " + tableNames[i] + " done"); } } @@ -111,6 +114,7 @@ public static void truncateTables() throws Exception { for (int i = 0; i < tableNames.length; i++) { String stmt = "TRUNCATE TABLE " + tableNames[i] + ";"; conn.createStatement().execute(stmt); + System.out.println("truncate table " + tableNames[i] + " done"); } } @@ -161,14 +165,91 @@ public void testPut() throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); hTable.init(); + System.out.println("put table " + tableNames[i] + " begin"); String family = getColumnFamilyName(tableNames[i]); String key = "putKey"; - String column1 = "putColumn"; + String column = "putColumn"; String value = "value"; long timestamp = System.currentTimeMillis(); Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(value)); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); hTable.put(put); + System.out.println("put table " + tableNames[i] + " done"); + + hTable.close(); + } + } + + @Test + public void testIncrement() throws Exception { + for (int i = 0; i < tableNames.length; i++) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); + hTable.init(); + + System.out.println("increment table " + tableNames[i] + " begin"); + String family = getColumnFamilyName(tableNames[i]); + String key = "Key"; + String column1 = "Column1"; + String column2 = "Column2"; + Increment increment = new Increment(key.getBytes()); + increment.addColumn(family.getBytes(), column1.getBytes(), 1L); + increment.addColumn(family.getBytes(), column2.getBytes(), 1L); + hTable.increment(increment); + System.out.println("increment table " + tableNames[i] + " done"); + + hTable.close(); + } + } + + @Test + public void testAppend() throws Exception { + for (int i = 0; i < tableNames.length; i++) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); + hTable.init(); + + System.out.println("append table " + tableNames[i] + " begin"); + String family = getColumnFamilyName(tableNames[i]); + String key = "Key"; + String column1 = "Column1"; + String column2 = "Column2"; + String value = "app"; + Append append = new Append(key.getBytes()); + KeyValue kv1 = new KeyValue(key.getBytes(), family.getBytes(), column1.getBytes(), value.getBytes()); + KeyValue kv2 = new KeyValue(key.getBytes(), family.getBytes(), column2.getBytes(), value.getBytes()); + append.add(kv1); + append.add(kv2); + hTable.append(append); + System.out.println("append table " + tableNames[i] + " done"); + + hTable.close(); + } + } + + @Test + public void testCheckAndMutate() throws Exception { + for (int i = 0; i < tableNames.length; i++) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); + hTable.init(); + + System.out.println("checkAndMutate table " + tableNames[i] + " begin"); + String family = getColumnFamilyName(tableNames[i]); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + String newValue = "newValue"; + RowMutations mutations = new RowMutations(key.getBytes()); + Put put = new Put(key.getBytes()); + long timestamp = System.currentTimeMillis(); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + + Put newPut = new Put(key.getBytes()); + newPut.add(family.getBytes(), column.getBytes(), timestamp, toBytes(newValue)); + mutations.add(newPut); + hTable.checkAndMutate(key.getBytes(), family.getBytes(), + column.getBytes(), CompareFilter.CompareOp.EQUAL, value.getBytes(), + mutations); + System.out.println("checkAndMutate table " + tableNames[i] + " done"); hTable.close(); } From 61c1e97f7ba6e318a44e3cd099d3a57521fd7703 Mon Sep 17 00:00:00 2001 From: vanson <43193589+WeiXinChan@users.noreply.github.com> Date: Wed, 5 Mar 2025 21:55:33 +0800 Subject: [PATCH 04/40] fix batch get route bug (#162) (cherry picked from commit 4c3d0ba93a5296845d40d34bd5a228306fb4a769) --- .../com/alipay/oceanbase/hbase/OHTable.java | 2 +- .../hbase/OHTableSecondaryPartMcfTest.java | 81 +++++++++++++++++-- .../hbase/OHTableSecondaryPartTest.java | 81 ++++++++++++++++++- 3 files changed, 153 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 2aecad25..a870b447 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -2086,7 +2086,7 @@ private BatchOperation buildBatchOperation(String tableName, List obTableQuery = buildObTableQuery(get, columnFilters); ObTableClientQueryImpl query = new ObTableClientQueryImpl(tableName, obTableQuery, obTableClient); try { - query.setRowKey(row(colVal("K", Bytes.toString(get.getRow())), colVal("Q", null), colVal("T", null))); + query.setRowKey(row(colVal("K", Bytes.toString(get.getRow())), colVal("Q", null), colVal("T", Integer.MAX_VALUE))); } catch (Exception e) { logger.error("unexpected error occurs when set row key", e); throw new IOException(e); diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java index 642ac140..235b06a6 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java @@ -18,12 +18,10 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Table; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.junit.*; import java.sql.Connection; @@ -283,4 +281,75 @@ public void testPut() throws Exception { hTable.close(); } } + + @Test + public void testIncrement() throws Exception { + for (int i = 0; i < tableNames.length; i++) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); + hTable.init(); + + String key = "Key"; + String column = "Column"; + Increment increment = new Increment(key.getBytes()); + for (int j = 0; j < tableNames[i].length; j++) { + String family = getColumnFamilyName(tableNames[i][j]); + increment.addColumn(family.getBytes(), column.getBytes(), 1L); + } + hTable.increment(increment); + + hTable.close(); + } + } + + @Test + public void testAppend() throws Exception { + for (int i = 0; i < tableNames.length; i++) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); + hTable.init(); + + String key = "Key"; + String column = "Column"; + String value = "app"; + Append append = new Append(key.getBytes()); + for (int j = 0; j < tableNames[i].length; j++) { + String family = getColumnFamilyName(tableNames[i][j]); + KeyValue kv1 = new KeyValue(key.getBytes(), family.getBytes(), column.getBytes(), value.getBytes()); + append.add(kv1); + } + hTable.append(append); + + hTable.close(); + } + } + + @Test + public void testGet() throws Exception { + for (int i = 0; i < tableNames.length; i++) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); + hTable.init(); + + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + for (int j = 0; j < tableNames[i].length; j++) { + String family = getColumnFamilyName(tableNames[i][j]); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + } + hTable.put(put); + + Get get = new Get(key.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + for (int j = 0; j < tableNames[i].length; j++) { + String family = getColumnFamilyName(tableNames[i][j]); + get.addColumn(family.getBytes(), column.getBytes()); + } + Result r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + hTable.close(); + } + } + } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java index 9b99bac4..029dd3c4 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java @@ -18,15 +18,16 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.CompareFilter; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; import java.sql.Connection; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; @@ -254,4 +255,76 @@ public void testCheckAndMutate() throws Exception { hTable.close(); } } + + @Test + public void testGet() throws Exception { + for (int i = 0; i < tableNames.length; i++) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); + hTable.init(); + + System.out.println("put table " + tableNames[i] + " begin"); + String family = getColumnFamilyName(tableNames[i]); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + System.out.println("put table " + tableNames[i] + " done"); + + System.out.println("get table " + tableNames[i] + " begin"); + Get get = new Get(key.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + get.addColumn(family.getBytes(), column.getBytes()); + Result r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + System.out.println("get table " + tableNames[i] + " begin"); + + hTable.close(); + } + } + + @Test + public void testBatchGet() throws Exception { + long batchSize = 10; + for (int i = 0; i < tableNames.length; i++) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); + hTable.init(); + + System.out.println("put table " + tableNames[i] + " begin"); + String family = getColumnFamilyName(tableNames[i]); + String column = "putColumn"; + String value = "value"; + long timestamp = System.currentTimeMillis(); + for (int j = 0; j < batchSize; j++) { + String key = "putKey" + j; + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + } + System.out.println("put table " + tableNames[i] + " done"); + + System.out.println("get table " + tableNames[i] + " begin"); + List gets = new ArrayList<>(); + for (int j = 0; j < batchSize; j++) { + String key = "putKey" + j; + Get get = new Get(key.getBytes()); + get.setMaxVersions(Integer.MAX_VALUE); + get.addColumn(family.getBytes(), column.getBytes()); + gets.add(get); + } + Result[] results = hTable.get(gets); + for (Result result : results) { + for (Cell cell : result.listCells()) { + String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); + String V = Bytes.toString(CellUtil.cloneValue(cell)); + System.out.println("Column: " + Q + ", Value: " + V); + } + } + System.out.println("get table " + tableNames[i] + " begin"); + + hTable.close(); + } + } } From 8201c3acd1b3c2bfbd70331ac56dc889f3a2f04d Mon Sep 17 00:00:00 2001 From: vanson <43193589+WeiXinChan@users.noreply.github.com> Date: Wed, 5 Mar 2025 22:32:32 +0800 Subject: [PATCH 05/40] add single cf scan test (#163) (cherry picked from commit c15b93f28a48d95e74b111fec45b176f7f5b123e) --- .../hbase/OHTableSecondaryPartTest.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java index 029dd3c4..235e4375 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java @@ -34,6 +34,7 @@ import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.getConnection; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; public class OHTableSecondaryPartTest { private static String tableNames[] = new String[] {"test$cf1", "test$cf2", "test$cf3", "test$cf4"}; @@ -327,4 +328,44 @@ public void testBatchGet() throws Exception { hTable.close(); } } + + @Test + public void testScan() throws Exception { + long rowSize = 10; + for (int i = 0; i < tableNames.length; i++) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); + hTable.init(); + + System.out.println("put table " + tableNames[i] + " begin"); + String key = "putKey"; + String family = getColumnFamilyName(tableNames[i]); + String column = "Column"; + String value = "value"; + long timestamp = System.currentTimeMillis(); + for (int j = 0; j < rowSize; j++) { + Put put = new Put(toBytes(key)); + String qualify = column + j; + put.add(family.getBytes(), qualify.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + } + System.out.println("put table " + tableNames[i] + " done"); + + System.out.println("scan table " + tableNames[i] + " begin"); + Scan scan = new Scan(toBytes(key)); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + assertEquals(column + count, Bytes.toString(keyValue.getQualifier())); + assertEquals(value, Bytes.toString(keyValue.getValue())); + count++; + } + } + assertEquals(rowSize, count); + System.out.println("scan table " + tableNames[i] + " end"); + + hTable.close(); + } + } } From d3e8e8a5fa761df0b725bb0dbf069cf807e68ffd Mon Sep 17 00:00:00 2001 From: stuBirdFly <84010733+stuBirdFly@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:31:09 +0800 Subject: [PATCH 06/40] finish connection function --- .../com/alipay/oceanbase/hbase/OHTable.java | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index a870b447..732f463d 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1420,8 +1420,9 @@ private boolean checkAndMutation(byte[] row, byte[] family, byte[] qualifier, "mutation family is not equal check family"); } byte[] filterString = buildCheckAndMutateFilterString(family, qualifier, compareOp, value); - ObHTableFilter filter = buildObHTableFilter(filterString, timeRange, 1, qualifier); - ObTableQuery obTableQuery = buildObTableQuery(filter, row, true, row, true, false); + ObHTableFilter filter = buildObHTableFilter(filterString, null, 1, qualifier); + ObTableQuery obTableQuery = buildObTableQuery(filter, row, true, row, true, false, + new TimeRange()); ObTableBatchOperation batch = buildObTableBatchOperation(mutations, null); ObTableQueryAndMutateRequest request = buildObTableQueryAndMutateRequest(obTableQuery, @@ -1458,7 +1459,8 @@ public Result append(Append append) throws IOException { Collections.singletonList(append), qualifiers); // the later hbase has supported timeRange ObHTableFilter filter = buildObHTableFilter(null, null, 1, qualifiers); - ObTableQuery obTableQuery = buildObTableQuery(filter, r, true, r, true, false); + ObTableQuery obTableQuery = buildObTableQuery(filter, r, true, r, true, false, + new TimeRange()); ObTableQueryAndMutate queryAndMutate = new ObTableQueryAndMutate(); queryAndMutate.setTableQuery(obTableQuery); queryAndMutate.setMutations(batchOperation); @@ -1510,7 +1512,7 @@ public Result increment(Increment increment) throws IOException { ObHTableFilter filter = buildObHTableFilter(null, increment.getTimeRange(), 1, qualifiers); - ObTableQuery obTableQuery = buildObTableQuery(filter, rowKey, true, rowKey, true, false); + ObTableQuery obTableQuery = buildObTableQuery(filter, rowKey, true, rowKey, true, false, increment.getTimeRange()); ObTableQueryAndMutateRequest request = buildObTableQueryAndMutateRequest(obTableQuery, batch, getTargetTableName(tableNameString, Bytes.toString(f), configuration)); @@ -1557,7 +1559,8 @@ public long incrementColumnValue(byte[] row, byte[] family, byte[] qualifier, lo ObHTableFilter filter = buildObHTableFilter(null, null, 1, qualifiers); - ObTableQuery obTableQuery = buildObTableQuery(filter, row, true, row, true, false); + ObTableQuery obTableQuery = buildObTableQuery(filter, row, true, row, true, false, + new TimeRange()); ObTableQueryAndMutate queryAndMutate = new ObTableQueryAndMutate(); queryAndMutate.setMutations(batch); queryAndMutate.setTableQuery(obTableQuery); @@ -1871,24 +1874,33 @@ private ObHTableFilter buildObHTableFilter(byte[] filterString, TimeRange timeRa private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, boolean includeStart, byte[] stop, boolean includeStop, - boolean isReversed) { + boolean isReversed, TimeRange ts) { ObNewRange obNewRange = new ObNewRange(); + ObBorderFlag obBorderFlag = new ObBorderFlag(); + Object left_ts = ObObj.getMin(); + Object right_ts = ObObj.getMax(); + if (!ts.isAllTime()) { + left_ts = -ts.getMax(); + right_ts = -ts.getMin(); + } if (Arrays.equals(start, HConstants.EMPTY_BYTE_ARRAY)) { - obNewRange.setStartKey(ObRowKey.getInstance(ObObj.getMin(), ObObj.getMin(), - ObObj.getMin())); + obNewRange.setStartKey(ObRowKey.getInstance(ObObj.getMin(), ObObj.getMin(), left_ts)); } else if (includeStart) { - obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMin(), ObObj.getMin())); + obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMin(), left_ts)); + obBorderFlag.setInclusiveStart(); } else { - obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMax(), ObObj.getMax())); + obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMax(), left_ts)); + obBorderFlag.unsetInclusiveStart(); } if (Arrays.equals(stop, HConstants.EMPTY_BYTE_ARRAY)) { - obNewRange.setEndKey(ObRowKey.getInstance(ObObj.getMax(), ObObj.getMax(), - ObObj.getMax())); + obNewRange.setEndKey(ObRowKey.getInstance(ObObj.getMax(), ObObj.getMax(), right_ts)); } else if (includeStop) { - obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMax(), ObObj.getMax())); + obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMax(), right_ts)); + obBorderFlag.setInclusiveEnd(); } else { - obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMin(), ObObj.getMin())); + obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMin(), right_ts)); + obBorderFlag.unsetInclusiveEnd(); } ObTableQuery obTableQuery = new ObTableQuery(); if (isReversed) { @@ -1902,6 +1914,7 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { ObTableQuery obTableQuery; + TimeRange ts = scan.getTimeRange(); if (scan.getMaxResultsPerColumnFamily() > 0) { filter.setLimitPerRowPerCf(scan.getMaxResultsPerColumnFamily()); } @@ -1909,11 +1922,11 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { filter.setOffsetPerRowPerCf(scan.getRowOffsetPerColumnFamily()); } if (scan.isReversed()) { - obTableQuery = buildObTableQuery(filter, scan.getStopRow(), scan.includeStopRow(), - scan.getStartRow(), scan.includeStartRow(), true); + obTableQuery = buildObTableQuery(filter, scan.getStopRow(), false, scan.getStartRow(), + true, true, ts); } else { - obTableQuery = buildObTableQuery(filter, scan.getStartRow(), scan.includeStartRow(), - scan.getStopRow(), scan.includeStopRow(), false); + obTableQuery = buildObTableQuery(filter, scan.getStartRow(), true, scan.getStopRow(), + false, false, ts); } obTableQuery.setBatchSize(scan.getBatch()); obTableQuery.setLimit(scan.getLimit()); @@ -1931,10 +1944,10 @@ private ObTableQuery buildObTableQuery(final Get get, Collection columnQ get.getMaxVersions(), columnQualifiers); if (get.isClosestRowBefore()) { obTableQuery = buildObTableQuery(filter, HConstants.EMPTY_BYTE_ARRAY, true, - get.getRow(), true, true); - obTableQuery.setLimit(1); + get.getRow(), true, true, get.getTimeRange()); } else { - obTableQuery = buildObTableQuery(filter, get.getRow(), true, get.getRow(), true, false); + obTableQuery = buildObTableQuery(filter, get.getRow(), true, get.getRow(), true, false, + get.getTimeRange()); } obTableQuery.setObKVParams(buildOBKVParams(get)); return obTableQuery; From 5458d78b3f8ea561d64dbc435358810f24544088 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 10 Mar 2025 18:01:20 +0800 Subject: [PATCH 07/40] add table template manager for hbase (#164) * refactor secondart test * fix * refactor (cherry picked from commit 3822d55a596c397094cdb52878345809cb3d4158) --- .../hbase/OHTableSecondaryPartMcfTest.java | 355 ----------------- .../hbase/OHTableSecondaryPartTest.java | 371 ------------------ .../OHTableSecondaryPartAppendTest.java | 103 +++++ .../OHTableSecondaryPartBatchGetTest.java | 106 +++++ ...HTableSecondaryPartCheckAndMutateTest.java | 93 +++++ .../OHTableSecondaryPartGetTest.java | 123 ++++++ .../OHTableSecondaryPartIncrementTest.java | 106 +++++ .../OHTableSecondaryPartPutTest.java | 104 +++++ .../OHTableSecondaryPartScanTest.java | 105 +++++ .../hbase/util/ObHTableSecondaryPartUtil.java | 164 ++++++++ .../hbase/util/ObHTableTestUtil.java | 24 ++ .../hbase/util/TableTemplateManager.java | 276 +++++++++++++ .../oceanbase/hbase/util/TimeGenerator.java | 49 +++ 13 files changed, 1253 insertions(+), 726 deletions(-) delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java deleted file mode 100644 index 235b06a6..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartMcfTest.java +++ /dev/null @@ -1,355 +0,0 @@ -/*- - * #%L - * OBKV HBase Client Framework - * %% - * Copyright (C) 2025 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.hbase; - -import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.filter.CompareFilter; -import org.junit.*; - -import java.sql.Connection; - -import static org.apache.hadoop.hbase.util.Bytes.toBytes; - -public class OHTableSecondaryPartMcfTest { - private static String tableGroupNames[] = { - "test_table_group1", - "test_table_group2", - "test_table_group3", - "test_table_group4", - }; - private static String tableNames[][] = new String[][] { - {tableGroupNames[0] + "$cf1", tableGroupNames[0] + "$cf2"}, - {tableGroupNames[1] + "$cf1", tableGroupNames[1] + "$cf2"}, - {tableGroupNames[2] + "$cf1", tableGroupNames[2] + "$cf2"}, - {tableGroupNames[3] + "$cf1", tableGroupNames[3] + "$cf2"} - }; - private static String createTableStmts[][] = { - {"CREATE TABLE IF NOT EXISTS `" + tableNames[0][0] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[0] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");", - "CREATE TABLE IF NOT EXISTS `" + tableNames[0][1] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[0] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");" - }, - {"CREATE TABLE IF NOT EXISTS `" + tableNames[1][0] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[1] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");", - "CREATE TABLE IF NOT EXISTS `" + tableNames[1][1] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[1] +" PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");" - - }, - {"CREATE TABLE IF NOT EXISTS `" + tableNames[2][0] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[2] +" PARTITION BY KEY(`K`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;", - "CREATE TABLE IF NOT EXISTS `" + tableNames[2][1] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[2] +" PARTITION BY KEY(`K`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;" - - }, - {"CREATE TABLE IF NOT EXISTS `" + tableNames[3][0] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[3] +" PARTITION BY KEY(`K_PREFIX`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;", - "CREATE TABLE IF NOT EXISTS `" + tableNames[3][1] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") TABLEGROUP = "+ tableGroupNames[3] +" PARTITION BY KEY(`K_PREFIX`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;" - - } - }; - - public static void createTableGroups() throws Exception { - System.out.print("create table group start"); - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableGroupNames.length; i++) { - System.out.print("."); - String stmt = "CREATE TABLEGROUP IF NOT EXISTS `"+ tableGroupNames[i] +"` SHARDING = 'ADAPTIVE';"; - conn.createStatement().execute(stmt); - } - System.out.println("end"); - } - - public static void dropTableGroups() throws Exception { - System.out.print("drop table group start"); - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableGroupNames.length; i++) { - System.out.print("."); - String stmt = "DROP TABLEGROUP IF EXISTS " + tableGroupNames[i] + ";"; - conn.createStatement().execute(stmt); - } - System.out.println("end"); - } - - public static void dropTables() throws Exception { - System.out.print("drop table start"); - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableNames.length; i++) { - for (int j = 0; j < tableNames[i].length; j++) { - System.out.print("."); - String stmt = "DROP TABLE IF EXISTS " + tableNames[i][j] + ";"; - conn.createStatement().execute(stmt); - } - } - System.out.println("end"); - } - - public static void createTables() throws Exception { - System.out.print("create table start"); - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < createTableStmts.length; i++) { - for (int j = 0; j < createTableStmts[i].length; j++) { - System.out.print("."); - conn.createStatement().execute(createTableStmts[i][j]); - } - } - System.out.println("end"); - } - - public static void truncateTables() throws Exception { - System.out.print("truncate table start"); - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableNames.length; i++) { - for (int j = 0; j < tableNames[i].length; j++) { - System.out.print("."); - String stmt = "TRUNCATE TABLE " + tableNames[i][j] + ";"; - conn.createStatement().execute(stmt); - } - } - System.out.println("end"); - } - - public static String getTableName(String input) throws Exception { - // 查找 '$' 的索引 - int index = input.indexOf('$'); - // 如果找到了 '$'，提取其前面的部分 - String result; - if (index != -1) { - result = input.substring(0, index); // 提取从开始到 '$' 的部分 - } else { - result = input; // 如果没有 '$' 则返回原字符串 - } - System.out.println("getTableName:" + result); - return result; - } - - public static String getColumnFamilyName(String input) throws Exception { - // 查找 '$' 的索引 - int index = input.indexOf('$'); - // 如果找到了 '$'，提取其后面的部分 - String result; - if (index != -1 && index + 1 < input.length()) { - result = input.substring(index + 1); // 提取从 '$' 后一个字符到结束的部分 - } else { - result = ""; // 如果没有 '$' 或 '$' 是最后一个字符，则返回空字符串 - } - System.out.println("getColumnFamilyName:" + result); - return result; - } - - @BeforeClass - public static void before() throws Exception { - createTableGroups(); - createTables(); - } - - @AfterClass - public static void finish() throws Exception { - dropTables(); - dropTableGroups(); - } - - @Before - public void prepareCase() throws Exception { - truncateTables(); - } - - @Test - public void testPut() throws Exception { - String key = "putKey"; - String value = "value"; - String column = "putColumn"; - long timestamp = System.currentTimeMillis(); - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); - hTable.init(); - Put put = new Put(toBytes(key)); - for (int j = 0; j < tableNames[i].length; j++) { - String family = getColumnFamilyName(tableNames[i][j]); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - } - hTable.put(put); - hTable.close(); - } - } - - @Test - public void testIncrement() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); - hTable.init(); - - String key = "Key"; - String column = "Column"; - Increment increment = new Increment(key.getBytes()); - for (int j = 0; j < tableNames[i].length; j++) { - String family = getColumnFamilyName(tableNames[i][j]); - increment.addColumn(family.getBytes(), column.getBytes(), 1L); - } - hTable.increment(increment); - - hTable.close(); - } - } - - @Test - public void testAppend() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); - hTable.init(); - - String key = "Key"; - String column = "Column"; - String value = "app"; - Append append = new Append(key.getBytes()); - for (int j = 0; j < tableNames[i].length; j++) { - String family = getColumnFamilyName(tableNames[i][j]); - KeyValue kv1 = new KeyValue(key.getBytes(), family.getBytes(), column.getBytes(), value.getBytes()); - append.add(kv1); - } - hTable.append(append); - - hTable.close(); - } - } - - @Test - public void testGet() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i][0])); - hTable.init(); - - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - for (int j = 0; j < tableNames[i].length; j++) { - String family = getColumnFamilyName(tableNames[i][j]); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - } - hTable.put(put); - - Get get = new Get(key.getBytes()); - get.setMaxVersions(Integer.MAX_VALUE); - for (int j = 0; j < tableNames[i].length; j++) { - String family = getColumnFamilyName(tableNames[i][j]); - get.addColumn(family.getBytes(), column.getBytes()); - } - Result r = hTable.get(get); - Assert.assertEquals(1, r.raw().length); - - hTable.close(); - } - } - -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java deleted file mode 100644 index 235e4375..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableSecondaryPartTest.java +++ /dev/null @@ -1,371 +0,0 @@ -/*- - * #%L - * OBKV HBase Client Framework - * %% - * Copyright (C) 2025 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.hbase; - -import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.filter.CompareFilter; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.*; - -import java.sql.Connection; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - -import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.getConnection; -import static org.apache.hadoop.hbase.util.Bytes.toBytes; -import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; - -public class OHTableSecondaryPartTest { - private static String tableNames[] = new String[] {"test$cf1", "test$cf2", "test$cf3", "test$cf4"}; - private static String createTableStmt1 = "CREATE TABLE IF NOT EXISTS `" + tableNames[0] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");"; - private static String createTableStmt2 = "CREATE TABLE IF NOT EXISTS `" + tableNames[1] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") PARTITION BY RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 3 (" + - " PARTITION `p0` VALUES LESS THAN (1728921600000)," + - " PARTITION `p1` VALUES LESS THAN (1729008000000)," + - " PARTITION `p2` VALUES LESS THAN (1729094400000)," + - " PARTITION `p3` VALUES LESS THAN MAXVALUE" + - ");"; - private static String createTableStmt3 = "CREATE TABLE IF NOT EXISTS `" + tableNames[2] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") PARTITION BY KEY(`K`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;"; - private static String createTableStmt4 = "CREATE TABLE IF NOT EXISTS `" + tableNames[3] + "` (" + - " `K` varbinary(1024) NOT NULL," + - " `Q` varbinary(1024) NOT NULL," + - " `T` bigint(20) NOT NULL," + - " `V` varbinary(1024) DEFAULT NULL," + - " `G` bigint(20) GENERATED ALWAYS AS (-T)," + - " K_PREFIX varbinary(1024) generated always as (substring(`K`, 1, 4))," + - " PRIMARY KEY (`K`, `Q`, `T`)" + - ") PARTITION BY KEY(`K_PREFIX`) SUBPARTITION BY RANGE COLUMNS(`G`) SUBPARTITION TEMPLATE (" + - " SUBPARTITION `p0` VALUES LESS THAN (1728921600000)," + - " SUBPARTITION `p1` VALUES LESS THAN (1729008000000)," + - " SUBPARTITION `p2` VALUES LESS THAN (1729094400000)," + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE" + - ") PARTITIONS 3;"; - private static String createTableStmts[] = new String[] {createTableStmt1, createTableStmt2, createTableStmt3, createTableStmt4}; - - public static void dropTables() throws Exception { - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableNames.length; i++) { - String stmt = "DROP TABLE IF EXISTS " + tableNames[i] + ";"; - conn.createStatement().execute(stmt); - System.out.println("drop table " + tableNames[i] + " done"); - } - } - - public static void createTables() throws Exception { - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < createTableStmts.length; i++) { - conn.createStatement().execute(createTableStmts[i]); - System.out.println("create table " + tableNames[i] + " done"); - } - } - - public static void truncateTables() throws Exception { - Connection conn = ObHTableTestUtil.getConnection(); - for (int i = 0; i < tableNames.length; i++) { - String stmt = "TRUNCATE TABLE " + tableNames[i] + ";"; - conn.createStatement().execute(stmt); - System.out.println("truncate table " + tableNames[i] + " done"); - } - } - - public static String getTableName(String input) throws Exception { - // 查找 '$' 的索引 - int index = input.indexOf('$'); - // 如果找到了 '$'，提取其前面的部分 - String result; - if (index != -1) { - result = input.substring(0, index); // 提取从开始到 '$' 的部分 - } else { - result = input; // 如果没有 '$' 则返回原字符串 - } - return result; - } - - public static String getColumnFamilyName(String input) throws Exception { - // 查找 '$' 的索引 - int index = input.indexOf('$'); - // 如果找到了 '$'，提取其后面的部分 - String result; - if (index != -1 && index + 1 < input.length()) { - result = input.substring(index + 1); // 提取从 '$' 后一个字符到结束的部分 - } else { - result = ""; // 如果没有 '$' 或 '$' 是最后一个字符，则返回空字符串 - } - return result; - } - - @BeforeClass - public static void before() throws Exception { - createTables(); - } - - @AfterClass - public static void finish() throws Exception { - dropTables(); - } - - @Before - public void prepareCase() throws Exception { - truncateTables(); - } - - @Test - public void testPut() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("put table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - System.out.println("put table " + tableNames[i] + " done"); - - hTable.close(); - } - } - - @Test - public void testIncrement() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("increment table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String key = "Key"; - String column1 = "Column1"; - String column2 = "Column2"; - Increment increment = new Increment(key.getBytes()); - increment.addColumn(family.getBytes(), column1.getBytes(), 1L); - increment.addColumn(family.getBytes(), column2.getBytes(), 1L); - hTable.increment(increment); - System.out.println("increment table " + tableNames[i] + " done"); - - hTable.close(); - } - } - - @Test - public void testAppend() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("append table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String key = "Key"; - String column1 = "Column1"; - String column2 = "Column2"; - String value = "app"; - Append append = new Append(key.getBytes()); - KeyValue kv1 = new KeyValue(key.getBytes(), family.getBytes(), column1.getBytes(), value.getBytes()); - KeyValue kv2 = new KeyValue(key.getBytes(), family.getBytes(), column2.getBytes(), value.getBytes()); - append.add(kv1); - append.add(kv2); - hTable.append(append); - System.out.println("append table " + tableNames[i] + " done"); - - hTable.close(); - } - } - - @Test - public void testCheckAndMutate() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("checkAndMutate table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - String newValue = "newValue"; - RowMutations mutations = new RowMutations(key.getBytes()); - Put put = new Put(key.getBytes()); - long timestamp = System.currentTimeMillis(); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - - Put newPut = new Put(key.getBytes()); - newPut.add(family.getBytes(), column.getBytes(), timestamp, toBytes(newValue)); - mutations.add(newPut); - hTable.checkAndMutate(key.getBytes(), family.getBytes(), - column.getBytes(), CompareFilter.CompareOp.EQUAL, value.getBytes(), - mutations); - System.out.println("checkAndMutate table " + tableNames[i] + " done"); - - hTable.close(); - } - } - - @Test - public void testGet() throws Exception { - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("put table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - System.out.println("put table " + tableNames[i] + " done"); - - System.out.println("get table " + tableNames[i] + " begin"); - Get get = new Get(key.getBytes()); - get.setMaxVersions(Integer.MAX_VALUE); - get.addColumn(family.getBytes(), column.getBytes()); - Result r = hTable.get(get); - Assert.assertEquals(1, r.raw().length); - System.out.println("get table " + tableNames[i] + " begin"); - - hTable.close(); - } - } - - @Test - public void testBatchGet() throws Exception { - long batchSize = 10; - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("put table " + tableNames[i] + " begin"); - String family = getColumnFamilyName(tableNames[i]); - String column = "putColumn"; - String value = "value"; - long timestamp = System.currentTimeMillis(); - for (int j = 0; j < batchSize; j++) { - String key = "putKey" + j; - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - } - System.out.println("put table " + tableNames[i] + " done"); - - System.out.println("get table " + tableNames[i] + " begin"); - List gets = new ArrayList<>(); - for (int j = 0; j < batchSize; j++) { - String key = "putKey" + j; - Get get = new Get(key.getBytes()); - get.setMaxVersions(Integer.MAX_VALUE); - get.addColumn(family.getBytes(), column.getBytes()); - gets.add(get); - } - Result[] results = hTable.get(gets); - for (Result result : results) { - for (Cell cell : result.listCells()) { - String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); - String V = Bytes.toString(CellUtil.cloneValue(cell)); - System.out.println("Column: " + Q + ", Value: " + V); - } - } - System.out.println("get table " + tableNames[i] + " begin"); - - hTable.close(); - } - } - - @Test - public void testScan() throws Exception { - long rowSize = 10; - for (int i = 0; i < tableNames.length; i++) { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableNames[i])); - hTable.init(); - - System.out.println("put table " + tableNames[i] + " begin"); - String key = "putKey"; - String family = getColumnFamilyName(tableNames[i]); - String column = "Column"; - String value = "value"; - long timestamp = System.currentTimeMillis(); - for (int j = 0; j < rowSize; j++) { - Put put = new Put(toBytes(key)); - String qualify = column + j; - put.add(family.getBytes(), qualify.getBytes(), timestamp, toBytes(value)); - hTable.put(put); - } - System.out.println("put table " + tableNames[i] + " done"); - - System.out.println("scan table " + tableNames[i] + " begin"); - Scan scan = new Scan(toBytes(key)); - scan.addFamily(family.getBytes()); - ResultScanner scanner = hTable.getScanner(scan); - int count = 0; - for (Result result : scanner) { - for (KeyValue keyValue : result.raw()) { - assertEquals(column + count, Bytes.toString(keyValue.getQualifier())); - assertEquals(value, Bytes.toString(keyValue.getValue())); - count++; - } - } - assertEquals(rowSize, count); - System.out.println("scan table " + tableNames[i] + " end"); - - hTable.close(); - } - } -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java new file mode 100644 index 00000000..d175a278 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java @@ -0,0 +1,103 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Append; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.junit.Assert.assertEquals; + +public class OHTableSecondaryPartAppendTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = null; + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testAppendImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + String value = "value"; + Append append = new Append(key.getBytes()); + KeyValue kv1 = new KeyValue(key.getBytes(), family.getBytes(), column1.getBytes(), value.getBytes()); + KeyValue kv2 = new KeyValue(key.getBytes(), family.getBytes(), column2.getBytes(), value.getBytes()); + append.add(kv1); + append.add(kv2); + hTable.append(append); + + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(2, cells.length); + + for (Cell cell : cells) { + if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column1)) { + assertEquals("valuevalue", Bytes.toString(CellUtil.cloneValue(cell))); + } else if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column2)) { + assertEquals("valuevalue", Bytes.toString(CellUtil.cloneValue(cell))); + } + } + hTable.close(); + } + + @Test + public void testAppend() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppendImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java new file mode 100644 index 00000000..f05772ab --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java @@ -0,0 +1,106 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableSecondaryPartBatchGetTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testBatchGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); + hTable.put(put); + + List gets = new ArrayList<>(); + Get get1 = new Get(key.getBytes()); + get1.addFamily(family.getBytes()); + gets.add(get1); + + Get get2 = new Get(key.getBytes()); + get2.addColumn(family.getBytes(), column1.getBytes()); + gets.add(get2); + + Get get3 = new Get(key.getBytes()); + get3.addColumn(family.getBytes(), column2.getBytes()); + gets.add(get3); + + + Result[] results = hTable.get(gets); + for (Result result : results) { + for (Cell cell : result.listCells()) { + String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); + String V = Bytes.toString(CellUtil.cloneValue(cell)); + System.out.println("Column: " + Q + ", Value: " + V); + } + } + } + + @Test + public void testBatchGet() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartBatchGetTest::testBatchGetImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java new file mode 100644 index 00000000..04c49fb1 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java @@ -0,0 +1,93 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableSecondaryPartCheckAndMutateTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testCheckAndMutateImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + String newValue = "newValue"; + RowMutations mutations = new RowMutations(key.getBytes()); + Put put = new Put(key.getBytes()); + long timestamp = System.currentTimeMillis(); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + + Put newPut = new Put(key.getBytes()); + newPut.add(family.getBytes(), column.getBytes(), timestamp, toBytes(newValue)); + mutations.add(newPut); + hTable.checkAndMutate(key.getBytes(), family.getBytes(), + column.getBytes(), CompareFilter.CompareOp.EQUAL, value.getBytes(), + mutations); + + hTable.close(); + } + + @Test + public void testCheckAndMutate() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateImpl); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java new file mode 100644 index 00000000..84a4ebf0 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -0,0 +1,123 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableSecondaryPartGetTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), toBytes(value)); + hTable.put(put); + + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(1, cells.length); + assertEquals(column, Bytes.toString(CellUtil.cloneQualifier(cells[0]))); + assertEquals("value", Bytes.toString(CellUtil.cloneValue(cells[0]))); + System.out.println("get table " + tableName + " done"); + + hTable.close(); + } + + public static void testMultiCFGetImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String column = "putColumn"; + + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + String value = family + "_value"; + long timestamp = System.currentTimeMillis(); + + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + + hTable.close(); + } + } + + + @Test + public void testGet() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartGetTest::testGetImpl); + } + + @Test + public void testMultiCFGet() throws Exception { + FOR_EACH(group2tableNames, OHTableSecondaryPartGetTest::testMultiCFGetImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java new file mode 100644 index 00000000..d4d7a4cc --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java @@ -0,0 +1,106 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Increment; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + +public class OHTableSecondaryPartIncrementTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = null; + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testIncrementImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); + hTable.put(put); + + Increment increment = new Increment(key.getBytes()); + increment.addColumn(family.getBytes(), column1.getBytes(), 1L); + increment.addColumn(family.getBytes(), column2.getBytes(), 1L); + hTable.increment(increment); + + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(2, cells.length); + for (Cell cell : cells) { + if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column1)) { + assertEquals("2", Bytes.toString(CellUtil.cloneValue(cell))); + } else if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column2)) { + assertEquals("2", Bytes.toString(CellUtil.cloneValue(cell))); + } + } + hTable.close(); + } + + @Test + public void testIncrement() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartIncrementTest::testIncrementImpl); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java new file mode 100644 index 00000000..add0d03a --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -0,0 +1,104 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.client.Put; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableSecondaryPartPutTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testPutImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + hTable.put(put); + + hTable.close(); + } + + public static void testMultiCFPutImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + long timestamp = System.currentTimeMillis(); + + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + } + hTable.put(put); + hTable.close(); + } + + @Test + public void testPut() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartPutTest::testPutImpl); + } + + @Test + public void testMultiCFPut() throws Exception { + FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMultiCFPutImpl); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java new file mode 100644 index 00000000..036d9789 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java @@ -0,0 +1,105 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + +public class OHTableSecondaryPartScanTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + + public static void testScanImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); + hTable.put(put); + + Scan scan = new Scan(key.getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (KeyValue keyValue : result.raw()) { + assertEquals(column1, Bytes.toString(keyValue.getQualifier())); + assertEquals("1", Bytes.toString(keyValue.getValue())); + count++; + } + } + assertEquals(1, count); + } + + @Test + public void testScan() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartScanTest::testScanImpl); + } + + @Test + public void testMultiCFScan() throws Exception { + // implement later + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java new file mode 100644 index 00000000..48535e96 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -0,0 +1,164 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + + +public class ObHTableSecondaryPartUtil { + public static void createTables(TableTemplateManager.TableType type, List tableNames, Map> group2tableNames, boolean printSql) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // single cf table + if (tableNames != null) { + createTables(conn, type, tableNames, printSql); + } + // multi cf table + if (group2tableNames != null) { + createTables(conn, type, group2tableNames, printSql); + } + } + + public static void createTables(Connection conn, TableTemplateManager.TableType type, List tableNames, boolean printSql) throws Exception { + // create single cf table + if (tableNames != null) { + TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); + String tableGroup = TableTemplateManager.getTableGroupName(type, false); + String tableGroupSql = TableTemplateManager.generateTableGroupSQL(tableGroup); + conn.createStatement().execute(tableGroupSql); + String tableName = TableTemplateManager.generateTableName(tableGroup, false, 1); + String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); + conn.createStatement().execute(sql); + tableNames.add(tableName); + System.out.println("============= create table: " + tableName + " table_group: " + getTableName(tableName) + " =============\n" + (printSql ? sql : "") + " \n============= done =============\n"); + } + } + + public static void createTables(Connection conn, TableTemplateManager.TableType type, Map> group2tableNames, boolean printSql) throws Exception { + if (group2tableNames != null) { + TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); + String tableGroup = TableTemplateManager.getTableGroupName(type, true); + String tableGroupSql = TableTemplateManager.generateTableGroupSQL(tableGroup); + conn.createStatement().execute(tableGroupSql); + group2tableNames.put(tableGroup, new LinkedList<>()); + for (int i = 1; i <= 3; ++i) { + String tableName = TableTemplateManager.generateTableName(tableGroup, true, i); + String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); + conn.createStatement().execute(sql); + group2tableNames.get(tableGroup).add(tableName); + System.out.println("============= create table: " + tableName + + " table_group: " + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + " \n============= done =============\n"); + } + } + } + + public static void truncateTables(List tableNames, Map> group2tableNames) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // truncate single cf table + truncateTables(conn, tableNames); + // truncate multi cf table + truncateTables(conn, group2tableNames); + } + + public static void truncateTables(Connection conn, List tableNames) throws Exception { + if (tableNames != null) { + for (int i = 0; i < tableNames.size(); i++) { + String stmt = "TRUNCATE TABLE " + tableNames.get(i) + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= truncate table " + tableNames.get(i) + " done ============="); + } + } + } + + public static void truncateTables(Connection conn, Map> group2tableNames) throws Exception { + if (group2tableNames != null) { + for (Map.Entry> entry : group2tableNames.entrySet()) { + for (String tableName : entry.getValue()) { + String stmt = "TRUNCATE TABLE " + tableName + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= truncate table " + tableName + " done ============="); + } + } + } + } + + + public static void dropTables(List tableNames, Map> group2tableNames) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + // drop single cf table + dropTables(conn, tableNames); + // drop multi cf table + dropTables(conn, group2tableNames); + } + + public static void dropTables(Connection conn, List tableNames) throws Exception { + if (tableNames != null) { + for (String tableName : tableNames) { + String stmt = "DROP TABLE IF EXISTS " + tableName + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= drop table " + tableName + " done ============="); + } + } + } + + public static void dropTables(Connection conn, Map> group2tableNames) throws Exception { + if (group2tableNames != null) { + for (Map.Entry> entry : group2tableNames.entrySet()) { + for (String tableName : entry.getValue()) { + String stmt = "DROP TABLE IF EXISTS " + tableName + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= drop table " + tableName + " done ============="); + } + String stmt = "DROP TABLEGROUP IF EXISTS " + entry.getKey() + ";"; + conn.createStatement().execute(stmt); + System.out.println("============= drop tablegroup " + entry.getKey() + " done ============="); + } + } + } + + + public static String getTableName(String input) throws Exception { + // 查找 '$' 的索引 + int index = input.indexOf('$'); + // 如果找到了 '$'，提取其前面的部分 + String result; + if (index != -1) { + result = input.substring(0, index); // 提取从开始到 '$' 的部分 + } else { + result = input; // 如果没有 '$' 则返回原字符串 + } + return result; + } + + public static String getColumnFamilyName(String input) throws Exception { + // 查找 '$' 的索引 + int index = input.indexOf('$'); + // 如果找到了 '$'，提取其后面的部分 + String result; + if (index != -1 && index + 1 < input.length()) { + result = input.substring(index + 1); // 提取从 '$' 后一个字符到结束的部分 + } else { + result = ""; // 如果没有 '$' 或 '$' 是最后一个字符，则返回空字符串 + } + return result; + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index 44583f74..584bdb39 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -156,4 +156,28 @@ static public Connection getConnection() { throw new RuntimeException(e); } } + + @FunctionalInterface + public interface CheckedConsumer { + void accept(T t) throws Exception; + default CheckedConsumer andThen(CheckedConsumer after) throws Exception { + if (after == null) { throw new NullPointerException();} + return (T t) -> { + accept(t); + after.accept(t); + }; + } + } + + public static void FOR_EACH(List tableNames, CheckedConsumer consumer) throws Exception { + for (String tableName : tableNames) { + consumer.accept(tableName); + } + } + + public static void FOR_EACH(Map> group2Tables, CheckedConsumer>> consumer) throws Exception { + for (Map.Entry> entry : group2Tables.entrySet()) { + consumer.accept(entry); + } + } } \ No newline at end of file diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java new file mode 100644 index 00000000..21c23081 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -0,0 +1,276 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + + +import java.util.EnumMap; +import java.util.Map; +public class TableTemplateManager { + public static final String TABLE_GROUP_PREFIX = "test_group_"; + public static final String COLUMN_FAMILY = "cf"; + public enum TableType { + NON_PARTITIONED_REGULAR, + NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_REGULAR, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_RANGE_KEY, // RANGE-KEY分区（使用K） + SECONDARY_PARTITIONED_RANGE_KEY_GEN, // RANGE-KEY分区（使用生成列） + SECONDARY_PARTITIONED_KEY_RANGE, // KEY-RANGE分区（使用K） + SECONDARY_PARTITIONED_KEY_RANGE_GEN, // KEY-RANGE分区（使用生成列） + SECONDARY_PARTITIONED_TIME_RANGE_KEY, // 时序表RANGE-KEY + SECONDARY_PARTITIONED_TIME_KEY_RANGE // 时序表KEY-RANGE + } + + private static final Map SQL_TEMPLATES = new EnumMap<>(TableType.class); + + static { + // 普通表非分区表模版 + SQL_TEMPLATES.put(TableType.NON_PARTITIONED_REGULAR, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s"); + // 时序表非分区表模版 + SQL_TEMPLATES.put(TableType.NON_PARTITIONED_TIME_SERIES, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s"); + // 普通表一级分区模板 + SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); + // 时序表一级分区模板 + SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_TIME_SERIES, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); + // 普通表RANGE-KEY分区（使用K） + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE)"); + + // 合并GEN类型的注释处理 + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 普通表KEY-RANGE分区（使用K） + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 普通表KEY-RANGE分区（使用生成列） + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 时序表RANGE-KEY分区 + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + // 时序表KEY-RANGE分区 + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_KEY_RANGE, + "CREATE TABLE `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE)"); + } + + public static String getCreateTableSQL(TableType type, String tableName, + TimeGenerator.TimeRange timeRange) { + String template = SQL_TEMPLATES.get(type); + Object[] params; + String tableGroup = extractTableGroup(tableName); + + switch (type) { + case NON_PARTITIONED_REGULAR: + case NON_PARTITIONED_TIME_SERIES: + params = new Object[]{tableName, tableGroup}; + break; + case SINGLE_PARTITIONED_REGULAR: + case SINGLE_PARTITIONED_TIME_SERIES: // 合并相同处理逻辑 + params = new Object[]{tableName, tableGroup, 97}; + break; + case SECONDARY_PARTITIONED_RANGE_KEY: + case SECONDARY_PARTITIONED_RANGE_KEY_GEN: + case SECONDARY_PARTITIONED_KEY_RANGE: + case SECONDARY_PARTITIONED_KEY_RANGE_GEN: + boolean isGen = type.name().contains("GEN"); + params = new Object[]{ + tableName, + getGeneratedColumn(type), + tableGroup, + isGen ? "K_PREFIX" : "K", + 97, + timeRange.lowerBound1(), + timeRange.lowerBound1() + 86400000, + timeRange.lowerBound1() + 172800000 + }; + break; + case SECONDARY_PARTITIONED_TIME_RANGE_KEY: + case SECONDARY_PARTITIONED_TIME_KEY_RANGE: // 合并时序表处理 + params = new Object[]{ + tableName, + "", + tableGroup, + "K", + 97, + timeRange.lowerBound1(), + timeRange.lowerBound1() + 86400000, + timeRange.lowerBound1() + 172800000 + }; + break; + default: + throw new IllegalArgumentException("Unsupported table type"); + } + + return String.format(template, params); + } + + private static String getGeneratedColumn(TableType type) { + StringBuilder sb = new StringBuilder(); + boolean needsKPrefix = type.name().startsWith("SECONDARY_PARTITIONED") + && !type.name().contains("TIME") + && type.name().contains("GEN"); + + if (needsKPrefix) { + sb.append(",\n K_PREFIX varbinary(1024) GENERATED ALWAYS AS (substring(`K`, 1, 4))"); + } + return sb.toString(); + } + + private static String getPartitionStrategy(TableType type) { + if (type.name().contains("RANGE_KEY")) { + return type.name().contains("GEN") + ? "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 97" + : "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 97"; + } + if (type.name().contains("KEY_RANGE")) { + return type.name().contains("GEN") + ? "KEY(`K_PREFIX`) PARTITIONS 97 SUBPARTITION BY RANGE COLUMNS(`G`)" + : "KEY(`K`) PARTITIONS 97 SUBPARTITION BY RANGE COLUMNS(`G`)"; + } + return ""; + } + + public static String generateTableGroupSQL(String tableGroup) { + return String.format("CREATE TABLEGROUP IF NOT EXISTS %s SHARDING = 'ADAPTIVE'", tableGroup); + } + + public static String getTableGroupName(TableTemplateManager.TableType type, boolean multiCf) { + return TABLE_GROUP_PREFIX + type.name().toLowerCase() + (multiCf ? "_mcf" : ""); + } + + public static String generateTableName(String tableGroup, boolean multiCf, int cfIndex) { + return String.format("%s$%s", + tableGroup, + multiCf ? COLUMN_FAMILY + cfIndex : COLUMN_FAMILY); + } + + public static String extractTableGroup(String tableName) { + int dollarIndex = tableName.indexOf('$'); + if (dollarIndex > 0) { + return tableName.substring(0, dollarIndex); + } + throw new IllegalArgumentException("Invalid table name: " + tableName); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java b/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java new file mode 100644 index 00000000..758dedb3 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java @@ -0,0 +1,49 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +public class TimeGenerator { + public static class TimeRange { + private final long lowerBound1; + private final long lowerBound2; + private final long upperBound1; + private final long upperBound2; + + public TimeRange(long lowerBound1, long lowerBound2, long upperBound1, long upperBound2) { + this.lowerBound1 = lowerBound1; + this.lowerBound2 = lowerBound2; + this.upperBound1 = upperBound1; + this.upperBound2 = upperBound2; + } + + public long lowerBound1() { return lowerBound1; } + public long lowerBound2() { return lowerBound2; } + public long upperBound1() { return upperBound1; } + public long upperBound2() { return upperBound2; } + } + + public static TimeRange generateTestTimeRange() { + long current = System.currentTimeMillis(); + return new TimeRange( + current - 86400000, // 24小时前 + current, + current + 86400000, // 24小时后 + current + 172800000 // 48小时后 + ); + } +} From 430b00aba7fb1a6e099bf2fbc1ed257822d81dbb Mon Sep 17 00:00:00 2001 From: WeiXinChan Date: Tue, 11 Mar 2025 16:18:37 +0800 Subject: [PATCH 08/40] open distributed execute before tests (cherry picked from commit ac17bc5764b541a8b442724fbbbff0adf200fd2e) --- .../OHTableSecondaryPartAppendTest.java | 2 ++ .../OHTableSecondaryPartBatchGetTest.java | 2 ++ ...HTableSecondaryPartCheckAndMutateTest.java | 2 ++ .../OHTableSecondaryPartGetTest.java | 2 ++ .../OHTableSecondaryPartIncrementTest.java | 2 ++ .../OHTableSecondaryPartPutTest.java | 4 ++- .../OHTableSecondaryPartScanTest.java | 2 ++ .../hbase/util/ObHTableSecondaryPartUtil.java | 12 +++++++ .../hbase/util/ObHTableTestUtil.java | 11 ++++++ .../hbase/util/TableTemplateManager.java | 35 ++++++++++--------- 10 files changed, 56 insertions(+), 18 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java index d175a278..32bc5ef1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java @@ -49,6 +49,7 @@ public class OHTableSecondaryPartAppendTest { @BeforeClass public static void before() throws Exception { + openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { createTables(type, tableNames, group2tableNames, true); } @@ -56,6 +57,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { + closeDistributedExecute(); dropTables(tableNames, group2tableNames); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java index f05772ab..2f59b330 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java @@ -45,6 +45,7 @@ public class OHTableSecondaryPartBatchGetTest { @BeforeClass public static void before() throws Exception { + openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { createTables(type, tableNames, group2tableNames, true); } @@ -52,6 +53,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { + closeDistributedExecute(); dropTables(tableNames, group2tableNames); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java index 04c49fb1..9c815871 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java @@ -45,6 +45,7 @@ public class OHTableSecondaryPartCheckAndMutateTest { @BeforeClass public static void before() throws Exception { + openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { createTables(type, tableNames, group2tableNames, true); } @@ -52,6 +53,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { + closeDistributedExecute(); dropTables(tableNames, group2tableNames); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java index 84a4ebf0..3fadf09c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -35,6 +35,7 @@ import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.SECONDARY_PARTITIONED_KEY_RANGE; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.assertEquals; @@ -53,6 +54,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { + closeDistributedExecute(); dropTables(tableNames, group2tableNames); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java index d4d7a4cc..26af03f8 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java @@ -49,6 +49,7 @@ public class OHTableSecondaryPartIncrementTest { @BeforeClass public static void before() throws Exception { + openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { createTables(type, tableNames, group2tableNames, true); } @@ -56,6 +57,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { + closeDistributedExecute(); dropTables(tableNames, group2tableNames); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java index add0d03a..45beb82b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -43,6 +43,7 @@ public class OHTableSecondaryPartPutTest { @BeforeClass public static void before() throws Exception { + openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { createTables(type, tableNames, group2tableNames, true); } @@ -50,7 +51,8 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { - dropTables(tableNames, group2tableNames); + closeDistributedExecute(); + dropTables(tableNames, group2tableNames); } @Before diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java index 036d9789..561a34f5 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java @@ -48,6 +48,7 @@ public class OHTableSecondaryPartScanTest { @BeforeClass public static void before() throws Exception { + openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { createTables(type, tableNames, group2tableNames, true); } @@ -55,6 +56,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { + closeDistributedExecute(); dropTables(tableNames, group2tableNames); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index 48535e96..59f95f3a 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -25,6 +25,18 @@ public class ObHTableSecondaryPartUtil { + public static void openDistributedExecute() throws Exception { + Connection conn = ObHTableTestUtil.getSysConnection(); + String stmt = "ALTER SYSTEM SET _obkv_feature_mode = 'distributed_execute=on';"; + conn.createStatement().execute(stmt); + } + + public static void closeDistributedExecute() throws Exception { + Connection conn = ObHTableTestUtil.getSysConnection(); + String stmt = "ALTER SYSTEM SET _obkv_feature_mode = 'distributed_execute=off';"; + conn.createStatement().execute(stmt); + } + public static void createTables(TableTemplateManager.TableType type, List tableNames, Map> group2tableNames, boolean printSql) throws Exception { Connection conn = ObHTableTestUtil.getConnection(); // single cf table diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index 584bdb39..b5e4be84 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -157,6 +157,17 @@ static public Connection getConnection() { } } + static public Connection getSysConnection() { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + Connection conn = DriverManager.getConnection(JDBC_URL, SYS_USER_NAME, SYS_PASSWORD); + + return conn; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @FunctionalInterface public interface CheckedConsumer { void accept(T t) throws Exception; diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index 21c23081..6a257fdc 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -21,6 +21,7 @@ import java.util.EnumMap; import java.util.Map; public class TableTemplateManager { + public static final long PART_NUM = 3; public static final String TABLE_GROUP_PREFIX = "test_group_"; public static final String COLUMN_FAMILY = "cf"; public enum TableType { @@ -41,7 +42,7 @@ public enum TableType { static { // 普通表非分区表模版 SQL_TEMPLATES.put(TableType.NON_PARTITIONED_REGULAR, - "CREATE TABLE `%s` (\n" + + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + @@ -50,7 +51,7 @@ public enum TableType { ") TABLEGROUP = %s"); // 时序表非分区表模版 SQL_TEMPLATES.put(TableType.NON_PARTITIONED_TIME_SERIES, - "CREATE TABLE `%s` (\n" + + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + @@ -59,7 +60,7 @@ public enum TableType { ") TABLEGROUP = %s"); // 普通表一级分区模板 SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR, - "CREATE TABLE `%s` (\n" + + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + @@ -68,7 +69,7 @@ public enum TableType { ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); // 时序表一级分区模板 SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_TIME_SERIES, - "CREATE TABLE `%s` (\n" + + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + @@ -77,7 +78,7 @@ public enum TableType { ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); // 普通表RANGE-KEY分区（使用K） SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY, - "CREATE TABLE `%s` (\n" + + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + @@ -93,7 +94,7 @@ public enum TableType { // 合并GEN类型的注释处理 SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN, - "CREATE TABLE `%s` (\n" + + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + @@ -109,7 +110,7 @@ public enum TableType { // 普通表KEY-RANGE分区（使用K） SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE, - "CREATE TABLE `%s` (\n" + + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + @@ -126,7 +127,7 @@ public enum TableType { // 普通表KEY-RANGE分区（使用生成列） SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN, - "CREATE TABLE `%s` (\n" + + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + @@ -143,7 +144,7 @@ public enum TableType { // 时序表RANGE-KEY分区 SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, - "CREATE TABLE `%s` (\n" + + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + @@ -159,7 +160,7 @@ public enum TableType { // 时序表KEY-RANGE分区 SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_KEY_RANGE, - "CREATE TABLE `%s` (\n" + + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + @@ -188,7 +189,7 @@ public static String getCreateTableSQL(TableType type, String tableName, break; case SINGLE_PARTITIONED_REGULAR: case SINGLE_PARTITIONED_TIME_SERIES: // 合并相同处理逻辑 - params = new Object[]{tableName, tableGroup, 97}; + params = new Object[]{tableName, tableGroup, PART_NUM}; break; case SECONDARY_PARTITIONED_RANGE_KEY: case SECONDARY_PARTITIONED_RANGE_KEY_GEN: @@ -200,7 +201,7 @@ public static String getCreateTableSQL(TableType type, String tableName, getGeneratedColumn(type), tableGroup, isGen ? "K_PREFIX" : "K", - 97, + PART_NUM, timeRange.lowerBound1(), timeRange.lowerBound1() + 86400000, timeRange.lowerBound1() + 172800000 @@ -213,7 +214,7 @@ public static String getCreateTableSQL(TableType type, String tableName, "", tableGroup, "K", - 97, + PART_NUM, timeRange.lowerBound1(), timeRange.lowerBound1() + 86400000, timeRange.lowerBound1() + 172800000 @@ -241,13 +242,13 @@ private static String getGeneratedColumn(TableType type) { private static String getPartitionStrategy(TableType type) { if (type.name().contains("RANGE_KEY")) { return type.name().contains("GEN") - ? "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS 97" - : "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS 97"; + ? "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS " + PART_NUM + : "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS "+ PART_NUM; } if (type.name().contains("KEY_RANGE")) { return type.name().contains("GEN") - ? "KEY(`K_PREFIX`) PARTITIONS 97 SUBPARTITION BY RANGE COLUMNS(`G`)" - : "KEY(`K`) PARTITIONS 97 SUBPARTITION BY RANGE COLUMNS(`G`)"; + ? "KEY(`K_PREFIX`) PARTITIONS "+ PART_NUM +" SUBPARTITION BY RANGE COLUMNS(`G`)" + : "KEY(`K`) PARTITIONS "+ PART_NUM +" SUBPARTITION BY RANGE COLUMNS(`G`)"; } return ""; } From 09e45da70487494d65dee41303854fc24c01b086 Mon Sep 17 00:00:00 2001 From: WeiXinChan Date: Tue, 11 Mar 2025 17:35:15 +0800 Subject: [PATCH 09/40] fix test (cherry picked from commit 2ef9df41b72b1a8a0ec2d4549d4b991ad1abfe80) --- .../oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java index 3fadf09c..386a91b3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -35,7 +35,6 @@ import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; -import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.SECONDARY_PARTITIONED_KEY_RANGE; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.assertEquals; @@ -47,6 +46,7 @@ public class OHTableSecondaryPartGetTest { @BeforeClass public static void before() throws Exception { + openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { createTables(type, tableNames, group2tableNames, true); } From da2c08dd894af5cec0581c41ffb837c9f5c1de98 Mon Sep 17 00:00:00 2001 From: vanson <43193589+WeiXinChan@users.noreply.github.com> Date: Tue, 11 Mar 2025 17:36:02 +0800 Subject: [PATCH 10/40] Merge pull request #166 from WeiXinChan/tmp11 open distributed execute before tests (cherry picked from commit 5d77cf92fd649c9025f63d222ba2bf56570ec747) From 8e9eb4534738639ad9509ff8de96dd94cfa28b93 Mon Sep 17 00:00:00 2001 From: stuBirdFly <1065492934@qq.com> Date: Thu, 13 Mar 2025 20:14:55 +0800 Subject: [PATCH 11/40] =?UTF-8?q?add=20append=E3=80=81increment=20and=20ch?= =?UTF-8?q?eckAndMutate=20test=20case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 7c3b9a8b929383f0cd4cf3c282a80428d864791b) --- .../com/alipay/oceanbase/hbase/OHTable.java | 11 +- .../OHTableSecondaryPartAppendTest.java | 199 ++++++++--- .../OHTableSecondaryPartBatchGetTest.java | 6 +- ...HTableSecondaryPartCheckAndMutateTest.java | 331 ++++++++++++++++-- .../OHTableSecondaryPartGetTest.java | 6 +- .../OHTableSecondaryPartIncrementTest.java | 182 +++++++--- .../OHTableSecondaryPartPutTest.java | 8 +- .../OHTableSecondaryPartScanTest.java | 13 +- .../hbase/util/ObHTableSecondaryPartUtil.java | 80 +++-- .../hbase/util/ObHTableTestUtil.java | 13 +- .../hbase/util/TableTemplateManager.java | 13 + .../oceanbase/hbase/util/TimeGenerator.java | 27 +- 12 files changed, 714 insertions(+), 175 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 732f463d..ae400af8 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1467,9 +1467,12 @@ public Result append(Append append) throws IOException { ObTableQueryAndMutateRequest request = buildObTableQueryAndMutateRequest(obTableQuery, batchOperation, getTargetTableName(tableNameString, Bytes.toString(f), configuration)); - request.setReturningAffectedEntity(true); + request.setReturningAffectedEntity(append.isReturnResults()); ObTableQueryAndMutateResult result = (ObTableQueryAndMutateResult) obTableClient .execute(request); + if (!append.isReturnResults()) { + return null; + } ObTableQueryResult queryResult = result.getAffectedEntity(); List keyValues = new ArrayList(); for (List row : queryResult.getPropertiesRows()) { @@ -1516,9 +1519,12 @@ public Result increment(Increment increment) throws IOException { ObTableQueryAndMutateRequest request = buildObTableQueryAndMutateRequest(obTableQuery, batch, getTargetTableName(tableNameString, Bytes.toString(f), configuration)); - request.setReturningAffectedEntity(true); + request.setReturningAffectedEntity(increment.isReturnResults()); ObTableQueryAndMutateResult result = (ObTableQueryAndMutateResult) obTableClient .execute(request); + if (!increment.isReturnResults()) { + return null; + } ObTableQueryResult queryResult = result.getAffectedEntity(); List keyValues = new ArrayList(); for (List row : queryResult.getPropertiesRows()) { @@ -1527,7 +1533,6 @@ public Result increment(Increment increment) throws IOException { long t = (Long) row.get(2).getValue(); byte[] v = (byte[]) row.get(3).getValue(); KeyValue kv = new KeyValue(k, f, q, t, v); - keyValues.add(kv); } return Result.create(keyValues); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java index 32bc5ef1..4156291e 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java @@ -15,42 +15,40 @@ * #L% */ - package com.alipay.oceanbase.hbase.secondary; +import com.alipay.oceanbase.hbase.OHTable; import com.alipay.oceanbase.hbase.OHTableClient; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.util.TableTemplateManager; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.Append; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.AfterClass; import org.junit.Test; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class OHTableSecondaryPartAppendTest { - private static List tableNames = new LinkedList(); + private static List tableNames = new LinkedList(); private static Map> group2tableNames = null; - @BeforeClass public static void before() throws Exception { openDistributedExecute(); - for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + for (TableTemplateManager.TableType type : NORMAL_TABLES) { createTables(type, tableNames, group2tableNames, true); } } @@ -58,48 +56,161 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); + // dropTables(tableNames, group2tableNames); } @Before public void prepareCase() throws Exception { truncateTables(tableNames, group2tableNames); } - - public static void testAppendImpl(String tableName) throws Exception { + + private static void assertNullResult(Result result) throws Exception { + assertTrue("expected null result but received a non-null result", result == null); + } + + private static void testAppend(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + byte[] ROW = "appendKey".getBytes(); + byte[] v1 = Bytes.toBytes("42"); + byte[] v2 = Bytes.toBytes("23"); + byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c") }; + Append a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[0], v1); + a.add(FAMILY, QUALIFIERS[1], v2); + a.setReturnResults(false); + assertNullResult(hTable.append(a)); + + a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[0], v2); + a.add(FAMILY, QUALIFIERS[1], v1); + a.add(FAMILY, QUALIFIERS[2], v2); + Result r = hTable.append(a); + assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0]))); + assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1]))); + // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct + assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2]))); + assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(), + r.getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp()); + + Get get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); + Result result = hTable.get(get); + assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[0]).size()); + assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[1]).size()); + assertEquals(1, result.getColumnCells(FAMILY, QUALIFIERS[2]).size()); + assertEquals( + 0, + Bytes.compareTo(Bytes.add(v1, v2), result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0) + .getValue())); + assertEquals(0, + Bytes.compareTo(v2, result.getColumnCells(FAMILY, QUALIFIERS[2]).get(0).getValue())); + + hTable.close(); + } + + private static void testAppendBorder(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); - String family = getColumnFamilyName(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String column2 = "putColumn2"; - String value = "value"; - Append append = new Append(key.getBytes()); - KeyValue kv1 = new KeyValue(key.getBytes(), family.getBytes(), column1.getBytes(), value.getBytes()); - KeyValue kv2 = new KeyValue(key.getBytes(), family.getBytes(), column2.getBytes(), value.getBytes()); - append.add(kv1); - append.add(kv2); - hTable.append(append); - - Get get = new Get(key.getBytes()); - get.addFamily(family.getBytes()); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + byte[] ROW = "appendKey".getBytes(); + byte[] v1 = Bytes.toBytes("ab"); + byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c") }; + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIERS[1], v1); + hTable.put(put); + Append a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[1], v1); + a.add(FAMILY, QUALIFIERS[2], "".getBytes()); + hTable.append(a); + Get get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); Result result = hTable.get(get); - Cell[] cells = result.rawCells(); - assertEquals(2, cells.length); - - for (Cell cell : cells) { - if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column1)) { - assertEquals("valuevalue", Bytes.toString(CellUtil.cloneValue(cell))); - } else if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column2)) { - assertEquals("valuevalue", Bytes.toString(CellUtil.cloneValue(cell))); - } + assertEquals(3, result.size()); + + a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[2], v1); + a.add(FAMILY, QUALIFIERS[2], "".getBytes()); + hTable.append(a); + get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); + result = hTable.get(get); + assertEquals(4, result.size()); + + byte[] randomBytes = new byte[1025]; + Random random = new Random(); + random.nextBytes(randomBytes); + a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[2], randomBytes); + try { + hTable.append(a); + } catch (IOException e) { + assertTrue(e.getCause().getMessage().contains("Data too long for column 'V'")); + } + + hTable.close(); + + } + + private static void testAppendCon(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + byte[] v = "a".getBytes(); + byte[] expect = "a".getBytes(); + ThreadPoolExecutor threadPoolExecutor = OHTable.createDefaultThreadPoolExecutor(1, 100,100); + AtomicInteger atomicInteger = new AtomicInteger(0); + CountDownLatch countDownLatch = new CountDownLatch(100); + for (int i = 0; i < 100; i++) { + Append append = new Append(ROW); + append.add(FAMILY, column.getBytes(), v); + threadPoolExecutor.submit(() -> { + try { + hTable.append(append); + atomicInteger.incrementAndGet(); + } catch (Exception e) { + if (!e.getCause().getMessage().contains("OB_TRY_LOCK_ROW_CONFLICT") && !e.getCause().getMessage().contains("OB_TIMEOUT")) { + throw new RuntimeException(e); + } + } finally { + countDownLatch.countDown(); + } + }); } + countDownLatch.await(100000, TimeUnit.MILLISECONDS); + for (int i = 0; i < atomicInteger.get() - 1; i++) { + expect = Bytes.add(expect, v); + } + Get get = new Get(ROW); + get.setMaxVersions(1); + get.addColumn(FAMILY, column.getBytes()); + Result result = hTable.get(get); + assertEquals(0, Bytes.compareTo(expect, result.getColumnCells(FAMILY, column.getBytes()).get(0).getValue())); hTable.close(); } - + + @Test + public void testAppend() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppend); + } + + @Test + public void testBorderAppend() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppendBorder); + } + @Test - public void testAppend() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppendImpl); + public void testAppendConcurrency() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppendCon); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java index 2f59b330..93c4a459 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; import org.junit.Before; @@ -35,11 +34,10 @@ import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; import static org.apache.hadoop.hbase.util.Bytes.toBytes; -import static org.junit.Assert.assertEquals; public class OHTableSecondaryPartBatchGetTest { - private static List tableNames = new LinkedList(); + private static List tableNames = new LinkedList(); private static Map> group2tableNames = new LinkedHashMap<>(); @@ -102,7 +100,7 @@ public static void testBatchGetImpl(String tableName) throws Exception { } @Test - public void testBatchGet() throws Exception { + public void testBatchGet() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartBatchGetTest::testBatchGetImpl); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java index 9c815871..0f7c64a1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java @@ -22,25 +22,28 @@ import com.alipay.oceanbase.hbase.util.TableTemplateManager; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; +import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; -import static org.apache.hadoop.hbase.util.Bytes.toBytes; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; public class OHTableSecondaryPartCheckAndMutateTest { - private static List tableNames = new LinkedList(); + private static List tableNames = new LinkedList(); private static Map> group2tableNames = new LinkedHashMap<>(); + private static byte [] ROW = Bytes.toBytes("testRow"); + private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); + private static byte [] VALUE_1 = Bytes.toBytes("testValue"); + private static byte [] ROW_1 = Bytes.toBytes("testRow1"); + private static byte [] VALUE_2 = Bytes.toBytes("abcd"); @BeforeClass @@ -54,42 +57,314 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); +// dropTables(tableNames, group2tableNames); } @Before public void prepareCase() throws Exception { truncateTables(tableNames, group2tableNames); } - - public static void testCheckAndMutateImpl(String tableName) throws Exception { + + public static void testCheckAndMutate(String tableName) throws Throwable { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); - String family = getColumnFamilyName(tableName); - String key = "putKey"; - String column = "putColumn"; - String value = "value"; - String newValue = "newValue"; - RowMutations mutations = new RowMutations(key.getBytes()); - Put put = new Put(key.getBytes()); - long timestamp = System.currentTimeMillis(); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + final byte[] ROW = Bytes.toBytes("12345"); + final byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + Put put = new Put(ROW); + put.add(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); + put.add(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); + put.add(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); hTable.put(put); + // get row back and assert the values + Get get = new Get(ROW); + Result result = hTable.get(get); + assertTrue("Column A value should be a", + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("A"))).equals("a")); + assertTrue("Column B value should be b", + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))).equals("b")); + assertTrue("Column C value should be c", + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("C"))).equals("c")); + + // put the same row again with C column deleted + RowMutations rm = new RowMutations(ROW); + put = new Put(ROW); + put.add(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); + put.add(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); + rm.add(put); + Delete del = new Delete(ROW); + del.deleteColumn(FAMILY, Bytes.toBytes("C")); + rm.add(del); + boolean res = hTable.checkAndMutate(ROW, FAMILY, Bytes.toBytes("A"), CompareFilter.CompareOp.EQUAL, + Bytes.toBytes("a"), rm); + assertTrue(res); - Put newPut = new Put(key.getBytes()); - newPut.add(family.getBytes(), column.getBytes(), timestamp, toBytes(newValue)); - mutations.add(newPut); - hTable.checkAndMutate(key.getBytes(), family.getBytes(), - column.getBytes(), CompareFilter.CompareOp.EQUAL, value.getBytes(), - mutations); + // get row back and assert the values + get = new Get(ROW); + get.addFamily(FAMILY); + result = hTable.get(get); + assertTrue("Column A value should be a", + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("A"))).equals("a")); + assertTrue("Column B value should be b", + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))).equals("b")); + assertTrue("Column C should not exist", + result.getValue(FAMILY, Bytes.toBytes("C")) == null); + //Test that we get a hTable level exception + try { + Put p = new Put(ROW); + p.add(new byte[]{'b', 'o', 'g', 'u', 's'}, new byte[]{'A'}, new byte[0]); + rm = new RowMutations(ROW); + rm.add(p); + hTable.checkAndMutate(ROW, FAMILY, Bytes.toBytes("A"), CompareFilter.CompareOp.EQUAL, + Bytes.toBytes("a"), rm); + fail("Expected NoSuchColumnFamilyException"); + } catch (RetriesExhaustedWithDetailsException e) { + try { + throw e.getCause(0); + } catch (NoSuchColumnFamilyException e1) { + // expected + } + } hTable.close(); } - + + public static void testCheckAndMutateDiffRow(String tableName) throws Throwable { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + byte[] family = getColumnFamilyName(tableName).getBytes(); + byte[] row1 = Bytes.toBytes("row1"); + byte[] qualifier = Bytes.toBytes("q1"); + byte[] value1 = Bytes.toBytes("value1"); + + Put put = new Put(row1); + put.addColumn(family, qualifier, value1); + hTable.put(put); + + Result result = hTable.get(new Get(row1)); + assertArrayEquals("the value of column q in row1 should be value1", + value1, result.getValue(family, qualifier)); + + byte[] row2 = Bytes.toBytes("row2"); + byte[] value2 = Bytes.toBytes("value2"); + RowMutations mutations = new RowMutations(row2); + put = new Put(row2); + put.addColumn(family, qualifier, value2); + mutations.add(put); + + // check row1 and put row2 + assertTrue(hTable.checkAndMutate(row1, family, qualifier, + CompareFilter.CompareOp.GREATER, value2, mutations)); + + result = hTable.get(new Get(row2)); + assertArrayEquals("the value of column q in row2 should be value2", + value2, result.getValue(family, qualifier)); + hTable.close(); + } + + public static void testCheckAndPut(String tableName) throws Throwable { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte [] FAMILY = getColumnFamilyName(tableName).getBytes(); + + Put put1 = new Put(ROW); + put1.add(FAMILY, QUALIFIER, VALUE_1); + + // row doesn't exist, so using non-null value should be considered "not match". + boolean ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE_1, put1); + assertEquals(ok, false); + + // row doesn't exist, so using "null" to check for existence should be considered "match". + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1); + assertEquals(ok, true); + + // row now exists, so using "null" to check for existence should be considered "not match". + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, null, put1); + assertEquals(ok, false); + + Put put2 = new Put(ROW); + put2.add(FAMILY, QUALIFIER, VALUE_2); + + // row now exists, use the matching value to check + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE_1, put2); + assertEquals(ok, true); + + Put put3 = new Put(ROW_1); + put3.add(FAMILY, QUALIFIER, VALUE_1); + + // try to do CheckAndPut on different rows + try { + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE_2, put3); + fail("trying to check and modify different rows should have failed."); + } catch(Exception e) {} + hTable.close(); + } + + public static void testCheckAndPutWithCompareOp(String tableName) throws Throwable { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + final byte [] value1 = Bytes.toBytes("aaaa"); + final byte [] value2 = Bytes.toBytes("bbbb"); + final byte [] value3 = Bytes.toBytes("cccc"); + final byte [] value4 = Bytes.toBytes("dddd"); + byte [] FAMILY = getColumnFamilyName(tableName).getBytes(); + + Put put2 = new Put(ROW); + put2.add(FAMILY, QUALIFIER, value2); + + Put put3 = new Put(ROW); + put3.add(FAMILY, QUALIFIER, value3); + + // row doesn't exist, so using "null" to check for existence should be considered "match". + boolean ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, null, put2); + assertEquals(ok, true); + + // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL + // turns out "match" + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value1, put2); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value1, put2); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value1, put2); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value1, put2); + assertEquals(ok, true); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value1, put2); + assertEquals(ok, true); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value1, put3); + assertEquals(ok, true); + + // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL + // turns out "match" + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value4, put3); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value4, put3); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value4, put3); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value4, put3); + assertEquals(ok, true); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value4, put3); + assertEquals(ok, true); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value4, put2); + assertEquals(ok, true); + + // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL + // turns out "match" + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value2, put2); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value2, put2); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value2, put2); + assertEquals(ok, false); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value2, put2); + assertEquals(ok, true); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value2, put2); + assertEquals(ok, true); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value2, put3); + assertEquals(ok, true); + } + + public static void testCheckAndDeleteWithCompareOp(String tableName) throws Throwable { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + final byte [] value1 = Bytes.toBytes("aaaa"); + final byte [] value2 = Bytes.toBytes("bbbb"); + final byte [] value3 = Bytes.toBytes("cccc"); + final byte [] value4 = Bytes.toBytes("dddd"); + byte [] FAMILY = getColumnFamilyName(tableName).getBytes(); + + Put put2 = new Put(ROW); + put2.add(FAMILY, QUALIFIER, value2); + hTable.put(put2); + + Put put3 = new Put(ROW); + put3.add(FAMILY, QUALIFIER, value3); + + Delete delete = new Delete(ROW); + delete.deleteColumns(FAMILY, QUALIFIER); + + // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL + // turns out "match" + boolean ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value1, delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value1, delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value1, delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value1, delete); + assertEquals(ok, true); + hTable.put(put2); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value1, delete); + assertEquals(ok, true); + hTable.put(put2); + + assertEquals(ok, true); + + // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL + // turns out "match" + hTable.put(put3); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value4, delete); + + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value4, delete); + + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value4, delete); + + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value4, delete); + + assertEquals(ok, true); + hTable.put(put3); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value4, delete); + assertEquals(ok, true); + hTable.put(put3); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value4, delete); + + assertEquals(ok, true); + + // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL + // turns out "match" + hTable.put(put2); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value2, delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value2, delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value2, delete); + assertEquals(ok, false); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value2, delete); + assertEquals(ok, true); + hTable.put(put2); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value2, delete); + assertEquals(ok, true); + hTable.put(put2); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value2, delete); + assertEquals(ok, true); + } + @Test + public void testCheckAndMutate() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutate); + } + @Test - public void testCheckAndMutate() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateImpl); + public void testCheckAndDelete() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndDeleteWithCompareOp); } + @Test + public void testCheckAndPut() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndPut); + } + + @Test + public void testCheckAndMutateDiffRow() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateDiffRow); + } + + @Test + public void testCheckAndPutWithCompareOp() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndPutWithCompareOp); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java index 386a91b3..7a613e47 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -40,7 +40,7 @@ public class OHTableSecondaryPartGetTest { - private static List tableNames = new LinkedList(); + private static List tableNames = new LinkedList(); private static Map> group2tableNames = new LinkedHashMap<>(); @@ -114,12 +114,12 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr @Test - public void testGet() throws Exception { + public void testGet() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartGetTest::testGetImpl); } @Test - public void testMultiCFGet() throws Exception { + public void testMultiCFGet() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartGetTest::testMultiCFGetImpl); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java index 26af03f8..395d3799 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java @@ -17,40 +17,37 @@ package com.alipay.oceanbase.hbase.secondary; +import com.alipay.oceanbase.hbase.OHTable; import com.alipay.oceanbase.hbase.OHTableClient; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.util.TableTemplateManager; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Increment; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; -import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class OHTableSecondaryPartIncrementTest { - private static List tableNames = new LinkedList(); + private static List tableNames = new LinkedList(); private static Map> group2tableNames = null; - @BeforeClass public static void before() throws Exception { openDistributedExecute(); - for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + for (TableTemplateManager.TableType type : NORMAL_TABLES) { createTables(type, tableNames, group2tableNames, true); } } @@ -58,7 +55,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); + // dropTables(tableNames, group2tableNames); } @Before @@ -66,43 +63,142 @@ public void prepareCase() throws Exception { truncateTables(tableNames, group2tableNames); } - public static void testIncrementImpl(String tableName) throws Exception { + public static void testIncrement(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); + byte[] cf = getColumnFamilyName(tableName).getBytes(); + byte[] row = Bytes.toBytes("rk"); + byte[] qualifier = Bytes.toBytes("qual"); + byte[] qualifier2 = Bytes.toBytes("qual2"); + byte[] val = Bytes.toBytes(0L); + Put p = new Put(row); + p.add(cf, qualifier, val); + hTable.put(p); - String family = getColumnFamilyName(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String column2 = "putColumn2"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); - hTable.put(put); + for (int count = 0; count < 13; count++) { + Increment inc = new Increment(row); + inc.addColumn(cf, qualifier, 100L); + hTable.increment(inc); + } + Get get = new Get(row); + get.setMaxVersions(1); + get.addFamily(cf); + Result result = hTable.get(get); + assertEquals(1300L, Bytes.toLong(result.raw()[0].getValue())); + get.setMaxVersions(100); + result = hTable.get(get); + assertEquals(14, result.size()); - Increment increment = new Increment(key.getBytes()); - increment.addColumn(family.getBytes(), column1.getBytes(), 1L); - increment.addColumn(family.getBytes(), column2.getBytes(), 1L); - hTable.increment(increment); + Increment inc = new Increment(row); + inc.addColumn(cf, qualifier, -100L); + inc.addColumn(cf, qualifier2, -100L); + hTable.increment(inc); + get.setMaxVersions(1); + result = hTable.get(get); + assertEquals(1200L, Bytes.toLong(result.getColumnCells(cf, qualifier).get(0).getValue())); + assertEquals(-100L, Bytes.toLong(result.getColumnCells(cf, qualifier2).get(0).getValue())); + hTable.close(); + } - Get get = new Get(key.getBytes()); - get.addFamily(family.getBytes()); + private static void testIncBorder(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + byte[] ROW = "incKey".getBytes(); + byte[] v1 = Bytes.toBytes("ab"); + byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c") }; + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIERS[1], v1); + hTable.put(put); + Increment inc = new Increment(ROW); + inc.addColumn(FAMILY, QUALIFIERS[1], 2L); + try { + hTable.increment(inc); + } catch (Exception e) { + assertTrue(e.getCause().getMessage().contains("OB_KV_HBASE_INCR_FIELD_IS_NOT_LONG")); + } + Get get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); Result result = hTable.get(get); - Cell[] cells = result.rawCells(); - assertEquals(2, cells.length); - for (Cell cell : cells) { - if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column1)) { - assertEquals("2", Bytes.toString(CellUtil.cloneValue(cell))); - } else if (Bytes.toString(CellUtil.cloneQualifier(cell)).equals(column2)) { - assertEquals("2", Bytes.toString(CellUtil.cloneValue(cell))); - } + assertEquals(1, result.size()); + byte[] ROW1 = "incKey1".getBytes(); + inc = new Increment(ROW1); + inc.addColumn(FAMILY, QUALIFIERS[1], 2L); + hTable.increment(inc); + get = new Get(ROW1); + get.setMaxVersions(10); + get.addFamily(FAMILY); + result = hTable.get(get); + assertEquals(1, result.size()); + assertEquals(2L, Bytes.toLong(result.raw()[0].getValue())); + inc.addColumn(FAMILY, QUALIFIERS[0], 2L); + hTable.increment(inc); + get.setMaxVersions(10); + get.addFamily(FAMILY); + result = hTable.get(get); + assertEquals(3, result.size()); + assertEquals(4L, + Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(0).getValue())); + assertEquals(2L, + Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(1).getValue())); + assertEquals(2L, + Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0).getValue())); + hTable.close(); + } + + private static void testIncCon(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "incColumn"; + byte[] ROW = "incKey".getBytes(); + long expect = 0; + ThreadPoolExecutor threadPoolExecutor = OHTable.createDefaultThreadPoolExecutor(1, 100,100); + AtomicInteger atomicInteger = new AtomicInteger(0); + CountDownLatch countDownLatch = new CountDownLatch(100); + for (int i = 0; i < 100; i++) { + Increment inc = new Increment(ROW); + inc.addColumn(FAMILY, column.getBytes(), 2L); + threadPoolExecutor.submit(() -> { + try { + hTable.increment(inc); + atomicInteger.incrementAndGet(); + } catch (Exception e) { + if (!e.getCause().getMessage().contains("OB_TRY_LOCK_ROW_CONFLICT") && !e.getCause().getMessage().contains("OB_TIMEOUT")) { + throw new RuntimeException(e); + } + } finally { + countDownLatch.countDown(); + } + }); } + countDownLatch.await(100000, TimeUnit.MILLISECONDS); + for (int i = 0; i < atomicInteger.get(); i++) { + expect += 2; + } + Get get = new Get(ROW); + get.setMaxVersions(1); + get.addColumn(FAMILY, column.getBytes()); + Result result = hTable.get(get); + assertEquals(expect, Bytes.toLong(result.getColumnCells(FAMILY, column.getBytes()).get(0).getValue())); hTable.close(); } - + + @Test + public void testIncrement() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartIncrementTest::testIncrement); + } + + @Test + public void testBorderInc() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartIncrementTest::testIncBorder); + } + @Test - public void testIncrement() throws Exception { - FOR_EACH(tableNames, OHTableSecondaryPartIncrementTest::testIncrementImpl); + public void testIncConcurrency() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartIncrementTest::testIncCon); } - } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java index 45beb82b..b83caaae 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -37,7 +37,7 @@ import static org.apache.hadoop.hbase.util.Bytes.toBytes; public class OHTableSecondaryPartPutTest { - private static List tableNames = new LinkedList(); + private static List tableNames = new LinkedList(); private static Map> group2tableNames = new LinkedHashMap<>(); @@ -67,10 +67,12 @@ public static void testPutImpl(String tableName) throws Exception { String family = getColumnFamilyName(tableName); String key = "putKey"; String column = "putColumn"; + String column1 = "putColumn1"; String value = "value"; long timestamp = System.currentTimeMillis(); Put put = new Put(toBytes(key)); put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(value)); hTable.put(put); hTable.close(); @@ -94,12 +96,12 @@ public static void testMultiCFPutImpl(Map.Entry> entry) thr } @Test - public void testPut() throws Exception { + public void testPut() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartPutTest::testPutImpl); } @Test - public void testMultiCFPut() throws Exception { + public void testMultiCFPut() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMultiCFPutImpl); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java index 561a34f5..337f8702 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java @@ -42,10 +42,9 @@ import static org.junit.Assert.assertEquals; public class OHTableSecondaryPartScanTest { - private static List tableNames = new LinkedList(); + private static List tableNames = new LinkedList(); private static Map> group2tableNames = new LinkedHashMap<>(); - @BeforeClass public static void before() throws Exception { openDistributedExecute(); @@ -65,8 +64,6 @@ public void prepareCase() throws Exception { truncateTables(tableNames, group2tableNames); } - - public static void testScanImpl(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); @@ -94,14 +91,14 @@ public static void testScanImpl(String tableName) throws Exception { } assertEquals(1, count); } - + @Test - public void testScan() throws Exception { + public void testScan() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartScanTest::testScanImpl); } - + @Test - public void testMultiCFScan() throws Exception { + public void testMultiCFScan() throws Throwable { // implement later } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index 59f95f3a..f0e9403f 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -18,12 +18,11 @@ package com.alipay.oceanbase.hbase.util; import java.sql.Connection; -import java.sql.SQLException; +import java.sql.SQLSyntaxErrorException; import java.util.LinkedList; import java.util.List; import java.util.Map; - public class ObHTableSecondaryPartUtil { public static void openDistributedExecute() throws Exception { Connection conn = ObHTableTestUtil.getSysConnection(); @@ -37,7 +36,9 @@ public static void closeDistributedExecute() throws Exception { conn.createStatement().execute(stmt); } - public static void createTables(TableTemplateManager.TableType type, List tableNames, Map> group2tableNames, boolean printSql) throws Exception { + public static void createTables(TableTemplateManager.TableType type, List tableNames, + Map> group2tableNames, boolean printSql) + throws Exception { Connection conn = ObHTableTestUtil.getConnection(); // single cf table if (tableNames != null) { @@ -48,8 +49,9 @@ public static void createTables(TableTemplateManager.TableType type, List tableNames, boolean printSql) throws Exception { + + public static void createTables(Connection conn, TableTemplateManager.TableType type, + List tableNames, boolean printSql) throws Exception { // create single cf table if (tableNames != null) { TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); @@ -58,12 +60,24 @@ public static void createTables(Connection conn, TableTemplateManager.TableType conn.createStatement().execute(tableGroupSql); String tableName = TableTemplateManager.generateTableName(tableGroup, false, 1); String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); - conn.createStatement().execute(sql); + try { + conn.createStatement().execute(sql); + System.out.println("============= create table: " + tableName + " table_group: " + + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table: " + tableName + " table_group: " + + getTableName(tableName) + " already exist ============="); + } + } tableNames.add(tableName); - System.out.println("============= create table: " + tableName + " table_group: " + getTableName(tableName) + " =============\n" + (printSql ? sql : "") + " \n============= done =============\n"); } } - + public static void createTables(Connection conn, TableTemplateManager.TableType type, Map> group2tableNames, boolean printSql) throws Exception { if (group2tableNames != null) { TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); @@ -74,54 +88,66 @@ public static void createTables(Connection conn, TableTemplateManager.TableType for (int i = 1; i <= 3; ++i) { String tableName = TableTemplateManager.generateTableName(tableGroup, true, i); String sql = TableTemplateManager.getCreateTableSQL(type, tableName, timeRange); - conn.createStatement().execute(sql); + try { + conn.createStatement().execute(sql); + System.out.println("============= create table: " + tableName + + " table_group: " + getTableName(tableName) + " =============\n" + + (printSql ? sql : "") + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table: " + tableName + " table_group: " + getTableName(tableName) + " already exist ============="); + } + } group2tableNames.get(tableGroup).add(tableName); - System.out.println("============= create table: " + tableName - + " table_group: " + getTableName(tableName) + " =============\n" - + (printSql ? sql : "") + " \n============= done =============\n"); } } } - public static void truncateTables(List tableNames, Map> group2tableNames) throws Exception { + public static void truncateTables(List tableNames, + Map> group2tableNames) throws Exception { Connection conn = ObHTableTestUtil.getConnection(); // truncate single cf table truncateTables(conn, tableNames); // truncate multi cf table truncateTables(conn, group2tableNames); } - + public static void truncateTables(Connection conn, List tableNames) throws Exception { if (tableNames != null) { for (int i = 0; i < tableNames.size(); i++) { String stmt = "TRUNCATE TABLE " + tableNames.get(i) + ";"; conn.createStatement().execute(stmt); - System.out.println("============= truncate table " + tableNames.get(i) + " done ============="); + System.out.println("============= truncate table " + tableNames.get(i) + + " done ============="); } } } - - public static void truncateTables(Connection conn, Map> group2tableNames) throws Exception { + + public static void truncateTables(Connection conn, Map> group2tableNames) + throws Exception { if (group2tableNames != null) { for (Map.Entry> entry : group2tableNames.entrySet()) { for (String tableName : entry.getValue()) { String stmt = "TRUNCATE TABLE " + tableName + ";"; conn.createStatement().execute(stmt); - System.out.println("============= truncate table " + tableName + " done ============="); + System.out.println("============= truncate table " + tableName + + " done ============="); } } } } - - public static void dropTables(List tableNames, Map> group2tableNames) throws Exception { + public static void dropTables(List tableNames, + Map> group2tableNames) throws Exception { Connection conn = ObHTableTestUtil.getConnection(); // drop single cf table dropTables(conn, tableNames); // drop multi cf table dropTables(conn, group2tableNames); } - + public static void dropTables(Connection conn, List tableNames) throws Exception { if (tableNames != null) { for (String tableName : tableNames) { @@ -131,23 +157,25 @@ public static void dropTables(Connection conn, List tableNames) throws E } } } - - public static void dropTables(Connection conn, Map> group2tableNames) throws Exception { + + public static void dropTables(Connection conn, Map> group2tableNames) + throws Exception { if (group2tableNames != null) { for (Map.Entry> entry : group2tableNames.entrySet()) { for (String tableName : entry.getValue()) { String stmt = "DROP TABLE IF EXISTS " + tableName + ";"; conn.createStatement().execute(stmt); - System.out.println("============= drop table " + tableName + " done ============="); + System.out.println("============= drop table " + tableName + + " done ============="); } String stmt = "DROP TABLEGROUP IF EXISTS " + entry.getKey() + ";"; conn.createStatement().execute(stmt); - System.out.println("============= drop tablegroup " + entry.getKey() + " done ============="); + System.out.println("============= drop tablegroup " + entry.getKey() + + " done ============="); } } } - public static String getTableName(String input) throws Exception { // 查找 '$' 的索引 int index = input.indexOf('$'); diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index b5e4be84..d3382053 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -52,6 +52,10 @@ public class ObHTableTestUtil { + JDBC_DATABASE + "?" + "useUnicode=TRUE&" + "characterEncoding=utf-8&" + "socketTimeout=3000000&" + "connectTimeout=60000"; + public static String SYS_JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + + "oceanbase?" + "useUnicode=TRUE&" + + "characterEncoding=utf-8&" + + "socketTimeout=3000000&" + "connectTimeout=60000"; public static String SQL_FORMAT = "truncate %s"; public static List tableNameList; @@ -160,7 +164,7 @@ static public Connection getConnection() { static public Connection getSysConnection() { try { Class.forName("com.mysql.cj.jdbc.Driver"); - Connection conn = DriverManager.getConnection(JDBC_URL, SYS_USER_NAME, SYS_PASSWORD); + Connection conn = DriverManager.getConnection(SYS_JDBC_URL, SYS_USER_NAME, SYS_PASSWORD); return conn; } catch (Exception e) { @@ -170,7 +174,7 @@ static public Connection getSysConnection() { @FunctionalInterface public interface CheckedConsumer { - void accept(T t) throws Exception; + void accept(T t) throws Throwable; default CheckedConsumer andThen(CheckedConsumer after) throws Exception { if (after == null) { throw new NullPointerException();} return (T t) -> { @@ -180,13 +184,14 @@ default CheckedConsumer andThen(CheckedConsumer after) throws Exce } } - public static void FOR_EACH(List tableNames, CheckedConsumer consumer) throws Exception { + public static void FOR_EACH(List tableNames, CheckedConsumer consumer) throws Throwable { for (String tableName : tableNames) { + System.out.println("============================= table::{" + tableName + "} ============================="); consumer.accept(tableName); } } - public static void FOR_EACH(Map> group2Tables, CheckedConsumer>> consumer) throws Exception { + public static void FOR_EACH(Map> group2Tables, CheckedConsumer>> consumer) throws Throwable { for (Map.Entry> entry : group2Tables.entrySet()) { consumer.accept(entry); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index 6a257fdc..983364b6 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -18,8 +18,14 @@ package com.alipay.oceanbase.hbase.util; +import java.util.Arrays; import java.util.EnumMap; +import java.util.List; import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.*; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN; + public class TableTemplateManager { public static final long PART_NUM = 3; public static final String TABLE_GROUP_PREFIX = "test_group_"; @@ -36,6 +42,13 @@ public enum TableType { SECONDARY_PARTITIONED_TIME_RANGE_KEY, // 时序表RANGE-KEY SECONDARY_PARTITIONED_TIME_KEY_RANGE // 时序表KEY-RANGE } + + public static List NORMAL_TABLES = Arrays.asList(NON_PARTITIONED_REGULAR, + SINGLE_PARTITIONED_REGULAR, + SECONDARY_PARTITIONED_RANGE_KEY, + SECONDARY_PARTITIONED_RANGE_KEY_GEN, + SECONDARY_PARTITIONED_KEY_RANGE, + SECONDARY_PARTITIONED_KEY_RANGE_GEN); private static final Map SQL_TEMPLATES = new EnumMap<>(TableType.class); diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java b/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java index 758dedb3..ae526872 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TimeGenerator.java @@ -31,19 +31,28 @@ public TimeRange(long lowerBound1, long lowerBound2, long upperBound1, long uppe this.upperBound2 = upperBound2; } - public long lowerBound1() { return lowerBound1; } - public long lowerBound2() { return lowerBound2; } - public long upperBound1() { return upperBound1; } - public long upperBound2() { return upperBound2; } + public long lowerBound1() { + return lowerBound1; + } + + public long lowerBound2() { + return lowerBound2; + } + + public long upperBound1() { + return upperBound1; + } + + public long upperBound2() { + return upperBound2; + } } public static TimeRange generateTestTimeRange() { long current = System.currentTimeMillis(); - return new TimeRange( - current - 86400000, // 24小时前 - current, - current + 86400000, // 24小时后 - current + 172800000 // 48小时后 + return new TimeRange(current - 86400000, // 24小时前 + current, current + 86400000, // 24小时后 + current + 172800000 // 48小时后 ); } } From 4cecf689d8a5af6658244a6bd3295a46b27e1225 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Sun, 2 Mar 2025 20:50:10 +0800 Subject: [PATCH 12/40] query and mutate delete add test case add delete test case (cherry picked from commit 7c7408043a2e018afdba4802cdb56966fe15e309) --- .../com/alipay/oceanbase/hbase/OHTable.java | 112 +++- .../OHTableSecondaryPartDeleteTest.java | 536 ++++++++++++++++++ .../hbase/util/ObHTableTestUtil.java | 37 ++ .../hbase/util/ResultSetPrinter.java | 169 ++++++ 4 files changed, 847 insertions(+), 7 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index ae400af8..c45c383f 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -40,6 +40,7 @@ import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.mutate.ObTableQueryAndMutateResult; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.query.*; import com.alipay.oceanbase.rpc.protocol.payload.impl.execute.syncquery.ObTableQueryAsyncRequest; +import com.alipay.oceanbase.rpc.queryandmutate.QueryAndMutate; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryAsyncStreamResult; import com.alipay.oceanbase.rpc.stream.ObTableClientQueryStreamResult; import com.alipay.oceanbase.rpc.table.ObHBaseParams; @@ -83,6 +84,7 @@ import static org.apache.commons.lang.StringUtils.isBlank; import static org.apache.commons.lang.StringUtils.isNotBlank; import static com.alipay.oceanbase.hbase.filter.HBaseFilterUtils.writeBytesWithEscape; +import static org.apache.hadoop.hbase.KeyValue.Type.*; public class OHTable implements Table { @@ -1998,6 +2000,81 @@ public static ObTableBatchOperation buildObTableBatchOperation(List ro batch.setSamePropertiesNames(true); return batch; } + private QueryAndMutate buildDeleteQueryAndMutate(KeyValue kv, + ObTableOperationType operationType, + boolean isTableGroup, Long TTL) { + KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType()); + com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, operationType, isTableGroup, TTL); + ObNewRange range = new ObNewRange(); + ObTableQuery tableQuery = new ObTableQuery(); + tableQuery.setObKVParams(buildOBKVParams((Scan)null)); + ObHTableFilter filter = null; +// buildObTableQuery(tableQuery, range, filter, null); + switch (kvType) { + case Delete: + if (kv.getTimestamp() == Long.MAX_VALUE) { + range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + filter = buildObHTableFilter(null, null, 1, kv.getQualifier()); + } else { + range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(),ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + filter = buildObHTableFilter(null, new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), 1, kv.getQualifier()); + } + break; + case DeleteColumn: + if (kv.getTimestamp() == Long.MAX_VALUE) { + range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE, kv.getQualifier()); + } else { + range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE, kv.getQualifier()); + } + break; + case DeleteFamily: + if (kv.getTimestamp() == Long.MAX_VALUE) { + range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + if (!isTableGroup) { + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE); + } else { + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE, kv.getQualifier()); + } + } else { + range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + if (!isTableGroup) { + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE); + } else { + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE, kv.getQualifier()); + } + } + break; + case DeleteFamilyVersion: + if (kv.getTimestamp() == Long.MAX_VALUE) { + range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE); + } else { + range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + if (!isTableGroup) { + filter = buildObHTableFilter(null, new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), Integer.MAX_VALUE); + } else { + filter = buildObHTableFilter(null, new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), Integer.MAX_VALUE, kv.getQualifier()); + } + } + break; + default: + return null; + } + tableQuery.sethTableFilter(filter); + tableQuery.addKeyRange(range); + return new QueryAndMutate(tableQuery, tableMutation); + } + private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(Cell kv, ObTableOperationType operationType, @@ -2114,7 +2191,7 @@ private BatchOperation buildBatchOperation(String tableName, List Put put = (Put) row; if (put.isEmpty()) { throw new IllegalArgumentException("No columns to insert for #" - + (posInList + 1) + " item"); + + (posInList + 1) + " item"); } for (Map.Entry> entry : put.getFamilyCellMap().entrySet()) { byte[] family = entry.getKey(); @@ -2129,18 +2206,39 @@ private BatchOperation buildBatchOperation(String tableName, List Delete delete = (Delete) row; if (delete.isEmpty()) { singleOpResultNum++; - KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp()); - batch.addOperation(com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, - new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, - null, null)); + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), + KeyValue.Type.Maximum); + com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, DEL, isTableGroup, Long.MAX_VALUE); + ObNewRange range = new ObNewRange(); + ObTableQuery tableQuery = new ObTableQuery(); + ObHTableFilter filter = null; + tableQuery.setObKVParams(buildOBKVParams((Scan)null)); + range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + if (kv.getTimestamp() == Long.MAX_VALUE) { + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE); + } else { + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE); + } + tableQuery.sethTableFilter(filter); + tableQuery.addKeyRange(range); + + tableMutation.setTable(tableName); + batch.addOperation(new QueryAndMutate(tableQuery, tableMutation)); } else { for (Map.Entry> entry : delete.getFamilyCellMap().entrySet()) { byte[] family = entry.getKey(); List keyValueList = entry.getValue(); for (Cell kv : keyValueList) { singleOpResultNum++; - batch.addOperation(buildMutation(kv, DEL, - isTableGroup, family, delete.getTTL())); + if (isTableGroup) { + KeyValue new_kv = modifyQualifier(kv, + (Bytes.toString(family) + "." + Bytes.toString(kv + .getQualifier())).getBytes()); + batch.addOperation(buildDeleteQueryAndMutate(new_kv, DEL, true, Long.MAX_VALUE)); + } else { + batch.addOperation(buildDeleteQueryAndMutate(kv, DEL, false, Long.MAX_VALUE)); + } } } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java new file mode 100644 index 00000000..5ce44ac8 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java @@ -0,0 +1,536 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.*; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableSecondaryPartDeleteTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testDeleteLastVersionImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + + Long ts1 = System.currentTimeMillis(); + Long ts2 = ts1 + 1000; + Long ts3 = ts2 + 1000; + Long ts4 = ts3 + 1000; + { // delete last version + Put put = new Put(toBytes(key)); + put.add(toBytes(family), toBytes(column), ts1, toBytes(value)); + put.add(toBytes(family), toBytes(column), ts2, toBytes(value)); + put.add(toBytes(family), toBytes(column), ts3, toBytes(value)); + put.add(toBytes(family), toBytes(column), ts4, toBytes(value)); +// hTable.put(put); + + Delete delete = new Delete(toBytes(key)); + delete.deleteColumn(toBytes(family), toBytes(column)); + hTable.delete(delete); + + Get get = new Get(toBytes(key)); + get.addColumn(toBytes(family), toBytes(column)); + get.setMaxVersions(); + Result result = hTable.get(get); + Assert.assertEquals(3, result.size()); // ts4 is deleted + Assert.assertEquals(value, result.getValue(family.getBytes(), column.getBytes())); + for (Cell cell : result.rawCells()) { + Assert.assertTrue(cell.getTimestamp() != ts4); + } + } + } + + public static void testDeleteSpecifiedImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + + Long ts1 = System.currentTimeMillis(); + Long ts2 = ts1 + 1000; + Long ts3 = ts2 + 1000; + Long ts4 = ts3 + 1000; + + { + Put put = new Put(toBytes(key)); + put.add(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.add(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.add(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.add(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + hTable.put(put); + + Delete delete = new Delete(toBytes(key)); + delete.addColumn(toBytes(family), toBytes(column), ts1); + hTable.delete(delete); + + Get get = new Get(toBytes(key)); + get.addColumn(toBytes(family), toBytes(column)); + get.setMaxVersions(); + Result result = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(3, result.size())); // ts1 is deleted + Assert(tableName, ()->Assert.assertTrue(secureCompare((value + ts4).getBytes(), result.getValue(family.getBytes(), column.getBytes())))); + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertTrue(cell.getTimestamp() != ts1)); + } + } + } + + public static void testDeleteColumnImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + + Long ts1 = System.currentTimeMillis(); + Long ts2 = ts1 + 1000; + Long ts3 = ts2 + 1000; + Long ts4 = ts3 + 1000; + + { + Put put = new Put(toBytes(key)); + put.add(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.add(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.add(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.add(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + hTable.put(put); + + Delete delete = new Delete(toBytes(key)); + delete.addColumns(toBytes(family), toBytes(column), ts3); + hTable.delete(delete); + + Get get = new Get(toBytes(key)); + get.addColumn(toBytes(family), toBytes(column)); + get.setMaxVersions(); + Result result = hTable.get(get); + Assert.assertEquals(1, result.size()); // ts1, ts2, ts3 is deleted + Assert.assertEquals(Arrays.toString((value + ts4).getBytes()), Arrays.toString(result.getValue(family.getBytes(), column.getBytes()))); + for (Cell cell : result.rawCells()) { + Assert.assertTrue(cell.getTimestamp() != ts1); + } + } + } + + public static void testDeleteFamilyImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + + Long ts1 = System.currentTimeMillis(); + Long ts2 = ts1 + 1000; + Long ts3 = ts2 + 1000; + Long ts4 = ts3 + 1000; + + { + Put put = new Put(toBytes(key)); + put.add(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.add(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.add(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.add(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + hTable.put(put); + + Delete delete = new Delete(toBytes(key)); + delete.addFamily(toBytes(family)); + hTable.delete(delete); + + Get get = new Get(toBytes(key)); + get.addColumn(toBytes(family), toBytes(column)); + get.setMaxVersions(); + Result result = hTable.get(get); + Assert.assertEquals(0, result.size()); + } + } + + public static void testDeleteFamilyVersionImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column = "putColumn"; + String value = "value"; + + Long ts1 = System.currentTimeMillis(); + Long ts2 = ts1 + 1000; + Long ts3 = ts2 + 1000; + Long ts4 = ts3 + 1000; + + { + Put put = new Put(toBytes(key)); + put.add(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.add(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.add(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.add(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + hTable.put(put); + + Delete delete = new Delete(toBytes(key)); + delete.addFamily(toBytes(family), ts2); + hTable.delete(delete); + + Get get = new Get(toBytes(key)); + get.addColumn(toBytes(family), toBytes(column)); + get.setMaxVersions(); + Result result = hTable.get(get); + Assert.assertEquals(2, result.size()); + for (Cell cell : result.rawCells()) { + Assert.assertTrue(cell.getTimestamp() != ts1); + Assert.assertTrue(cell.getTimestamp() != ts2); + } + } + } + + public static void testMultiCFDeleteLastVersionImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + delete.addColumn(toBytes(family), toBytes(column + family)); + } + hTable.put(put); + hTable.delete(delete); + + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + // last version ts4 is deleted, so the last version is ts3 + Assert(tableName, ()->Assert.assertTrue(secureCompare((value + ts3).getBytes(), + result.getValue(family.getBytes(), (column + family).getBytes())))); + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertTrue("should not found last version ts4", cell.getTimestamp() != ts4)); + } + } + } + + public static void testMultiCFDeleteSpecifiedImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + delete.addColumn(toBytes(family), toBytes(column + family), ts2); + } + hTable.put(put); + hTable.delete(delete); + + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + + for (String tableName : entry.getValue()) { + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertTrue("should not found last version ts2",cell.getTimestamp() != ts2)); + } + } + } + + public static void testMultiCFDeleteColumnImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + delete.addColumns(toBytes(family), toBytes(column + family), ts2); + } + hTable.put(put); + hTable.delete(delete); + + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + + for (String tableName : entry.getValue()) { + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertTrue("should not found last version ts2",cell.getTimestamp() != ts2)); + } + } + } + + + public static void testMultiCFDeleteFamilyImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + delete.addFamily(toBytes(family)); + } + hTable.put(put); + hTable.delete(delete); + + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + Assert.assertEquals(0, result.size()); + } + + public static void testMultiCFDeleteFamilyVersionImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + delete.addFamily(toBytes(family), ts2); + } + hTable.put(put); + hTable.delete(delete); + + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + Assert.assertEquals(2 * 3, result.size()); + } + + public static void testDeleteAllImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String value = "value"; + String column = "putColumn"; + + long ts1 = System.currentTimeMillis(); + long ts2 = ts1 + 1000; + long ts3 = ts2 + 1000; + long ts4 = ts3 + 1000; + + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + Put put = new Put(toBytes(key)); + Delete delete = new Delete(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.addColumn(toBytes(family), toBytes(column + family), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column + family), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column + family), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column + family), ts4, toBytes(value + ts4)); + } + hTable.put(put); + hTable.delete(delete); + + + Get get = new Get(toBytes(key)); + get.setMaxVersions(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(column + family)); + } + Result result = hTable.get(get); + Assert.assertEquals(0, result.size()); + } + + @Test + public void testDelete() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteSpecifiedImpl); + } + @Test + public void testDeleteLastVersion() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteLastVersionImpl); + } + + @Test + public void testDeleteColumn() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteColumnImpl); + } + + @Test + public void testDeleteFamily() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteFamilyImpl); + } + + @Test + public void testDeleteFamilyVersion() throws Exception { + FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteFamilyVersionImpl); + } + + + @Test + public void testMultiCFDelete() throws Exception { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteLastVersionImpl); + } + + @Test + public void testMultiCFDeleteSpecified() throws Exception { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteSpecifiedImpl); + } + + @Test + public void testMultiCFDeleteColumn() throws Exception { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteColumnImpl); + } + + @Test + public void testMultiCFDeleteFamily() throws Exception { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteFamilyImpl); + } + + @Test + public void testMultiCFDeleteFamilyVersion() throws Exception { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteFamilyVersionImpl); + } + + + @Test + public void testDeleteAll() throws Exception { + FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testDeleteAllImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index d3382053..effeb457 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -196,4 +196,41 @@ public static void FOR_EACH(Map> group2Tables, CheckedConsu consumer.accept(entry); } } + + public static void Assert(String tableName, Runnable assertMethod) throws SQLException { + try { + assertMethod.run(); + } catch (AssertionError e) { + Connection conn = ObHTableTestUtil.getConnection(); + String selectSql = "select * from " + tableName; + System.out.println("assert fail, execute sql: " + selectSql); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery(selectSql); + ResultSetPrinter.print(resultSet); + throw e; + } + } + + public static void Assert(List tableNames, Runnable assertMethod) throws SQLException { + try { + assertMethod.run(); + } catch (AssertionError e) { + for (String tableName : tableNames) { + Connection conn = ObHTableTestUtil.getConnection(); + String selectSql = "select * from " + tableName; + System.out.println("assert fail, execute sql: " + selectSql); + java.sql.ResultSet resultSet = conn.createStatement().executeQuery(selectSql); + ResultSetPrinter.print(resultSet); + } + throw e; + } + } + + + public static boolean secureCompare(byte[] a, byte[] b) { + int diff = a.length ^ b.length; + for (int i = 0; i < a.length && i < b.length; i++) { + diff |= a[i] ^ b[i]; + } + return diff == 0; + } } \ No newline at end of file diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java b/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java new file mode 100644 index 00000000..726e7870 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java @@ -0,0 +1,169 @@ +package com.alipay.oceanbase.hbase.util; + +import java.sql.*; +import java.sql.Date; +import java.text.SimpleDateFormat; +import java.util.*; + +public class ResultSetPrinter { + private static final int MAX_COL_WIDTH = 30; + private static final String VERTICAL = "│"; + private static final String HORIZONTAL = "─"; + private static final String TOP_LEFT = "┌"; + private static final String TOP_RIGHT = "┐"; + private static final String MID_LEFT = "├"; + private static final String MID_RIGHT = "┤"; + private static final String BOTTOM_LEFT = "└"; + private static final String BOTTOM_RIGHT = "┘"; + private static final String CROSS = "┼"; + private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + public static void print(ResultSet rs) throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + int colCount = meta.getColumnCount(); + + List columns = new ArrayList(); + for (int i = 1; i <= colCount; i++) { + columns.add(new ColumnMeta( + meta.getColumnLabel(i), + meta.getColumnType(i), + meta.getPrecision(i) + )); + } + + // 计算初始列宽 + List widths = new ArrayList(); + for (ColumnMeta col : columns) { + int width = Math.min( + Math.max(col.name.length(), getTypeWidth(col)), + MAX_COL_WIDTH + ); + widths.add(width); + } + + List> rows = new ArrayList>(); + while (rs.next()) { + List row = new ArrayList(); + for (int i = 0; i < colCount; i++) { + Object value = rs.getObject(i + 1); + String str = formatValue(value, widths.get(i)); + + int actualWidth = str.length(); + if (actualWidth > widths.get(i)) { + widths.set(i, Math.min(actualWidth, MAX_COL_WIDTH)); + } + row.add(str); + } + rows.add(row); + } + + StringBuilder fmt = new StringBuilder(VERTICAL); + for (int w : widths) { + fmt.append(" %-").append(w).append("s ").append(VERTICAL); + } + + printDivider(widths, TOP_LEFT, CROSS, TOP_RIGHT); + System.out.printf(fmt.toString() + "%n", getHeaders(columns)); + printDivider(widths, MID_LEFT, CROSS, MID_RIGHT); + for (List row : rows) { + System.out.printf(fmt.toString() + "%n", row.toArray()); + } + printDivider(widths, BOTTOM_LEFT, CROSS, BOTTOM_RIGHT); + } + + private static class ColumnMeta { + final String name; + final int type; + final int precision; + + ColumnMeta(String name, int type, int precision) { + this.name = name; + this.type = type; + this.precision = precision; + } + } + + private static int getTypeWidth(ColumnMeta col) { + switch (col.type) { + case Types.DATE: return 10; + case Types.TIMESTAMP: return 19; + case Types.INTEGER: return 11; + case Types.DECIMAL: + case Types.NUMERIC: return col.precision + 2; + default: return Math.min(col.precision, MAX_COL_WIDTH); + } + } + + private static String formatValue(Object value, int maxWidth) { + if (value == null) return "NULL"; + + if (value instanceof byte[]) { + return formatByteArray((byte[]) value, maxWidth); + } + if (value instanceof Date) { + return DATE_FORMAT.format((Date) value); + } + return truncateString(value.toString(), maxWidth); + } + + private static String formatByteArray(byte[] bytes, int maxWidth) { + if (bytes.length == 0) return "[NULL]"; + if (isPrintable(bytes)) { + return truncateString(new String(bytes), maxWidth); + } + + if (bytes.length <= 4) { + return hexFormat(bytes, maxWidth); + } + + return base64Format(bytes, maxWidth); + } + + private static boolean isPrintable(byte[] bytes) { + for (byte b : bytes) { + if (b < 0x20 || b > 0x7E) return false; + } + return true; + } + + private static String hexFormat(byte[] bytes, int maxWidth) { + int maxBytes = (maxWidth - 3) / 3; // [XX XX...] + maxBytes = Math.max(1, maxBytes); + + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < Math.min(bytes.length, maxBytes); i++) { + sb.append(String.format("%02X ", bytes[i])); + } + if (bytes.length > maxBytes) sb.append("..."); + sb.setLength(sb.length() - 1); + return sb.append("]").toString(); + } + + private static String base64Format(byte[] bytes, int maxWidth) { + String base64 = Base64.getEncoder().encodeToString(bytes); + return truncateString("b64:" + base64, maxWidth); + } + + private static String truncateString(String str, int maxWidth) { + if (str.length() <= maxWidth) return str; + return str.substring(0, maxWidth - 3) + "..."; + } + + private static Object[] getHeaders(List columns) { + String[] headers = new String[columns.size()]; + for (int i = 0; i < headers.length; i++) { + headers[i] = columns.get(i).name; + } + return headers; + } + + private static void printDivider(List widths, String left, String mid, String right) { + StringBuilder sb = new StringBuilder(left); + for (int i = 0; i < widths.size(); i++) { + sb.append(String.join("", Collections.nCopies(widths.get(i) + 2, HORIZONTAL))); + sb.append(i < widths.size() - 1 ? mid : right); + } + System.out.println(sb); + } + +} \ No newline at end of file From e915dfed778dea5d3c6c8e1e92de2a744bb8638c Mon Sep 17 00:00:00 2001 From: maochongxin Date: Thu, 13 Mar 2025 21:21:13 +0800 Subject: [PATCH 13/40] header (cherry picked from commit fff7a2d21ebcdecd14957e1074d2b6cc382b0c75) --- .../hbase/util/ResultSetPrinter.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java b/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java index 726e7870..65b3964c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java @@ -1,3 +1,20 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + package com.alipay.oceanbase.hbase.util; import java.sql.*; @@ -166,4 +183,4 @@ private static void printDivider(List widths, String left, String mid, System.out.println(sb); } -} \ No newline at end of file +} From a9f3c433b9e7c7cdfd227913fce23b751f943b8d Mon Sep 17 00:00:00 2001 From: maochongxin Date: Fri, 14 Mar 2025 14:23:56 +0800 Subject: [PATCH 14/40] fix review (cherry picked from commit f76d169b5c9cf7f905f7afa59fc304ab9b344567) --- .../com/alipay/oceanbase/hbase/OHTable.java | 1 - .../OHTableSecondaryPartDeleteTest.java | 45 ++++++++++--------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index c45c383f..252a6337 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -2009,7 +2009,6 @@ private QueryAndMutate buildDeleteQueryAndMutate(KeyValue kv, ObTableQuery tableQuery = new ObTableQuery(); tableQuery.setObKVParams(buildOBKVParams((Scan)null)); ObHTableFilter filter = null; -// buildObTableQuery(tableQuery, range, filter, null); switch (kvType) { case Delete: if (kv.getTimestamp() == Long.MAX_VALUE) { diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java index 5ce44ac8..af937fc9 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java @@ -77,7 +77,7 @@ public static void testDeleteLastVersionImpl(String tableName) throws Exception put.add(toBytes(family), toBytes(column), ts2, toBytes(value)); put.add(toBytes(family), toBytes(column), ts3, toBytes(value)); put.add(toBytes(family), toBytes(column), ts4, toBytes(value)); -// hTable.put(put); + hTable.put(put); Delete delete = new Delete(toBytes(key)); delete.deleteColumn(toBytes(family), toBytes(column)); @@ -163,7 +163,7 @@ public static void testDeleteColumnImpl(String tableName) throws Exception { get.addColumn(toBytes(family), toBytes(column)); get.setMaxVersions(); Result result = hTable.get(get); - Assert.assertEquals(1, result.size()); // ts1, ts2, ts3 is deleted + Assert(tableName, ()->Assert.assertEquals(2, result.size())); // ts1, ts2, ts3 is deleted Assert.assertEquals(Arrays.toString((value + ts4).getBytes()), Arrays.toString(result.getValue(family.getBytes(), column.getBytes()))); for (Cell cell : result.rawCells()) { Assert.assertTrue(cell.getTimestamp() != ts1); @@ -237,8 +237,8 @@ public static void testDeleteFamilyVersionImpl(String tableName) throws Exceptio Result result = hTable.get(get); Assert.assertEquals(2, result.size()); for (Cell cell : result.rawCells()) { - Assert.assertTrue(cell.getTimestamp() != ts1); - Assert.assertTrue(cell.getTimestamp() != ts2); + Assert(tableName, ()->Assert.assertTrue(cell.getTimestamp() != ts1)); + Assert(tableName, ()->Assert.assertTrue(cell.getTimestamp() != ts2)); } } } @@ -252,7 +252,7 @@ public static void testMultiCFDeleteLastVersionImpl(Map.Entry long ts2 = ts1 + 1000; long ts3 = ts2 + 1000; long ts4 = ts3 + 1000; - + System.out.println("ts1:" + ts1 + ", ts2:" + ts2 + ", ts3:" + ts3 + ", ts4:" + ts4); OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); hTable.init(); Put put = new Put(toBytes(key)); @@ -338,7 +338,7 @@ public static void testMultiCFDeleteColumnImpl(Map.Entry> e long ts2 = ts1 + 1000; long ts3 = ts2 + 1000; long ts4 = ts3 + 1000; - + System.out.println("ts1:" + ts1 + ", ts2:" + ts2 + ", ts3:" + ts3 + ", ts4:" + ts4); OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); hTable.init(); Put put = new Put(toBytes(key)); @@ -380,7 +380,7 @@ public static void testMultiCFDeleteFamilyImpl(Map.Entry> e long ts2 = ts1 + 1000; long ts3 = ts2 + 1000; long ts4 = ts3 + 1000; - + System.out.println("ts1:" + ts1 + ", ts2:" + ts2 + ", ts3:" + ts3 + ", ts4:" + ts4); OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); hTable.init(); Put put = new Put(toBytes(key)); @@ -416,7 +416,7 @@ public static void testMultiCFDeleteFamilyVersionImpl(Map.EntryAssert.assertEquals(0, result.size())); } public static void testDeleteAllImpl(Map.Entry> entry) throws Exception { @@ -452,7 +453,7 @@ public static void testDeleteAllImpl(Map.Entry> entry) thro long ts2 = ts1 + 1000; long ts3 = ts2 + 1000; long ts4 = ts3 + 1000; - + System.out.println("ts1:" + ts1 + ", ts2:" + ts2 + ", ts3:" + ts3 + ", ts4:" + ts4); OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); hTable.init(); Put put = new Put(toBytes(key)); @@ -475,62 +476,62 @@ public static void testDeleteAllImpl(Map.Entry> entry) thro get.addColumn(toBytes(family), toBytes(column + family)); } Result result = hTable.get(get); - Assert.assertEquals(0, result.size()); + Assert(entry.getValue(), ()->Assert.assertEquals(0, result.size())); } @Test - public void testDelete() throws Exception { + public void testDelete() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteSpecifiedImpl); } @Test - public void testDeleteLastVersion() throws Exception { + public void testDeleteLastVersion() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteLastVersionImpl); } @Test - public void testDeleteColumn() throws Exception { + public void testDeleteColumn() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteColumnImpl); } @Test - public void testDeleteFamily() throws Exception { + public void testDeleteFamily() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteFamilyImpl); } @Test - public void testDeleteFamilyVersion() throws Exception { + public void testDeleteFamilyVersion() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteFamilyVersionImpl); } @Test - public void testMultiCFDelete() throws Exception { + public void testMultiCFDelete() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteLastVersionImpl); } @Test - public void testMultiCFDeleteSpecified() throws Exception { + public void testMultiCFDeleteSpecified() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteSpecifiedImpl); } @Test - public void testMultiCFDeleteColumn() throws Exception { + public void testMultiCFDeleteColumn() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteColumnImpl); } @Test - public void testMultiCFDeleteFamily() throws Exception { + public void testMultiCFDeleteFamily() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteFamilyImpl); } @Test - public void testMultiCFDeleteFamilyVersion() throws Exception { + public void testMultiCFDeleteFamilyVersion() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteFamilyVersionImpl); } @Test - public void testDeleteAll() throws Exception { + public void testDeleteAll() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testDeleteAllImpl); } } From 2769ef48468bb8031d8ffccc1770f6d5eb7d29b3 Mon Sep 17 00:00:00 2001 From: shen Date: Fri, 14 Mar 2025 14:32:36 +0800 Subject: [PATCH 15/40] [Test] add hbase get test cases for secondary part (cherry picked from commit 2fc226413d98c6d1ca3bf48eb21a2587ef4fc2fa) --- .../OHTableSecondaryPartGetTest.java | 241 +++++++++++++++--- .../hbase/util/ObHTableSecondaryPartUtil.java | 17 ++ 2 files changed, 228 insertions(+), 30 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java index 7a613e47..e06c94a3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -18,6 +18,7 @@ package com.alipay.oceanbase.hbase.secondary; import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.util.TableTemplateManager; import org.apache.hadoop.hbase.Cell; @@ -25,7 +26,9 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; import org.junit.*; import java.util.LinkedHashMap; @@ -35,6 +38,7 @@ import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.*; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.assertEquals; @@ -48,7 +52,12 @@ public class OHTableSecondaryPartGetTest { public static void before() throws Exception { openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { - createTables(type, tableNames, group2tableNames, true); + if (type != SECONDARY_PARTITIONED_TIME_RANGE_KEY && + type != SECONDARY_PARTITIONED_TIME_KEY_RANGE && + type != NON_PARTITIONED_TIME_SERIES && + type != SINGLE_PARTITIONED_TIME_SERIES) { + createTables(type, tableNames, group2tableNames, true); + } } } @@ -68,51 +77,223 @@ public static void testGetImpl(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); + // 0. prepare data String family = getColumnFamilyName(tableName); String key = "putKey"; - String column = "putColumn"; - String value = "value"; - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), toBytes(value)); - hTable.put(put); - - Get get = new Get(key.getBytes()); - get.addFamily(family.getBytes()); - Result result = hTable.get(get); - Cell[] cells = result.rawCells(); - assertEquals(1, cells.length); - assertEquals(column, Bytes.toString(CellUtil.cloneQualifier(cells[0]))); - assertEquals("value", Bytes.toString(CellUtil.cloneValue(cells[0]))); - System.out.println("get table " + tableName + " done"); + String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; + String[] values = {"version1", "version2"}; // each column have two versions + long curTs = System.currentTimeMillis(); + long[] ts = {curTs, curTs+1}; // each column have two versions + String latestValue = values[1]; + long lastTs = ts[1]; + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns.length; j++) { + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i])); + hTable.put(put); + } + } + + // 1. get specify column + { + int index = 0; + Get get = new Get(key.getBytes()); + get.addColumn(family.getBytes(), columns[index].getBytes()); + Result r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + AssertKeyValue(key, columns[index], lastTs, latestValue, r.rawCells()[0]); + } + + // 2. get do not specify column + { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(columns.length, cells.length); + for (int i = 0; i < columns.length; i++) { + ObHTableSecondaryPartUtil.AssertKeyValue(key, columns[i], lastTs, latestValue, cells[i]); + } + } + + // 3. get specify versions + { + int index = 0; + Get get = new Get(key.getBytes()); + get.addColumn(family.getBytes(), columns[index].getBytes()); + get.setMaxVersions(2); + Result r = hTable.get(get); + Assert.assertEquals(2, r.raw().length); + AssertKeyValue(key, columns[index], ts[1], values[1], r.raw()[0]); + AssertKeyValue(key, columns[index], ts[0], values[0], r.raw()[1]); + } + + // 4. get specify time range + { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + get.setTimeStamp(ts[1]); + Result r = hTable.get(get); + Assert.assertEquals(columns.length, r.raw().length); + for (int i = 0; i < columns.length; i++) { + AssertKeyValue(key, columns[i], values[1], r.raw()[i]); + } + } + + // 5. get specify filter + { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(values[0]))); + get.setFilter(valueFilter); + Result r = hTable.get(get); + Assert.assertEquals(columns.length, r.raw().length); + for (int i = 0; i < columns.length; i++) { + AssertKeyValue(key, columns[i], values[0], r.raw()[i]); + } + } hTable.close(); } public static void testMultiCFGetImpl(Map.Entry> entry) throws Exception { + + // 0. prepare data String key = "putKey"; - String column = "putColumn"; + String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; + String groupName = getTableName(entry.getKey()); + String[] values = {"version1", "version2"}; // each column have two versions + String latestValue = values[1]; + List tableNames = entry.getValue(); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + long timestamp = System.currentTimeMillis(); + long[] ts = {timestamp, timestamp+1}; + long lastTs = ts[1]; + hTable.init(); - for (String tableName : entry.getValue()) { + for (String tableName : tableNames) { String family = getColumnFamilyName(tableName); - String value = family + "_value"; - long timestamp = System.currentTimeMillis(); + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns.length; j++) { + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i])); + hTable.put(put); + } + } + } - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - hTable.put(put); + // 1. get specify column + { + int columnIndex = 0; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + Get get = new Get(key.getBytes()); + get.addColumn(family.getBytes(), columns[columnIndex].getBytes()); + Result r = hTable.get(get); + Assert.assertEquals(1, r.raw().length); + AssertKeyValue(key, columns[columnIndex], lastTs, latestValue, r.rawCells()[0]); + } + } + // 2. get do not specify column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(columns.length, cells.length); + for (int i = 0; i < columns.length; i++) { + ObHTableSecondaryPartUtil.AssertKeyValue(key, columns[i], lastTs, latestValue, cells[i]); + } + } + } + + // 3. get do not specify column family + { Get get = new Get(key.getBytes()); - get.addFamily(family.getBytes()); Result r = hTable.get(get); - Assert.assertEquals(1, r.raw().length); + Assert.assertEquals(tableNames.size() * columns.length, r.raw().length); + int cur = 0; + for (int i = 0; i < columns.length; i++) { + for (String tableName : tableNames) { + AssertKeyValue(key, columns[i], lastTs, latestValue, r.raw()[cur]); + cur++; + } + } + } - hTable.close(); + // 4. get specify multi cf and column + { + int columnIndex = 0; + Get get = new Get(key.getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addColumn(family.getBytes(), columns[columnIndex].getBytes()); + } + Result r = hTable.get(get); + Assert.assertEquals(tableNames.size(), r.rawCells().length); + for (int i = 0; i < tableNames.size(); i++) { + AssertKeyValue(key, columns[columnIndex], lastTs, latestValue, r.rawCells()[i]); + } + } + + // 5. get specify multi cf and versions + { + Get get = new Get(key.getBytes()); + get.setMaxVersions(2); + Result r = hTable.get(get); + Assert.assertEquals(tableNames.size() * columns.length * ts.length, r.raw().length); + int cur = 0; + for (int i = 0; i < columns.length; i++) { + for (String tableName : tableNames) { + for (int k = ts.length-1; k >= 0; k--) { + AssertKeyValue(key, columns[i], ts[k], values[k], r.raw()[cur]); + cur++; + } + } + } + } + + // 6. get specify multi cf and time range + { + Get get = new Get(key.getBytes()); + get.setMaxVersions(2); + get.setTimeStamp(ts[1]); + Result r = hTable.get(get); + Assert.assertEquals(tableNames.size() * columns.length, r.raw().length); + int cur = 0; + for (int i = 0; i < columns.length; i++) { + for (String tableName : tableNames) { + AssertKeyValue(key, columns[i], ts[1], values[1], r.raw()[cur]); + cur++; + } + } + } + + // 7. get specify multi cf and filter + { + Get get = new Get(key.getBytes()); + get.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(values[0]))); + get.setFilter(valueFilter); + Result r = hTable.get(get); + Assert.assertEquals(tableNames.size() * columns.length, r.raw().length); + int cur = 0; + for (int i = 0; i < columns.length; i++) { + for (String tableName : tableNames) { + AssertKeyValue(key, columns[i], ts[0], values[0], r.raw()[cur]); + cur++; + } + } } } - - + @Test public void testGet() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartGetTest::testGetImpl); diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index f0e9403f..34283f17 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -17,6 +17,10 @@ package com.alipay.oceanbase.hbase.util; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; + import java.sql.Connection; import java.sql.SQLSyntaxErrorException; import java.util.LinkedList; @@ -201,4 +205,17 @@ public static String getColumnFamilyName(String input) throws Exception { } return result; } + + public static void AssertKeyValue(String key, String qualifier, long timestamp, String value, Cell cell) { + Assert.assertEquals(key, Bytes.toString(cell.getRow())); + Assert.assertEquals(qualifier, Bytes.toString(cell.getQualifier())); + Assert.assertEquals(timestamp, cell.getTimestamp()); + Assert.assertEquals(value, Bytes.toString(cell.getValue())); + } + + public static void AssertKeyValue(String key, String qualifier, String value, Cell cell) { + Assert.assertEquals(key, Bytes.toString(cell.getRow())); + Assert.assertEquals(qualifier, Bytes.toString(cell.getQualifier())); + Assert.assertEquals(value, Bytes.toString(cell.getValue())); + } } From 67aea58bdfb35a171d13e5386c4c0ddb38952a54 Mon Sep 17 00:00:00 2001 From: shen Date: Fri, 14 Mar 2025 14:50:17 +0800 Subject: [PATCH 16/40] [Fix] fix review (cherry picked from commit aa412ea7a8c7a34f9652f24f85b806da1f606e44) --- .../hbase/secondary/OHTableSecondaryPartGetTest.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java index e06c94a3..11616adf 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -52,11 +52,8 @@ public class OHTableSecondaryPartGetTest { public static void before() throws Exception { openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { - if (type != SECONDARY_PARTITIONED_TIME_RANGE_KEY && - type != SECONDARY_PARTITIONED_TIME_KEY_RANGE && - type != NON_PARTITIONED_TIME_SERIES && - type != SINGLE_PARTITIONED_TIME_SERIES) { - createTables(type, tableNames, group2tableNames, true); + if (!type.name().contains("TIME")) { + createTables(type, tableNames, group2tableNames, true); } } } From cadb2eb2db1da24595b00cb98b8606dc8db566ce Mon Sep 17 00:00:00 2001 From: WeiXinChan Date: Fri, 14 Mar 2025 15:50:15 +0800 Subject: [PATCH 17/40] change partition defination (cherry picked from commit e7e6ba1e985b4618463915d6ac84c8602e98e2d8) --- .../oceanbase/hbase/util/TableTemplateManager.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index 983364b6..90d0e6cc 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -96,7 +96,7 @@ public enum TableType { " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + @@ -112,7 +112,7 @@ public enum TableType { " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + @@ -128,7 +128,7 @@ public enum TableType { " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + @@ -145,7 +145,7 @@ public enum TableType { " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + " `V` varbinary(1024) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + @@ -162,7 +162,7 @@ public enum TableType { " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + " `V` json NOT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + " PRIMARY KEY (`K`, `T`, `S`)\n" + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + @@ -178,7 +178,7 @@ public enum TableType { " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + " `V` json NOT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (-T)%s,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + " PRIMARY KEY (`K`, `T`, `S`)\n" + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + From 362590be2fd2d6b30a741dd3947b8c7e1045ddf0 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Fri, 14 Mar 2025 18:46:20 +0800 Subject: [PATCH 18/40] put test case (cherry picked from commit 304c1656b39dabb9ecdab2286506b3c0f2916e8d) --- .../OHTableSecondaryPartPutTest.java | 280 ++++++++++++++++-- 1 file changed, 254 insertions(+), 26 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java index b83caaae..89c3bd7d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -21,19 +21,17 @@ import com.alipay.oceanbase.hbase.OHTableClient; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.*; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; -import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.*; import static org.apache.hadoop.hbase.util.Bytes.toBytes; public class OHTableSecondaryPartPutTest { @@ -45,7 +43,9 @@ public class OHTableSecondaryPartPutTest { public static void before() throws Exception { openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { - createTables(type, tableNames, group2tableNames, true); + if (!type.name().contains("TIME")) { + createTables(type, tableNames, group2tableNames, true); + } } } @@ -66,43 +66,271 @@ public static void testPutImpl(String tableName) throws Exception { String family = getColumnFamilyName(tableName); String key = "putKey"; - String column = "putColumn"; String column1 = "putColumn1"; + String column2 = "putColumn2"; String value = "value"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(value)); - hTable.put(put); + { // put new key and get + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), toBytes(column1 + value)); + put.add(family.getBytes(), column2.getBytes(), toBytes(column2 + value)); + hTable.put(put); + + Get get = new Get(toBytes(key)); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + Result r = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(2, r.size())); + Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare((column1 + value).getBytes(), r.getValue(family.getBytes(), column1.getBytes())))); + Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare((column1 + value).getBytes(), r.getValue(family.getBytes(), column1.getBytes())))); + } + + { // put exist key and get + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), toBytes(column1 + value + timestamp)); + hTable.put(put); + + Get get = new Get(toBytes(key)); + get.addColumn(family.getBytes(), column1.getBytes()); + Result r = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(1, r.size())); + Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare(toBytes(column1 + value + timestamp), r.getValue(family.getBytes(), column1.getBytes())))); + } + + { // test timestamp update + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); + hTable.put(put); + + Get get = new Get(toBytes(key)); + get.addColumn(family.getBytes(), column1.getBytes()); + Result r = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(1, r.size())); + Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare(toBytes(column1 + value + timestamp), r.getValue(family.getBytes(), column1.getBytes())))); + Assert(tableName, ()->Assert.assertEquals(timestamp, r.rawCells()[0].getTimestamp())); + + Put put1 = new Put(toBytes(key)); + put1.add(family.getBytes(), column1.getBytes(), toBytes(column1 + value)); + hTable.put(put1); + + Result r2 = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(1, r2.size())); + Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare(toBytes(column1 + value), r2.getValue(family.getBytes(), column1.getBytes())))); + Assert(tableName, ()->Assert.assertTrue(timestamp < r2.rawCells()[0].getTimestamp())); + } + + + + hTable.close(); + } + + public static void testBatchPutImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + String value = "value"; + { + long timestamp = System.currentTimeMillis(); + List puts = new ArrayList<>(); + Get get = new Get(toBytes(key)); + for (int i = 0; i < 10; ++i) { + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp + i)); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp + i)); + puts.add(put); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + } + + hTable.batch(puts); + Result result = hTable.get(get); + Assert(tableName, ()->Assert.assertEquals(2, result.size())); + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertEquals(timestamp, cell.getTimestamp())); + } + Assert(tableName, () -> Assert.assertTrue(secureCompare((column1 + value + timestamp + 9).getBytes(), result.getValue(family.getBytes(), column1.getBytes())))); + } + { + long timestamp = System.currentTimeMillis(); + List puts = new ArrayList<>(); + List gets = new ArrayList<>(); + + for (int i = 0; i < 10; ++i) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp + i)); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp + i)); + puts.add(put); + Get get = new Get(toBytes(key + i)); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + gets.add(get); + } + + hTable.batch(puts); + + for (int i = 0; i < 10; ++i) { + Result result = hTable.get(gets.get(i)); + Assert(tableName, ()->Assert.assertEquals(2, result.size())); + for (Cell cell : result.rawCells()) { + Assert(tableName, ()->Assert.assertEquals(timestamp, cell.getTimestamp())); + } + int finalI = i; + Assert(tableName, () -> Assert.assertTrue(secureCompare((column1 + value + timestamp + finalI).getBytes(), result.getValue(family.getBytes(), column1.getBytes())))); + } + } + + hTable.close(); } public static void testMultiCFPutImpl(Map.Entry> entry) throws Exception { String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; String value = "value"; - String column = "putColumn"; - long timestamp = System.currentTimeMillis(); - - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); hTable.init(); - Put put = new Put(toBytes(key)); - for (String tableName : entry.getValue()) { - String family = getColumnFamilyName(tableName); - put.add(family.getBytes(), column.getBytes(), timestamp, toBytes(value)); + { + long currentTime = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + Get get = new Get(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.add(family.getBytes(), column1.getBytes(), toBytes(column1 + value)); + put.add(family.getBytes(), column2.getBytes(), toBytes(column2 + value)); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + } + hTable.put(put); + Result r = hTable.get(get); + Assert(entry.getValue(), ()->Assert.assertEquals(entry.getValue().size() * 2, r.size())); + long remoteTimestamp = r.rawCells()[0].getTimestamp(); + Assert(entry.getValue(), ()->Assert.assertTrue(remoteTimestamp >= currentTime)); + for (Cell cell : r.rawCells()) { + Assert(entry.getValue(), ()->Assert.assertEquals(remoteTimestamp, cell.getTimestamp())); + } + } + + { + long timestamp = System.currentTimeMillis(); + Put put = new Put(toBytes(key)); + Get get = new Get(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value)); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value)); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + } + + hTable.put(put); + Result r = hTable.get(get); + Assert(entry.getValue(), ()->Assert.assertEquals(entry.getValue().size() * 2, r.size())); + for (Cell cell : r.rawCells()) { + Assert(entry.getValue(), ()->Assert.assertEquals(timestamp, cell.getTimestamp())); + } + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column1 + value).getBytes(), r.getValue(family.getBytes(), column1.getBytes())))); + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column2 + value).getBytes(), r.getValue(family.getBytes(), column2.getBytes())))); + } } - hTable.put(put); + hTable.close(); } + public static void testMltiCFPutBatchImpl(Map.Entry> entry) throws Exception { + String key = "putKey"; + String column1 = "putColumn1"; + String column2 = "putColumn2"; + String value = "value"; + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); + hTable.init(); + + { + long timestamp = System.currentTimeMillis(); + List puts = new ArrayList<>(); + Get get = new Get(toBytes(key)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp)); + puts.add(put); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + } + hTable.batch(puts); + Result result = hTable.get(get); + Assert(entry.getValue(), ()->Assert.assertEquals(entry.getValue().size() * 2, result.size())); + for (Cell cell : result.rawCells()) { + Assert(entry.getValue(), ()->Assert.assertEquals(timestamp, cell.getTimestamp())); + } + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column1 + value + timestamp).getBytes(), result.getValue(family.getBytes(), column1.getBytes())))); + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column2 + value + timestamp).getBytes(), result.getValue(family.getBytes(), column2.getBytes())))); + } + } + + { + long timestamp = System.currentTimeMillis(); + List puts = new ArrayList<>(); + List> gets = new ArrayList<>(); + + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < 10; ++i) { + Put put = new Put(toBytes(key + i)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp)); + puts.add(put); + Get get = new Get(toBytes(key + i)); + get.addColumn(family.getBytes(), column1.getBytes()); + get.addColumn(family.getBytes(), column2.getBytes()); + gets.add(new Pair<>(get, family)); + } + } + hTable.batch(puts); + for (int i = 0; i < 10; ++i) { + Result result = hTable.get(gets.get(i).getFirst()); + Assert(entry.getValue(), () -> Assert.assertEquals(2, result.size())); + for (Cell cell : result.rawCells()) { + Assert(entry.getValue(), () -> Assert.assertEquals(timestamp, cell.getTimestamp())); + } + int finalI = i; + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column1 + value + timestamp).getBytes(), result.getValue(gets.get(finalI).getSecond().getBytes(), column1.getBytes())))); + Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column2 + value + timestamp).getBytes(), result.getValue(gets.get(finalI).getSecond().getBytes(), column2.getBytes())))); + } + + } + hTable.close(); + } + + @Test public void testPut() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartPutTest::testPutImpl); } - + + @Test + public void testBatchPut() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartPutTest::testBatchPutImpl); + } + @Test public void testMultiCFPut() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMultiCFPutImpl); } + + @Test + public void testMultiCFPutBatch() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMltiCFPutBatchImpl); + } } From 659a7afcc7c75a2bc8ec2727aca5de217085e838 Mon Sep 17 00:00:00 2001 From: GroundWu <1175416256@qq.com> Date: Tue, 18 Mar 2025 17:57:52 +0800 Subject: [PATCH 19/40] add batch test case (cherry picked from commit d88bc54579f106cd0397712f52ba457f5e60e419) --- .../OHTableSecondaryPartBatchTest.java | 465 ++++++++++++++++++ 1 file changed, 465 insertions(+) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java new file mode 100644 index 00000000..0c4a1351 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java @@ -0,0 +1,465 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableSecondaryPartBatchTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testBatchPutImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + List puts = new ArrayList<>(); + long timestamp = System.currentTimeMillis() - 100; + int batchSize = 10; + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + put.addColumn(toBytes(family), toBytes("col1"), timestamp ,toBytes("value-1-" + i)); + put.addColumn(toBytes(family), toBytes("col2"), timestamp, toBytes("value-2-" + i)); + put.addColumn(toBytes(family), toBytes("col3") ,toBytes("value-3-" + i)); + put.addColumn(toBytes(family), toBytes("col4"), toBytes("value-4-" + i)); + puts.add(put); + } + hTable.put(puts); + + // verify result + long timestamp2 = System.currentTimeMillis(); + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes("row-" + i)); + get.addColumn(toBytes(family), toBytes("col1")); + get.addColumn(toBytes(family), toBytes("col2")); + get.addColumn(toBytes(family), toBytes("col3")); + get.addColumn(toBytes(family), toBytes("col4")); + + get.setMaxVersions(1); + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(4, result.size())); + for (Cell cell: result.listCells()) { + Assert.assertEquals(family, Bytes.toString(CellUtil.cloneFamily(cell))); + String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); + if (Q.equals("col1")) { + Assert.assertEquals("value-1-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col2")) { + Assert.assertEquals("value-2-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col3")) { + Assert.assertEquals("value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp < cell.getTimestamp()); + timestamp2 = cell.getTimestamp(); + } else if (Q.equals("col4")) { + Assert.assertEquals("value-4-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp < cell.getTimestamp()); + } else { + Assert.fail(); + } + } + } + + // put exist key + puts.clear(); + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + // update + put.addColumn(toBytes(family), toBytes("col1"), timestamp ,toBytes("update-value-1-" + i)); + put.addColumn(toBytes(family), toBytes("col2"), timestamp, toBytes("update-value-2-" + i)); + // insert + put.addColumn(toBytes(family), toBytes("col3") ,toBytes("update-value-3-" + i)); + put.addColumn(toBytes(family), toBytes("col4"), toBytes("update-value-4-" + i)); + puts.add(put); + } + hTable.put(puts); + + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes("row-" + i)); + get.addColumn(toBytes(family), toBytes("col1")); + get.addColumn(toBytes(family), toBytes("col2")); + get.addColumn(toBytes(family), toBytes("col3")); + get.addColumn(toBytes(family), toBytes("col4")); + get.setMaxVersions(); + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(6, result.size())); + for (Cell cell: result.listCells()) { + Assert.assertEquals(family, Bytes.toString(CellUtil.cloneFamily(cell))); + String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); + if (Q.equals("col1")) { + Assert.assertEquals("update-value-1-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col2")) { + Assert.assertEquals("update-value-2-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col3")) { + if (timestamp2 == cell.getTimestamp()) { + Assert.assertEquals("value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + } else { + Assert.assertEquals("update-value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp2 < cell.getTimestamp()); + } + } else if (Q.equals("col4")) { + if (timestamp2 == cell.getTimestamp()) { + Assert.assertEquals("value-4-" + i, Bytes.toString(CellUtil.cloneValue(cell))); + } else { + Assert.assertEquals("update-value-4-" + i, Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp2 < cell.getTimestamp()); + } + } else { + Assert.fail(); + } + } + } + hTable.close(); + } + + public static void testMultiCFBatchPutImpl(Map.Entry> entry) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + int batchSize = 10; + List puts = new ArrayList<>(); + String[] qualifier = new String[] {"col-1", "col-2", "col-3", "col-4"}; + String[] value = new String[] {"value-1", "value-2", "value-3", "value-4"}; + long timestamp = System.currentTimeMillis(); + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.add(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(value[0])); + put.add(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(value[1])); + put.add(toBytes(family), toBytes(qualifier[2]), toBytes(value[2])); + put.add(toBytes(family), toBytes(qualifier[3]), toBytes(value[3])); + } + puts.add(put); + } + hTable.put(puts); + // verify initial put result + for (int i = 0; i < batchSize; i++) { + for (String tableName : entry.getValue()) { + Get get = new Get(toBytes("row-" + i)); + get.setMaxVersions(1); + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(qualifier[0])); + get.addColumn(toBytes(family), toBytes(qualifier[1])); + get.addColumn(toBytes(family), toBytes(qualifier[2])); + get.addColumn(toBytes(family), toBytes(qualifier[3])); + + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(4, result.size())); + Assert.assertEquals(value[0], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[0])))); + Assert.assertEquals(timestamp, result.getColumnLatestCell(toBytes(family), toBytes(qualifier[0])).getTimestamp()); + Assert.assertEquals(value[1], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[1])))); + Assert.assertEquals(timestamp, result.getColumnLatestCell(toBytes(family), toBytes(qualifier[1])).getTimestamp()); + Assert.assertEquals(value[2], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[2])))); + Assert.assertTrue(timestamp < result.getColumnLatestCell(toBytes(family), toBytes(qualifier[2])).getTimestamp()); + Assert.assertEquals(value[3], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[3])))); + Assert.assertTrue(timestamp < result.getColumnLatestCell(toBytes(family), toBytes(qualifier[3])).getTimestamp()); + } + } + + // put exist key + puts.clear(); + String[] updateValue = new String[] {"update-value-1", "update-value-2", "update-value-3", "update-value-4"}; + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.add(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(updateValue[0])); + put.add(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(updateValue[1])); + put.add(toBytes(family), toBytes(qualifier[2]), toBytes(updateValue[2])); + put.add(toBytes(family), toBytes(qualifier[3]), toBytes(updateValue[3])); + } + puts.add(put); + } + hTable.put(puts); + + // verify update result + for (int i = 0; i < batchSize; i++) { + for (String tableName : entry.getValue()) { + Get get = new Get(toBytes("row-" + i)); + get.setMaxVersions(); // Get both versions + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(qualifier[0])); + get.addColumn(toBytes(family), toBytes(qualifier[1])); + get.addColumn(toBytes(family), toBytes(qualifier[2])); + get.addColumn(toBytes(family), toBytes(qualifier[3])); + + // Verify values + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(6, result.size())); + // qualifier[0] - 1 versions + List col0Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[0])); + Assert.assertEquals(1, col0Cells.size()); + Assert.assertEquals(updateValue[0], Bytes.toString(CellUtil.cloneValue(col0Cells.get(0)))); + Assert.assertEquals(timestamp, col0Cells.get(0).getTimestamp()); + + // qualifier[1] - 1 versions + List col1Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[1])); + Assert.assertEquals(1, col1Cells.size()); + Assert.assertEquals(updateValue[1], Bytes.toString(CellUtil.cloneValue(col1Cells.get(0)))); + Assert.assertEquals(timestamp, col1Cells.get(0).getTimestamp()); + + // qualifier[2] - 2 version + List col2Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[2])); + Assert.assertEquals(2, col2Cells.size()); + Assert.assertEquals(updateValue[2], Bytes.toString(CellUtil.cloneValue(col2Cells.get(0)))); + Assert.assertTrue(timestamp < col2Cells.get(0).getTimestamp()); + Assert.assertEquals(value[2], Bytes.toString(CellUtil.cloneValue(col2Cells.get(1)))); + + // qualifier[3] - 2 version + List col3Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[3])); + Assert.assertEquals(2, col3Cells.size()); + Assert.assertEquals(updateValue[3], Bytes.toString(CellUtil.cloneValue(col3Cells.get(0)))); + Assert.assertTrue(timestamp < col3Cells.get(0).getTimestamp()); + Assert.assertEquals(value[3], Bytes.toString(CellUtil.cloneValue(col3Cells.get(1)))); + + } + } + + hTable.close(); + } + + public static void testBatchGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String keyPrefix = "putKey_"; + String family = getColumnFamilyName(tableName); + String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; + long curTs = System.currentTimeMillis(); + long[] ts = {curTs, curTs - 100}; + String valuePrefix = "value_"; + int batchSize = 10; + // prepare data + List puts = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes(keyPrefix + i)); + for (int j = 0; j < columns.length; j++) { + put.add(toBytes(family), toBytes(columns[j]), ts[0], toBytes(valuePrefix + "1_" +i)); + put.add(toBytes(family), toBytes(columns[j]), ts[1], toBytes(valuePrefix + "2_" +i)); + } + puts.add(put); + } + hTable.put(puts); + // start to batch get test + List gets = new ArrayList<>(); + // 1. get specify column + { + int index = 0; + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.addColumn(family.getBytes(), columns[index].getBytes()); + get.setMaxVersions(); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(2, r.size()); + List cells = r.listCells(); + for (int j = 0; j < r.size(); j++) { + Cell cell = cells.get(j); + Assert.assertEquals(keyPrefix+i, Bytes.toString(cell.getRow())); + Assert.assertEquals(columns[index], Bytes.toString(cell.getQualifier())); + Assert.assertEquals(ts[j], cell.getTimestamp()); + if (j == 0) { + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cell.getValue())); + } else { + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cell.getValue())); + } + } + } + } + gets.clear(); + // 2. get do not specify column + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.setMaxVersions(); + get.addFamily(family.getBytes()); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(6, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(toBytes(family), toBytes(columns[j])); + Assert.assertEquals(2, cells.size()); + for (int k = 0; k < cells.size(); k++) { + Assert.assertEquals(ts[k], cells.get(k).getTimestamp()); + if (k == 0) { + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cells.get(k).getValue())); + } else { + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(k).getValue())); + } + } + } + } + } + // 3. get specify version + gets.clear(); + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(6, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); + Assert.assertEquals(2, cells.size()); + for (int k = 0; k < cells.size(); k++) { + Assert.assertEquals(ts[k], cells.get(k).getTimestamp()); + if (k == 0) { + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cells.get(k).getValue())); + } else { + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(k).getValue())); + } + } + } + } + } + // 4. get specify time range + gets.clear(); + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + get.setTimeStamp(ts[1]); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(3, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); + Assert.assertEquals(1, cells.size()); + Assert.assertEquals(ts[1], cells.get(0).getTimestamp()); + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(0).getValue())); + } + } + } + // 5. get specify filter + gets.clear(); + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix+i)); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(valuePrefix+"2_"+i))); + get.setFilter(valueFilter); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(3, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); + Assert.assertEquals(1, cells.size()); + Assert.assertEquals(ts[1], cells.get(0).getTimestamp()); + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(0).getValue())); + } + } + } + hTable.close(); + } + + + public static void testMultiCFBatchGetImpl(Map.Entry> entry) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); + hTable.init(); + // prepare data + } + + @Test + public void testBatchPut() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testBatchPutImpl); + truncateTables(ObHTableTestUtil.getConnection(), tableNames); + } + + @Test + public void testMultiCFPut() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchPutImpl); + truncateTables(ObHTableTestUtil.getConnection(), group2tableNames); + } + + @Test + public void testBatchGet() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testBatchGetImpl); + truncateTables(ObHTableTestUtil.getConnection(), tableNames); + } + + @Test + public void testMultiCFGet() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchGetImpl); + truncateTables(ObHTableTestUtil.getConnection(), group2tableNames); + } + +} From 5a9f0e8f5dc5e6b6274cfdd7a247fd5e4c013618 Mon Sep 17 00:00:00 2001 From: GroundWu <1175416256@qq.com> Date: Tue, 18 Mar 2025 17:59:47 +0800 Subject: [PATCH 20/40] Revert "add batch test case" This reverts commit d88bc54579f106cd0397712f52ba457f5e60e419. (cherry picked from commit 8a33b64ee189bf59c4ffdc4b9bcc5331f65a3bf5) --- .../OHTableSecondaryPartBatchTest.java | 465 ------------------ 1 file changed, 465 deletions(-) delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java deleted file mode 100644 index 0c4a1351..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java +++ /dev/null @@ -1,465 +0,0 @@ -/*- - * #%L - * OBKV HBase Client Framework - * %% - * Copyright (C) 2025 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - - -package com.alipay.oceanbase.hbase.secondary; - -import com.alipay.oceanbase.hbase.OHTableClient; -import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; -import com.alipay.oceanbase.hbase.util.TableTemplateManager; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.filter.BinaryComparator; -import org.apache.hadoop.hbase.filter.CompareFilter; -import org.apache.hadoop.hbase.filter.ValueFilter; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.*; - -import java.util.*; - -import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; -import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; -import static org.apache.hadoop.hbase.util.Bytes.toBytes; - -public class OHTableSecondaryPartBatchTest { - private static List tableNames = new LinkedList(); - private static Map> group2tableNames = new LinkedHashMap<>(); - - - @BeforeClass - public static void before() throws Exception { - openDistributedExecute(); - for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { - createTables(type, tableNames, group2tableNames, true); - } - } - - @AfterClass - public static void finish() throws Exception { - closeDistributedExecute(); - dropTables(tableNames, group2tableNames); - } - - @Before - public void prepareCase() throws Exception { - truncateTables(tableNames, group2tableNames); - } - - public static void testBatchPutImpl(String tableName) throws Exception { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - String family = getColumnFamilyName(tableName); - List puts = new ArrayList<>(); - long timestamp = System.currentTimeMillis() - 100; - int batchSize = 10; - for (int i = 0; i < batchSize; i++) { - Put put = new Put(toBytes("row-" + i)); - put.addColumn(toBytes(family), toBytes("col1"), timestamp ,toBytes("value-1-" + i)); - put.addColumn(toBytes(family), toBytes("col2"), timestamp, toBytes("value-2-" + i)); - put.addColumn(toBytes(family), toBytes("col3") ,toBytes("value-3-" + i)); - put.addColumn(toBytes(family), toBytes("col4"), toBytes("value-4-" + i)); - puts.add(put); - } - hTable.put(puts); - - // verify result - long timestamp2 = System.currentTimeMillis(); - for (int i = 0; i < batchSize; i++) { - Get get = new Get(toBytes("row-" + i)); - get.addColumn(toBytes(family), toBytes("col1")); - get.addColumn(toBytes(family), toBytes("col2")); - get.addColumn(toBytes(family), toBytes("col3")); - get.addColumn(toBytes(family), toBytes("col4")); - - get.setMaxVersions(1); - Result result = hTable.get(get); - ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(4, result.size())); - for (Cell cell: result.listCells()) { - Assert.assertEquals(family, Bytes.toString(CellUtil.cloneFamily(cell))); - String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); - if (Q.equals("col1")) { - Assert.assertEquals("value-1-" + i,Bytes.toString(CellUtil.cloneValue(cell))); - Assert.assertEquals(timestamp, cell.getTimestamp()); - } else if (Q.equals("col2")) { - Assert.assertEquals("value-2-" + i,Bytes.toString(CellUtil.cloneValue(cell))); - Assert.assertEquals(timestamp, cell.getTimestamp()); - } else if (Q.equals("col3")) { - Assert.assertEquals("value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); - Assert.assertTrue(timestamp < cell.getTimestamp()); - timestamp2 = cell.getTimestamp(); - } else if (Q.equals("col4")) { - Assert.assertEquals("value-4-" + i,Bytes.toString(CellUtil.cloneValue(cell))); - Assert.assertTrue(timestamp < cell.getTimestamp()); - } else { - Assert.fail(); - } - } - } - - // put exist key - puts.clear(); - for (int i = 0; i < batchSize; i++) { - Put put = new Put(toBytes("row-" + i)); - // update - put.addColumn(toBytes(family), toBytes("col1"), timestamp ,toBytes("update-value-1-" + i)); - put.addColumn(toBytes(family), toBytes("col2"), timestamp, toBytes("update-value-2-" + i)); - // insert - put.addColumn(toBytes(family), toBytes("col3") ,toBytes("update-value-3-" + i)); - put.addColumn(toBytes(family), toBytes("col4"), toBytes("update-value-4-" + i)); - puts.add(put); - } - hTable.put(puts); - - for (int i = 0; i < batchSize; i++) { - Get get = new Get(toBytes("row-" + i)); - get.addColumn(toBytes(family), toBytes("col1")); - get.addColumn(toBytes(family), toBytes("col2")); - get.addColumn(toBytes(family), toBytes("col3")); - get.addColumn(toBytes(family), toBytes("col4")); - get.setMaxVersions(); - Result result = hTable.get(get); - ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(6, result.size())); - for (Cell cell: result.listCells()) { - Assert.assertEquals(family, Bytes.toString(CellUtil.cloneFamily(cell))); - String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); - if (Q.equals("col1")) { - Assert.assertEquals("update-value-1-" + i,Bytes.toString(CellUtil.cloneValue(cell))); - Assert.assertEquals(timestamp, cell.getTimestamp()); - } else if (Q.equals("col2")) { - Assert.assertEquals("update-value-2-" + i,Bytes.toString(CellUtil.cloneValue(cell))); - Assert.assertEquals(timestamp, cell.getTimestamp()); - } else if (Q.equals("col3")) { - if (timestamp2 == cell.getTimestamp()) { - Assert.assertEquals("value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); - } else { - Assert.assertEquals("update-value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); - Assert.assertTrue(timestamp2 < cell.getTimestamp()); - } - } else if (Q.equals("col4")) { - if (timestamp2 == cell.getTimestamp()) { - Assert.assertEquals("value-4-" + i, Bytes.toString(CellUtil.cloneValue(cell))); - } else { - Assert.assertEquals("update-value-4-" + i, Bytes.toString(CellUtil.cloneValue(cell))); - Assert.assertTrue(timestamp2 < cell.getTimestamp()); - } - } else { - Assert.fail(); - } - } - } - hTable.close(); - } - - public static void testMultiCFBatchPutImpl(Map.Entry> entry) throws Exception { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); - hTable.init(); - int batchSize = 10; - List puts = new ArrayList<>(); - String[] qualifier = new String[] {"col-1", "col-2", "col-3", "col-4"}; - String[] value = new String[] {"value-1", "value-2", "value-3", "value-4"}; - long timestamp = System.currentTimeMillis(); - for (int i = 0; i < batchSize; i++) { - Put put = new Put(toBytes("row-" + i)); - for (String tableName : entry.getValue()) { - String family = getColumnFamilyName(tableName); - put.add(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(value[0])); - put.add(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(value[1])); - put.add(toBytes(family), toBytes(qualifier[2]), toBytes(value[2])); - put.add(toBytes(family), toBytes(qualifier[3]), toBytes(value[3])); - } - puts.add(put); - } - hTable.put(puts); - // verify initial put result - for (int i = 0; i < batchSize; i++) { - for (String tableName : entry.getValue()) { - Get get = new Get(toBytes("row-" + i)); - get.setMaxVersions(1); - String family = getColumnFamilyName(tableName); - get.addColumn(toBytes(family), toBytes(qualifier[0])); - get.addColumn(toBytes(family), toBytes(qualifier[1])); - get.addColumn(toBytes(family), toBytes(qualifier[2])); - get.addColumn(toBytes(family), toBytes(qualifier[3])); - - Result result = hTable.get(get); - ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(4, result.size())); - Assert.assertEquals(value[0], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[0])))); - Assert.assertEquals(timestamp, result.getColumnLatestCell(toBytes(family), toBytes(qualifier[0])).getTimestamp()); - Assert.assertEquals(value[1], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[1])))); - Assert.assertEquals(timestamp, result.getColumnLatestCell(toBytes(family), toBytes(qualifier[1])).getTimestamp()); - Assert.assertEquals(value[2], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[2])))); - Assert.assertTrue(timestamp < result.getColumnLatestCell(toBytes(family), toBytes(qualifier[2])).getTimestamp()); - Assert.assertEquals(value[3], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[3])))); - Assert.assertTrue(timestamp < result.getColumnLatestCell(toBytes(family), toBytes(qualifier[3])).getTimestamp()); - } - } - - // put exist key - puts.clear(); - String[] updateValue = new String[] {"update-value-1", "update-value-2", "update-value-3", "update-value-4"}; - for (int i = 0; i < batchSize; i++) { - Put put = new Put(toBytes("row-" + i)); - for (String tableName : entry.getValue()) { - String family = getColumnFamilyName(tableName); - put.add(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(updateValue[0])); - put.add(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(updateValue[1])); - put.add(toBytes(family), toBytes(qualifier[2]), toBytes(updateValue[2])); - put.add(toBytes(family), toBytes(qualifier[3]), toBytes(updateValue[3])); - } - puts.add(put); - } - hTable.put(puts); - - // verify update result - for (int i = 0; i < batchSize; i++) { - for (String tableName : entry.getValue()) { - Get get = new Get(toBytes("row-" + i)); - get.setMaxVersions(); // Get both versions - String family = getColumnFamilyName(tableName); - get.addColumn(toBytes(family), toBytes(qualifier[0])); - get.addColumn(toBytes(family), toBytes(qualifier[1])); - get.addColumn(toBytes(family), toBytes(qualifier[2])); - get.addColumn(toBytes(family), toBytes(qualifier[3])); - - // Verify values - Result result = hTable.get(get); - ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(6, result.size())); - // qualifier[0] - 1 versions - List col0Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[0])); - Assert.assertEquals(1, col0Cells.size()); - Assert.assertEquals(updateValue[0], Bytes.toString(CellUtil.cloneValue(col0Cells.get(0)))); - Assert.assertEquals(timestamp, col0Cells.get(0).getTimestamp()); - - // qualifier[1] - 1 versions - List col1Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[1])); - Assert.assertEquals(1, col1Cells.size()); - Assert.assertEquals(updateValue[1], Bytes.toString(CellUtil.cloneValue(col1Cells.get(0)))); - Assert.assertEquals(timestamp, col1Cells.get(0).getTimestamp()); - - // qualifier[2] - 2 version - List col2Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[2])); - Assert.assertEquals(2, col2Cells.size()); - Assert.assertEquals(updateValue[2], Bytes.toString(CellUtil.cloneValue(col2Cells.get(0)))); - Assert.assertTrue(timestamp < col2Cells.get(0).getTimestamp()); - Assert.assertEquals(value[2], Bytes.toString(CellUtil.cloneValue(col2Cells.get(1)))); - - // qualifier[3] - 2 version - List col3Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[3])); - Assert.assertEquals(2, col3Cells.size()); - Assert.assertEquals(updateValue[3], Bytes.toString(CellUtil.cloneValue(col3Cells.get(0)))); - Assert.assertTrue(timestamp < col3Cells.get(0).getTimestamp()); - Assert.assertEquals(value[3], Bytes.toString(CellUtil.cloneValue(col3Cells.get(1)))); - - } - } - - hTable.close(); - } - - public static void testBatchGetImpl(String tableName) throws Exception { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - String keyPrefix = "putKey_"; - String family = getColumnFamilyName(tableName); - String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; - long curTs = System.currentTimeMillis(); - long[] ts = {curTs, curTs - 100}; - String valuePrefix = "value_"; - int batchSize = 10; - // prepare data - List puts = new ArrayList<>(); - for (int i = 0; i < batchSize; i++) { - Put put = new Put(toBytes(keyPrefix + i)); - for (int j = 0; j < columns.length; j++) { - put.add(toBytes(family), toBytes(columns[j]), ts[0], toBytes(valuePrefix + "1_" +i)); - put.add(toBytes(family), toBytes(columns[j]), ts[1], toBytes(valuePrefix + "2_" +i)); - } - puts.add(put); - } - hTable.put(puts); - // start to batch get test - List gets = new ArrayList<>(); - // 1. get specify column - { - int index = 0; - for (int i = 0; i < batchSize; i++) { - Get get = new Get(toBytes(keyPrefix + i)); - get.addColumn(family.getBytes(), columns[index].getBytes()); - get.setMaxVersions(); - gets.add(get); - } - Result[] batchResults = hTable.get(gets); - Assert.assertEquals(batchSize, batchResults.length); - for (int i = 0; i < batchSize; i++) { - Result r = batchResults[i]; - Assert.assertEquals(2, r.size()); - List cells = r.listCells(); - for (int j = 0; j < r.size(); j++) { - Cell cell = cells.get(j); - Assert.assertEquals(keyPrefix+i, Bytes.toString(cell.getRow())); - Assert.assertEquals(columns[index], Bytes.toString(cell.getQualifier())); - Assert.assertEquals(ts[j], cell.getTimestamp()); - if (j == 0) { - Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cell.getValue())); - } else { - Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cell.getValue())); - } - } - } - } - gets.clear(); - // 2. get do not specify column - { - for (int i = 0; i < batchSize; i++) { - Get get = new Get(toBytes(keyPrefix + i)); - get.setMaxVersions(); - get.addFamily(family.getBytes()); - gets.add(get); - } - Result[] batchResults = hTable.get(gets); - Assert.assertEquals(batchSize, batchResults.length); - for (int i = 0; i < batchSize; i++) { - Result r = batchResults[i]; - Assert.assertEquals(6, r.size()); - for (int j = 0; j < columns.length; j++) { - List cells = r.getColumnCells(toBytes(family), toBytes(columns[j])); - Assert.assertEquals(2, cells.size()); - for (int k = 0; k < cells.size(); k++) { - Assert.assertEquals(ts[k], cells.get(k).getTimestamp()); - if (k == 0) { - Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cells.get(k).getValue())); - } else { - Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(k).getValue())); - } - } - } - } - } - // 3. get specify version - gets.clear(); - { - for (int i = 0; i < batchSize; i++) { - Get get = new Get(toBytes(keyPrefix + i)); - get.addFamily(family.getBytes()); - get.setMaxVersions(2); - gets.add(get); - } - Result[] batchResults = hTable.get(gets); - Assert.assertEquals(batchSize, batchResults.length); - for (int i = 0; i < batchSize; i++) { - Result r = batchResults[i]; - Assert.assertEquals(6, r.size()); - for (int j = 0; j < columns.length; j++) { - List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); - Assert.assertEquals(2, cells.size()); - for (int k = 0; k < cells.size(); k++) { - Assert.assertEquals(ts[k], cells.get(k).getTimestamp()); - if (k == 0) { - Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cells.get(k).getValue())); - } else { - Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(k).getValue())); - } - } - } - } - } - // 4. get specify time range - gets.clear(); - { - for (int i = 0; i < batchSize; i++) { - Get get = new Get(toBytes(keyPrefix + i)); - get.addFamily(family.getBytes()); - get.setMaxVersions(2); - get.setTimeStamp(ts[1]); - gets.add(get); - } - Result[] batchResults = hTable.get(gets); - Assert.assertEquals(batchSize, batchResults.length); - for (int i = 0; i < batchSize; i++) { - Result r = batchResults[i]; - Assert.assertEquals(3, r.size()); - for (int j = 0; j < columns.length; j++) { - List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); - Assert.assertEquals(1, cells.size()); - Assert.assertEquals(ts[1], cells.get(0).getTimestamp()); - Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(0).getValue())); - } - } - } - // 5. get specify filter - gets.clear(); - { - for (int i = 0; i < batchSize; i++) { - Get get = new Get(toBytes(keyPrefix+i)); - get.addFamily(family.getBytes()); - get.setMaxVersions(2); - ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(valuePrefix+"2_"+i))); - get.setFilter(valueFilter); - gets.add(get); - } - Result[] batchResults = hTable.get(gets); - Assert.assertEquals(batchSize, batchResults.length); - for (int i = 0; i < batchSize; i++) { - Result r = batchResults[i]; - Assert.assertEquals(3, r.size()); - for (int j = 0; j < columns.length; j++) { - List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); - Assert.assertEquals(1, cells.size()); - Assert.assertEquals(ts[1], cells.get(0).getTimestamp()); - Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(0).getValue())); - } - } - } - hTable.close(); - } - - - public static void testMultiCFBatchGetImpl(Map.Entry> entry) throws Exception { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); - hTable.init(); - // prepare data - } - - @Test - public void testBatchPut() throws Throwable { - FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testBatchPutImpl); - truncateTables(ObHTableTestUtil.getConnection(), tableNames); - } - - @Test - public void testMultiCFPut() throws Throwable { - FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchPutImpl); - truncateTables(ObHTableTestUtil.getConnection(), group2tableNames); - } - - @Test - public void testBatchGet() throws Throwable { - FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testBatchGetImpl); - truncateTables(ObHTableTestUtil.getConnection(), tableNames); - } - - @Test - public void testMultiCFGet() throws Throwable { - FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchGetImpl); - truncateTables(ObHTableTestUtil.getConnection(), group2tableNames); - } - -} From 473b35ad049d0049b46d42309e75d6c9d018838a Mon Sep 17 00:00:00 2001 From: GroundWu <1175416256@qq.com> Date: Tue, 18 Mar 2025 17:57:52 +0800 Subject: [PATCH 21/40] add batch test case (cherry picked from commit 5d23854e3ded8ede2db63422a0f487d626e113f6) --- .../OHTableSecondaryPartBatchTest.java | 465 ++++++++++++++++++ 1 file changed, 465 insertions(+) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java new file mode 100644 index 00000000..0c4a1351 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java @@ -0,0 +1,465 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; + +public class OHTableSecondaryPartBatchTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testBatchPutImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + List puts = new ArrayList<>(); + long timestamp = System.currentTimeMillis() - 100; + int batchSize = 10; + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + put.addColumn(toBytes(family), toBytes("col1"), timestamp ,toBytes("value-1-" + i)); + put.addColumn(toBytes(family), toBytes("col2"), timestamp, toBytes("value-2-" + i)); + put.addColumn(toBytes(family), toBytes("col3") ,toBytes("value-3-" + i)); + put.addColumn(toBytes(family), toBytes("col4"), toBytes("value-4-" + i)); + puts.add(put); + } + hTable.put(puts); + + // verify result + long timestamp2 = System.currentTimeMillis(); + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes("row-" + i)); + get.addColumn(toBytes(family), toBytes("col1")); + get.addColumn(toBytes(family), toBytes("col2")); + get.addColumn(toBytes(family), toBytes("col3")); + get.addColumn(toBytes(family), toBytes("col4")); + + get.setMaxVersions(1); + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(4, result.size())); + for (Cell cell: result.listCells()) { + Assert.assertEquals(family, Bytes.toString(CellUtil.cloneFamily(cell))); + String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); + if (Q.equals("col1")) { + Assert.assertEquals("value-1-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col2")) { + Assert.assertEquals("value-2-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col3")) { + Assert.assertEquals("value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp < cell.getTimestamp()); + timestamp2 = cell.getTimestamp(); + } else if (Q.equals("col4")) { + Assert.assertEquals("value-4-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp < cell.getTimestamp()); + } else { + Assert.fail(); + } + } + } + + // put exist key + puts.clear(); + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + // update + put.addColumn(toBytes(family), toBytes("col1"), timestamp ,toBytes("update-value-1-" + i)); + put.addColumn(toBytes(family), toBytes("col2"), timestamp, toBytes("update-value-2-" + i)); + // insert + put.addColumn(toBytes(family), toBytes("col3") ,toBytes("update-value-3-" + i)); + put.addColumn(toBytes(family), toBytes("col4"), toBytes("update-value-4-" + i)); + puts.add(put); + } + hTable.put(puts); + + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes("row-" + i)); + get.addColumn(toBytes(family), toBytes("col1")); + get.addColumn(toBytes(family), toBytes("col2")); + get.addColumn(toBytes(family), toBytes("col3")); + get.addColumn(toBytes(family), toBytes("col4")); + get.setMaxVersions(); + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(6, result.size())); + for (Cell cell: result.listCells()) { + Assert.assertEquals(family, Bytes.toString(CellUtil.cloneFamily(cell))); + String Q = Bytes.toString(CellUtil.cloneQualifier(cell)); + if (Q.equals("col1")) { + Assert.assertEquals("update-value-1-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col2")) { + Assert.assertEquals("update-value-2-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertEquals(timestamp, cell.getTimestamp()); + } else if (Q.equals("col3")) { + if (timestamp2 == cell.getTimestamp()) { + Assert.assertEquals("value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + } else { + Assert.assertEquals("update-value-3-" + i,Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp2 < cell.getTimestamp()); + } + } else if (Q.equals("col4")) { + if (timestamp2 == cell.getTimestamp()) { + Assert.assertEquals("value-4-" + i, Bytes.toString(CellUtil.cloneValue(cell))); + } else { + Assert.assertEquals("update-value-4-" + i, Bytes.toString(CellUtil.cloneValue(cell))); + Assert.assertTrue(timestamp2 < cell.getTimestamp()); + } + } else { + Assert.fail(); + } + } + } + hTable.close(); + } + + public static void testMultiCFBatchPutImpl(Map.Entry> entry) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(entry.getKey()); + hTable.init(); + int batchSize = 10; + List puts = new ArrayList<>(); + String[] qualifier = new String[] {"col-1", "col-2", "col-3", "col-4"}; + String[] value = new String[] {"value-1", "value-2", "value-3", "value-4"}; + long timestamp = System.currentTimeMillis(); + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.add(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(value[0])); + put.add(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(value[1])); + put.add(toBytes(family), toBytes(qualifier[2]), toBytes(value[2])); + put.add(toBytes(family), toBytes(qualifier[3]), toBytes(value[3])); + } + puts.add(put); + } + hTable.put(puts); + // verify initial put result + for (int i = 0; i < batchSize; i++) { + for (String tableName : entry.getValue()) { + Get get = new Get(toBytes("row-" + i)); + get.setMaxVersions(1); + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(qualifier[0])); + get.addColumn(toBytes(family), toBytes(qualifier[1])); + get.addColumn(toBytes(family), toBytes(qualifier[2])); + get.addColumn(toBytes(family), toBytes(qualifier[3])); + + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(4, result.size())); + Assert.assertEquals(value[0], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[0])))); + Assert.assertEquals(timestamp, result.getColumnLatestCell(toBytes(family), toBytes(qualifier[0])).getTimestamp()); + Assert.assertEquals(value[1], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[1])))); + Assert.assertEquals(timestamp, result.getColumnLatestCell(toBytes(family), toBytes(qualifier[1])).getTimestamp()); + Assert.assertEquals(value[2], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[2])))); + Assert.assertTrue(timestamp < result.getColumnLatestCell(toBytes(family), toBytes(qualifier[2])).getTimestamp()); + Assert.assertEquals(value[3], Bytes.toString(result.getValue(toBytes(family), toBytes(qualifier[3])))); + Assert.assertTrue(timestamp < result.getColumnLatestCell(toBytes(family), toBytes(qualifier[3])).getTimestamp()); + } + } + + // put exist key + puts.clear(); + String[] updateValue = new String[] {"update-value-1", "update-value-2", "update-value-3", "update-value-4"}; + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes("row-" + i)); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + put.add(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(updateValue[0])); + put.add(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(updateValue[1])); + put.add(toBytes(family), toBytes(qualifier[2]), toBytes(updateValue[2])); + put.add(toBytes(family), toBytes(qualifier[3]), toBytes(updateValue[3])); + } + puts.add(put); + } + hTable.put(puts); + + // verify update result + for (int i = 0; i < batchSize; i++) { + for (String tableName : entry.getValue()) { + Get get = new Get(toBytes("row-" + i)); + get.setMaxVersions(); // Get both versions + String family = getColumnFamilyName(tableName); + get.addColumn(toBytes(family), toBytes(qualifier[0])); + get.addColumn(toBytes(family), toBytes(qualifier[1])); + get.addColumn(toBytes(family), toBytes(qualifier[2])); + get.addColumn(toBytes(family), toBytes(qualifier[3])); + + // Verify values + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(6, result.size())); + // qualifier[0] - 1 versions + List col0Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[0])); + Assert.assertEquals(1, col0Cells.size()); + Assert.assertEquals(updateValue[0], Bytes.toString(CellUtil.cloneValue(col0Cells.get(0)))); + Assert.assertEquals(timestamp, col0Cells.get(0).getTimestamp()); + + // qualifier[1] - 1 versions + List col1Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[1])); + Assert.assertEquals(1, col1Cells.size()); + Assert.assertEquals(updateValue[1], Bytes.toString(CellUtil.cloneValue(col1Cells.get(0)))); + Assert.assertEquals(timestamp, col1Cells.get(0).getTimestamp()); + + // qualifier[2] - 2 version + List col2Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[2])); + Assert.assertEquals(2, col2Cells.size()); + Assert.assertEquals(updateValue[2], Bytes.toString(CellUtil.cloneValue(col2Cells.get(0)))); + Assert.assertTrue(timestamp < col2Cells.get(0).getTimestamp()); + Assert.assertEquals(value[2], Bytes.toString(CellUtil.cloneValue(col2Cells.get(1)))); + + // qualifier[3] - 2 version + List col3Cells = result.getColumnCells(toBytes(family), toBytes(qualifier[3])); + Assert.assertEquals(2, col3Cells.size()); + Assert.assertEquals(updateValue[3], Bytes.toString(CellUtil.cloneValue(col3Cells.get(0)))); + Assert.assertTrue(timestamp < col3Cells.get(0).getTimestamp()); + Assert.assertEquals(value[3], Bytes.toString(CellUtil.cloneValue(col3Cells.get(1)))); + + } + } + + hTable.close(); + } + + public static void testBatchGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String keyPrefix = "putKey_"; + String family = getColumnFamilyName(tableName); + String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; + long curTs = System.currentTimeMillis(); + long[] ts = {curTs, curTs - 100}; + String valuePrefix = "value_"; + int batchSize = 10; + // prepare data + List puts = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + Put put = new Put(toBytes(keyPrefix + i)); + for (int j = 0; j < columns.length; j++) { + put.add(toBytes(family), toBytes(columns[j]), ts[0], toBytes(valuePrefix + "1_" +i)); + put.add(toBytes(family), toBytes(columns[j]), ts[1], toBytes(valuePrefix + "2_" +i)); + } + puts.add(put); + } + hTable.put(puts); + // start to batch get test + List gets = new ArrayList<>(); + // 1. get specify column + { + int index = 0; + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.addColumn(family.getBytes(), columns[index].getBytes()); + get.setMaxVersions(); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(2, r.size()); + List cells = r.listCells(); + for (int j = 0; j < r.size(); j++) { + Cell cell = cells.get(j); + Assert.assertEquals(keyPrefix+i, Bytes.toString(cell.getRow())); + Assert.assertEquals(columns[index], Bytes.toString(cell.getQualifier())); + Assert.assertEquals(ts[j], cell.getTimestamp()); + if (j == 0) { + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cell.getValue())); + } else { + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cell.getValue())); + } + } + } + } + gets.clear(); + // 2. get do not specify column + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.setMaxVersions(); + get.addFamily(family.getBytes()); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(6, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(toBytes(family), toBytes(columns[j])); + Assert.assertEquals(2, cells.size()); + for (int k = 0; k < cells.size(); k++) { + Assert.assertEquals(ts[k], cells.get(k).getTimestamp()); + if (k == 0) { + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cells.get(k).getValue())); + } else { + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(k).getValue())); + } + } + } + } + } + // 3. get specify version + gets.clear(); + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(6, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); + Assert.assertEquals(2, cells.size()); + for (int k = 0; k < cells.size(); k++) { + Assert.assertEquals(ts[k], cells.get(k).getTimestamp()); + if (k == 0) { + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cells.get(k).getValue())); + } else { + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(k).getValue())); + } + } + } + } + } + // 4. get specify time range + gets.clear(); + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix + i)); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + get.setTimeStamp(ts[1]); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(3, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); + Assert.assertEquals(1, cells.size()); + Assert.assertEquals(ts[1], cells.get(0).getTimestamp()); + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(0).getValue())); + } + } + } + // 5. get specify filter + gets.clear(); + { + for (int i = 0; i < batchSize; i++) { + Get get = new Get(toBytes(keyPrefix+i)); + get.addFamily(family.getBytes()); + get.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(valuePrefix+"2_"+i))); + get.setFilter(valueFilter); + gets.add(get); + } + Result[] batchResults = hTable.get(gets); + Assert.assertEquals(batchSize, batchResults.length); + for (int i = 0; i < batchSize; i++) { + Result r = batchResults[i]; + Assert.assertEquals(3, r.size()); + for (int j = 0; j < columns.length; j++) { + List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); + Assert.assertEquals(1, cells.size()); + Assert.assertEquals(ts[1], cells.get(0).getTimestamp()); + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(0).getValue())); + } + } + } + hTable.close(); + } + + + public static void testMultiCFBatchGetImpl(Map.Entry> entry) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); + hTable.init(); + // prepare data + } + + @Test + public void testBatchPut() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testBatchPutImpl); + truncateTables(ObHTableTestUtil.getConnection(), tableNames); + } + + @Test + public void testMultiCFPut() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchPutImpl); + truncateTables(ObHTableTestUtil.getConnection(), group2tableNames); + } + + @Test + public void testBatchGet() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testBatchGetImpl); + truncateTables(ObHTableTestUtil.getConnection(), tableNames); + } + + @Test + public void testMultiCFGet() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchGetImpl); + truncateTables(ObHTableTestUtil.getConnection(), group2tableNames); + } + +} From 535fe6587bed7d725dc67bde425824323799170a Mon Sep 17 00:00:00 2001 From: GroundWu <1175416256@qq.com> Date: Tue, 18 Mar 2025 19:20:19 +0800 Subject: [PATCH 22/40] code format (cherry picked from commit ed6507512581de15a19ff15077780a1d96b253e0) --- .../hbase/secondary/OHTableSecondaryPartBatchTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java index 0c4a1351..ec431320 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java @@ -54,7 +54,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); + dropTables(tableNames, group2tableNames); } @Before From 0cad6959f1e6f2f6935e8dbbc1224e053a2a4529 Mon Sep 17 00:00:00 2001 From: shen Date: Tue, 18 Mar 2025 19:36:18 +0800 Subject: [PATCH 23/40] [Test] add test cases for hbase scan and ttl in secondary partition scenario (cherry picked from commit 7d64a151706ace3818f5cc83352fc70c4769efce) --- .../OHTableSecondaryPartScanTest.java | 471 ++++++++++++++++-- .../OHTableSecondaryPartTTLTest.java | 200 ++++++++ .../hbase/util/ObHTableSecondaryPartUtil.java | 109 ++++ 3 files changed, 742 insertions(+), 38 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java index 337f8702..90045ec0 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java @@ -20,21 +20,14 @@ import com.alipay.oceanbase.hbase.OHTableClient; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.util.TableTemplateManager; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.BinaryComparator; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.ValueFilter; +import org.junit.*; + +import java.util.*; import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; @@ -49,7 +42,9 @@ public class OHTableSecondaryPartScanTest { public static void before() throws Exception { openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { - createTables(type, tableNames, group2tableNames, true); + if (!type.name().contains("TIME")) { + createTables(type, tableNames, group2tableNames, true); + } } } @@ -68,28 +63,428 @@ public static void testScanImpl(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); + // 0. prepare data + // putKey1 putColumn1 putValue1,ts + // putKey1 putColumn1 putValue2,ts+1 + // putKey1 putColumn2 putValue1,ts + // putKey1 putColumn2 putValue2,ts+1 + // ... String family = getColumnFamilyName(tableName); - String key = "putKey"; - String column1 = "putColumn1"; - String column2 = "putColumn2"; - long timestamp = System.currentTimeMillis(); - Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); - hTable.put(put); - - Scan scan = new Scan(key.getBytes()); - scan.addFamily(family.getBytes()); - ResultScanner scanner = hTable.getScanner(scan); - int count = 0; - for (Result result : scanner) { - for (KeyValue keyValue : result.raw()) { - assertEquals(column1, Bytes.toString(keyValue.getQualifier())); - assertEquals("1", Bytes.toString(keyValue.getValue())); - count++; - } - } - assertEquals(1, count); + long ts = System.currentTimeMillis(); + + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2"}; + long tss[] = {ts, ts + 1}; + long lastTs = tss[1]; + String latestValue = values[1]; + + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); + hTable.put(put); + } + } + } + + // 1. scan specify column + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.addColumn(family.getBytes(), columns[0].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + int count = 0; + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + AssertKeyValue(keys[count], columns[0], lastTs, latestValue, cell); + count++; + } + } + assertEquals(2, count); + } + + // 2. scan do not specify column + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String column : columns) { + AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + assertEquals(columns.length * 2, cells.size()); + } + + // 3. scan specify versions + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + scan.addColumn(family.getBytes(), columns[0].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + assertEquals(tss.length * 2, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (int k = tss.length - 1; k >= 0; k--) { + AssertKeyValue(keys[i], columns[0], tss[k], values[k], cells.get(cellIndex)); + cellIndex++; + } + } + } + + // 4. scan specify time range + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + scan.setTimeStamp(tss[1]); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(columns.length * 2, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String column : columns) { + AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + } + + // 5. scan specify filter + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + scan.addFamily(family.getBytes()); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(values[0]))); + scan.setFilter(valueFilter); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + Assert.assertEquals(columns.length * 2, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String column : columns) { + AssertKeyValue(keys[i], column, values[0], cells.get(cellIndex)); + cellIndex++; + } + } + } + + // 6. scan using setStartRow/setEndRow + { + Scan scan = new Scan(); + scan.setStartRow(keys[0].getBytes()); + scan.setStopRow(keys[2].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String column : columns) { + AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + assertEquals(columns.length * 2, cells.size()); + } + + // 7. scan using batch + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.addFamily(family.getBytes()); + scan.setBatch(2); + ResultScanner scanner = hTable.getScanner(scan); + Result result = scanner.next(); + Assert.assertEquals(2, result.size()); + result = scanner.next(); + Assert.assertEquals(2, result.size()); + result = scanner.next(); + Assert.assertEquals(null, result); + } + + // 7. scan using setAllowPartialResults/setAllowPartialResults + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.addFamily(family.getBytes()); + scan.setMaxResultSize(10); + scan.setAllowPartialResults(true); + ResultScanner scanner = hTable.getScanner(scan); + for (int i = 0; i < 4; i++) { + Result result = scanner.next(); + Assert.assertEquals(1, result.size()); + } + Result result = scanner.next(); + Assert.assertEquals(null, result); + } + + // 8. scan in reverse + { +// Scan scan = new Scan(keys[2].getBytes(), keys[0].getBytes()); +// scan.addFamily(family.getBytes()); +// scan.setReversed(true); +// ResultScanner scanner = hTable.getScanner(scan); +// List cells = getCellsFromScanner(scanner); +// +// int cellIndex = 0; +// for (int i = 1; i >= 0; i--) { +// for (String column : columns) { +// AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); +// cellIndex++; +// } +// } +// assertEquals(columns.length * 2, cells.size()); + } + } + + public static void testMultiCFScanImpl(Map.Entry> entry) throws Exception { + // 0. prepare data + // putKey1 putColumn1 putValue1,ts + // putKey1 putColumn1 putValue2,ts+1 + // putKey1 putColumn2 putValue1,ts + // putKey1 putColumn2 putValue2,ts+1 + // ... + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + long ts = System.currentTimeMillis(); + + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2"}; + long tss[] = {ts, ts + 1}; + long lastTs = tss[1]; + String latestValue = values[1]; + List tableNames = entry.getValue(); + + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); + hTable.put(put); + } + } + } + } + + // 1. multi cf scan specify one cf and one column + { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.addColumn(family.getBytes(), columns[0].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + Assert.assertEquals(2, cells.size()); + for (int i = 0; i < 2; i++) { + AssertKeyValue(keys[i], family, columns[0], lastTs, latestValue, cells.get(i)); + } + } + } + + // 2. multi cf scan specify one cf without specify column + { + for (String tableName : tableNames) { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + String family = getColumnFamilyName(tableName); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String column : columns) { + AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + assertEquals(columns.length * 2, cells.size()); + } + } + + // 3. multi cf scan do not specify cf + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + assertEquals(tableNames.size() * 2 * columns.length, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + Cell cell = cells.get(cellIndex); + AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + } + } + + // 4. multi cf scan specify multi cf and multi column + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + scan.addColumn(family.getBytes(), column.getBytes()); + } + } + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + assertEquals(tableNames.size() * 2 * columns.length, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + } + } + + // 5. multi cf scan specify versions + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + assertEquals(tableNames.size() * 2 * columns.length * tss.length, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + for (int j = values.length - 1; j >= 0; j--) { + AssertKeyValue(keys[i], family, column, tss[j], values[j], cells.get(cellIndex)); + cellIndex++; + } + } + } + } + } + + // 6. multi cf scan specify time range + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + scan.setTimeStamp(tss[1]); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(tableNames.size() * columns.length * 2, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + } + } + + // 7. multi cf scan specify filter + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(values[0]))); + scan.setFilter(valueFilter); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + Assert.assertEquals(tableNames.size() * columns.length * 2, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + AssertKeyValue(keys[i], family, column, tss[0], values[0], cells.get(cellIndex)); + cellIndex++; + } + } + } + } + + // 8. multi cf scan using setStartRow/setEndRow + { + Scan scan = new Scan(); + scan.setStartRow(keys[0].getBytes()); + scan.setStopRow(keys[2].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(tableNames.size() * columns.length * 2, cells.size()); + + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + } + } + + // 9. multi cf scan using batch + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + scan.setBatch(2); + ResultScanner scanner = hTable.getScanner(scan); + Result result = null; + for (String tableName : tableNames) { + result = scanner.next(); + Assert.assertEquals(2, result.size()); + result = scanner.next(); + Assert.assertEquals(2, result.size()); + } + result = scanner.next(); + Assert.assertEquals(null, result); + } + + + // 10. multi cf scan with family scan and column-specific scan + { + Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); + for (int i = 0; i < tableNames.size(); i++) { + String family = getColumnFamilyName(tableNames.get(i)); + if (i % 2 == 0 ) { + scan.addFamily(family.getBytes()); + } else { + for (String column : columns) { + scan.addColumn(family.getBytes(), column.getBytes()); + } + } + } + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(tableNames.size() * 2 * columns.length, cells.size()); + int cellIndex = 0; + for (int i = 0; i < 2; i++) { + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String column : columns) { + AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + } + } } @Test @@ -99,6 +494,6 @@ public void testScan() throws Throwable { @Test public void testMultiCFScan() throws Throwable { - // implement later + FOR_EACH(group2tableNames, OHTableSecondaryPartScanTest::testMultiCFScanImpl); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java new file mode 100644 index 00000000..4914c802 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java @@ -0,0 +1,200 @@ +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.*; +import org.junit.*; + +import java.util.*; +import java.util.stream.Collectors; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static java.lang.Thread.sleep; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableSecondaryPartTTLTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + if (!type.name().contains("TIME")) { + createTables(type, tableNames, group2tableNames, true); + alterTableTimeToLive(tableNames, true, 10); + for (List groupTableNames : group2tableNames.values()) { + alterTableTimeToLive(groupTableNames, true, 10); + } + } + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testTTLImpl(List tableNames) throws Exception { + closeTTLExecute(); + // 0. prepare data + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String endKey = "putKey4"; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2", "putValue3", "putValue4"}; + for (String tableName : tableNames) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), values[i].getBytes()); + hTable.put(put); + } + } + } + } + + // 1. sleep util data expired + sleep(12000); + + // 2. hbase scan/get cannot not return expired data + for (String tableName : tableNames) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(0, cells.size()); + + Get get = new Get(keys[0].getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Assert.assertEquals(0, result.rawCells().length); + } + + // 3. using sql to scan expired but not delete yet hbase data + for (String tableName : tableNames) { + Assert.assertEquals(keys.length * columns.length * values.length, + getSQLTableRowCnt(tableName)); + } + + // 4. open ttl knob to delete expired hbase data + openTTLExecute(); + + // 5. check util expired hbase data is deleted by ttl tasks + checkUtilTimeout(()-> { + try { + return getRunningNormalTTLTaskCnt() == 0; + } catch (Exception e) { + throw new RuntimeException(e); + } + }, 50000, 3000); + + for (String tableName : tableNames) { + Assert.assertEquals(0, getSQLTableRowCnt(tableName)); + } + + // 6. close ttl knob + closeTTLExecute(); + } + + public static void testMultiCFTTLImpl(Map> group2tableNames) throws Exception { + closeTTLExecute(); + List allTableNames = group2tableNames.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + // 0. prepare data + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String endKey = "putKey4"; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2", "putValue3", "putValue4"}; + + for (Map.Entry> entry : group2tableNames.entrySet()) { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + List tableNames = entry.getValue(); + + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), values[i].getBytes()); + hTable.put(put); + } + } + } + } + + } + + // 1. sleep util data expired + sleep(12000); + + // 2. hbase multi cf scan cannot not return expired data + for (Map.Entry> entry : group2tableNames.entrySet()) { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(0, cells.size()); + + Get get = new Get(keys[0].getBytes()); + Result result = hTable.get(get); + Assert.assertEquals(0, result.rawCells().length); + } + + // 3. using sql to scan expired but not delete yet hbase data + for (String tableName : allTableNames) { + Assert.assertEquals(keys.length * columns.length * values.length, + getSQLTableRowCnt(tableName)); + } + + // 4. open ttl knob to delete expired hbase data + openTTLExecute(); + + // 5. check util expired hbase data is deleted by ttl tasks + checkUtilTimeout(()-> { + try { + return getRunningNormalTTLTaskCnt() == 0; + } catch (Exception e) { + throw new RuntimeException(e); + } + }, 50000, 3000); + + for (String tableName : allTableNames) { + Assert.assertEquals(0, getSQLTableRowCnt(tableName)); + } + + // 6. close ttl knob + closeTTLExecute(); + } + + + @Test + public void testTTL() throws Throwable { + testTTLImpl(tableNames); + } + + @Test + public void testMultiCFTTL() throws Throwable { + testMultiCFTTLImpl(group2tableNames); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index 34283f17..47ff5215 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -18,14 +18,19 @@ package com.alipay.oceanbase.hbase.util; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLSyntaxErrorException; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class ObHTableSecondaryPartUtil { public static void openDistributedExecute() throws Exception { @@ -206,6 +211,82 @@ public static String getColumnFamilyName(String input) throws Exception { return result; } + public static void alterTableTimeToLive(List tableNames, boolean printSql, long timeToLive) + throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + if (tableNames != null) { + for (String tableName : tableNames) { + String alterTableTTLSQL = "ALTER TABLE " + tableName + + String.format(" kv_attributes ='{\"Hbase\": {\"TimeToLive\": %d}}';", timeToLive); + try { + conn.createStatement().execute(alterTableTTLSQL); + System.out.println("============= alter table ttl: " + tableName + " table_group: " + + getTableName(tableName) + " =============\n" + + (printSql ? alterTableTTLSQL : "") + + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + throw e; + } + } + } + } + + public static void alterTableMaxVersion(List tableNames, boolean printSql, long maxVersion) + throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + if (tableNames != null) { + for (String tableName : tableNames) { + String alterTableTTLSQL = "ALTER TABLE " + tableName + + String.format(" kv_attributes ='{\"Hbase\": {\"MaxVersions\": %d}}';", maxVersion); + try { + conn.createStatement().execute(alterTableTTLSQL); + System.out.println("============= alter table ttl: " + tableName + " table_group: " + + getTableName(tableName) + " =============\n" + + (printSql ? alterTableTTLSQL : "") + + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + throw e; + } + } + } + } + + public static int getSQLTableRowCnt(String tableName) throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + String RowCountSQL = "SELECT COUNT(*) FROM " + tableName + ";"; + ResultSet resultSet = conn.createStatement().executeQuery(RowCountSQL); + int rowCnt = 0; + if (resultSet.next()) { + rowCnt = resultSet.getInt(1); + } + return rowCnt; + } + + public static int getRunningNormalTTLTaskCnt() throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + String RowCountSQL = "SELECT COUNT(*) FROM " + "OCEANBASE.DBA_OB_KV_TTL_TASKS where TASK_TYPE = 'NORMAL'"; + ResultSet resultSet = conn.createStatement().executeQuery(RowCountSQL); + int rowCnt = 0; + if (resultSet.next()) { + rowCnt = resultSet.getInt(1); + } + return rowCnt; + } + + public static void openTTLExecute() throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + String stmt1 = "ALTER SYSTEM set enable_kv_ttl = true;"; + String stmt2 = "ALTER SYSTEM trigger TTL;"; + conn.createStatement().execute(stmt1); + conn.createStatement().execute(stmt2); + } + + public static void closeTTLExecute() throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + String stmt = "ALTER SYSTEM set enable_kv_ttl = false;"; + conn.createStatement().execute(stmt); + } + public static void AssertKeyValue(String key, String qualifier, long timestamp, String value, Cell cell) { Assert.assertEquals(key, Bytes.toString(cell.getRow())); Assert.assertEquals(qualifier, Bytes.toString(cell.getQualifier())); @@ -218,4 +299,32 @@ public static void AssertKeyValue(String key, String qualifier, String value, Ce Assert.assertEquals(qualifier, Bytes.toString(cell.getQualifier())); Assert.assertEquals(value, Bytes.toString(cell.getValue())); } + + public static void AssertKeyValue(String key, String family, String qualifier, long timestamp, String value, Cell cell) { + Assert.assertEquals(key, Bytes.toString(cell.getRow())); + Assert.assertEquals(family, Bytes.toString(cell.getFamily())); + Assert.assertEquals(qualifier, Bytes.toString(cell.getQualifier())); + Assert.assertEquals(timestamp, cell.getTimestamp()); + Assert.assertEquals(value, Bytes.toString(cell.getValue())); + } + + public static List getCellsFromScanner(ResultScanner scanner) { + List cells = new ArrayList(); + for (Result result : scanner) { + for (Cell cell : result.rawCells()) { + cells.add(cell); + } + } + return cells; + } + + + public static void checkUtilTimeout(Supplier function, long timeout, long interval) throws Exception { + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < timeout) { + if (function.get()) { return; } + Thread.sleep(interval); + } + Assert.assertTrue("Timeout while waiting for the function to return expected result", false); + } } From 1698da52b8611b02cfa3d9450caf862a4d041b9c Mon Sep 17 00:00:00 2001 From: stuBirdFly <1065492934@qq.com> Date: Tue, 18 Mar 2025 21:01:41 +0800 Subject: [PATCH 24/40] add cell ttl test (cherry picked from commit 46e9bc2f6932097fb00cae2d77513a34cd2e021d) --- .../OHTableSecondaryPartCellTTLTest.java | 381 ++++++++++++++++++ ...HTableSecondaryPartCheckAndMutateTest.java | 43 +- .../hbase/util/TableTemplateManager.java | 110 ++++- 3 files changed, 493 insertions(+), 41 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java new file mode 100644 index 00000000..b7012c3d --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java @@ -0,0 +1,381 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.*; +import static org.junit.Assert.assertTrue; + +public class OHTableSecondaryPartCellTTLTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TableTemplateManager.CELL_TTL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); +// dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testCellTTL(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String key1 = "key1"; + String column1 = "cf1"; + String column2 = "cf2"; + String column3 = "cf3"; + String family = getColumnFamilyName(tableName); + String value1 = "v1"; + String value2 = "v2"; + String app = "app"; + + Result r; + Put put1 = new Put(key1.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put1.setTTL(5000); + Put put2 = new Put(key1.getBytes()); + put2.addColumn(family.getBytes(), column1.getBytes(), toBytes(22L)); + put2.addColumn(family.getBytes(), column2.getBytes(), toBytes(33L)); + put2.setTTL(10000); + Put put3 = new Put(key1.getBytes()); + put3.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put3.setTTL(-3000); + Put put4 = new Put(key1.getBytes()); + put4.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put4.setTTL(0); + Put errorPut = new Put(key1.getBytes()); + errorPut.addColumn("family1".getBytes(), column1.getBytes(), toBytes(11L)); + errorPut.setTTL(10); + + Get get = new Get(key1.getBytes()); + get.addFamily(family.getBytes()); + get.setMaxVersions(10); + try { + hTable.put(errorPut); + } catch (Exception e) { + assertTrue(e.getCause().toString().contains("Unknown column 'TTL'")); + } + // test put and get + hTable.put(put1); + hTable.put(put2); + hTable.put(put3); + hTable.put(put4); + r = hTable.get(get); + assertEquals(3, r.size()); + Thread.sleep(5000); + r = hTable.get(get); + assertEquals(2, r.size()); + Thread.sleep(5000); + r = hTable.get(get); + assertEquals(0, r.size()); + + // test increment + hTable.put(put1); + hTable.put(put2); + Thread.sleep(1000); + Increment increment = new Increment(key1.getBytes()); + increment.addColumn(family.getBytes(), column1.getBytes(), 1L); + increment.addColumn(family.getBytes(), column2.getBytes(), 2L); + increment.addColumn(family.getBytes(), column3.getBytes(), 5L); + increment.setTTL(-5000); + hTable.increment(increment); + increment.setTTL(5000); + hTable.increment(increment); + get.setMaxVersions(1); + r = hTable.get(get); + + assertEquals(3, r.size()); + assertEquals(23L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(35L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + assertEquals(5L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column3.getBytes()).get(0)))); + + Thread.sleep(10000); + r = hTable.get(get); + assertEquals(0, r.size()); + + increment = new Increment(key1.getBytes()); + increment.addColumn(family.getBytes(), column1.getBytes(), 1L); + increment.addColumn(family.getBytes(), column2.getBytes(), 2L); + increment.setTTL(5000); + hTable.increment(increment); + r = hTable.get(get); + + assertEquals(2, r.size()); + assertEquals(1L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(2L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + Thread.sleep(5000); + r = hTable.get(get); + assertEquals(0, r.size()); + + hTable.put(put1); + hTable.put(put2); + increment.addColumn(family.getBytes(), column1.getBytes(), 4L); + hTable.increment(increment); + r = hTable.get(get); + assertEquals(2, r.size()); + assertEquals(26L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(35L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + // test append + Thread.sleep(10000); + r = hTable.get(get); + assertEquals(0, r.size()); + + put3 = new Put(key1.getBytes()); + put3.addColumn(family.getBytes(), column1.getBytes(), toBytes(value1)); + put3.addColumn(family.getBytes(), column2.getBytes(), toBytes(value2)); + put3.setTTL(10000); + hTable.put(put3); + Append append = new Append(key1.getBytes()); + KeyValue kv = new KeyValue(key1.getBytes(), family.getBytes(), column1.getBytes(), + app.getBytes()); + append.add(kv); + append.setTTL(-3000); + hTable.append(append); + append.setTTL(3000); + hTable.append(append); + + r = hTable.get(get); + assertEquals(2, r.size()); + assertEquals( + value1 + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + + Thread.sleep(3000); + r = hTable.get(get); + assertEquals(2, r.size()); + assertEquals( + value1, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + + Thread.sleep(7000); + r = hTable.get(get); + assertEquals(0, r.size()); + + append.add(family.getBytes(), column1.getBytes(), app.getBytes()); + hTable.append(append); + r = hTable.get(get); + assertEquals(1, r.size()); + assertEquals( + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + + Thread.sleep(3000); + append.add(family.getBytes(), column2.getBytes(), app.getBytes()); + hTable.append(append); + r = hTable.get(get); + assertEquals(2, r.size()); + assertEquals( + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals( + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + // test checkAndMutate + Thread.sleep(3000); + r = hTable.get(get); + assertEquals(0, r.size()); + hTable.put(put1); + RowMutations rowMutations = new RowMutations(key1.getBytes()); + rowMutations.add(put2); + Delete delete = new Delete(key1.getBytes()); + delete.addColumn(family.getBytes(), column1.getBytes()); + rowMutations.add(delete); + boolean succ = hTable.checkAndMutate(key1.getBytes(), family.getBytes(), + column1.getBytes(), CompareFilter.CompareOp.EQUAL, toBytes(11L), rowMutations); + assertTrue(succ); + r = hTable.get(get); + assertEquals(r.size(), 2); + assertEquals(22L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(33L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + Thread.sleep(10000); + r = hTable.get(get); + assertEquals(r.size(), 0); + + hTable.put(put1); + rowMutations = new RowMutations(key1.getBytes()); + put4 = new Put(key1.getBytes()); + put4.addColumn(family.getBytes(), column1.getBytes(), toBytes(22L)); + put4.addColumn(family.getBytes(), column2.getBytes(), toBytes(33L)); + put4.setTTL(10000); + rowMutations.add(put4); + succ = hTable.checkAndMutate(key1.getBytes(), family.getBytes(), column1.getBytes(), + CompareFilter.CompareOp.EQUAL, toBytes(1L), rowMutations); + assertFalse(succ); + succ = hTable.checkAndMutate(key1.getBytes(), family.getBytes(), column1.getBytes(), + CompareFilter.CompareOp.EQUAL, toBytes(11L), rowMutations); + assertTrue(succ); + + r = hTable.get(get); + assertEquals(r.size(), 2); + assertEquals(22L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(33L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + Thread.sleep(5000); + r = hTable.get(get); + assertEquals(2, r.size()); + assertEquals(22L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); + assertEquals(33L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); + + Thread.sleep(5000); + r = hTable.get(get); + assertEquals(r.size(), 0); + put1 = new Put(key1.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + hTable.put(put1); + + increment = new Increment(key1.getBytes()); + increment.addColumn(family.getBytes(), column1.getBytes(), 1L); + hTable.increment(increment); + r = hTable.get(get); + assertEquals(r.size(), 1); + assertEquals(Bytes.toLong(r.raw()[0].getValue()), 12L); + get.setMaxVersions(10); + r = hTable.get(get); + assertEquals(r.size(), 2); + + put1.setTTL(-100); + hTable.put(put1); + Delete delete1 = new Delete(key1.getBytes()); + delete1.addColumn(family.getBytes(), column1.getBytes()); + hTable.delete(delete1); + get.setMaxVersions(10); + r = hTable.get(get); + assertEquals(r.size(), 1); + assertEquals(Bytes.toLong(r.raw()[0].getValue()), 11L); + hTable.close(); + } + + public static void testCellTTLSQL(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String key1 = "key1"; + String column1 = "cf1"; + String column2 = "cf2"; + String column3 = "cf3"; + String family = getColumnFamilyName(tableName); + String value1 = "v1"; + String value2 = "v2"; + String app = "app"; + + Result r; + Put put1 = new Put(key1.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put1.setTTL(5000); + Put put2 = new Put(key1.getBytes()); + put2.addColumn(family.getBytes(), column1.getBytes(), toBytes(22L)); + put2.addColumn(family.getBytes(), column2.getBytes(), toBytes(33L)); + put2.setTTL(10000); + Put put3 = new Put(key1.getBytes()); + put3.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put3.setTTL(-3000); + Put put4 = new Put(key1.getBytes()); + put4.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put4.setTTL(0); + Put errorPut = new Put(key1.getBytes()); + errorPut.addColumn("family1".getBytes(), column1.getBytes(), toBytes(11L)); + errorPut.setTTL(10); + + // test put and get + hTable.put(put1); + hTable.put(put2); + hTable.put(put3); + hTable.put(put4); + + Assert.assertEquals(5, getSQLTableRowCnt(tableName)); + Thread.sleep(10000); + openTTLExecute(); + + checkUtilTimeout(()-> { + try { + return getRunningNormalTTLTaskCnt() == 0; + } catch (Exception e) { + throw new RuntimeException(e); + } + }, 50000, 3000); + + Assert.assertEquals(0, getSQLTableRowCnt(tableName)); + + // 6. close ttl knob + closeTTLExecute(); + hTable.close(); + } + + @Test + public void testCellTTL() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCellTTLTest::testCellTTL); + } + + @Test + public void testCellTTLSQL() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCellTTLTest::testCellTTLSQL); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java index 0f7c64a1..9b329c5d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java @@ -33,6 +33,7 @@ import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES; import static org.junit.Assert.*; @@ -49,7 +50,7 @@ public class OHTableSecondaryPartCheckAndMutateTest { @BeforeClass public static void before() throws Exception { openDistributedExecute(); - for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + for (TableTemplateManager.TableType type : NORMAL_TABLES) { createTables(type, tableNames, group2tableNames, true); } } @@ -78,6 +79,7 @@ public static void testCheckAndMutate(String tableName) throws Throwable { hTable.put(put); // get row back and assert the values Get get = new Get(ROW); + get.addFamily(FAMILY); Result result = hTable.get(get); assertTrue("Column A value should be a", Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("A"))).equals("a")); @@ -129,40 +131,6 @@ public static void testCheckAndMutate(String tableName) throws Throwable { hTable.close(); } - public static void testCheckAndMutateDiffRow(String tableName) throws Throwable { - OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); - hTable.init(); - - byte[] family = getColumnFamilyName(tableName).getBytes(); - byte[] row1 = Bytes.toBytes("row1"); - byte[] qualifier = Bytes.toBytes("q1"); - byte[] value1 = Bytes.toBytes("value1"); - - Put put = new Put(row1); - put.addColumn(family, qualifier, value1); - hTable.put(put); - - Result result = hTable.get(new Get(row1)); - assertArrayEquals("the value of column q in row1 should be value1", - value1, result.getValue(family, qualifier)); - - byte[] row2 = Bytes.toBytes("row2"); - byte[] value2 = Bytes.toBytes("value2"); - RowMutations mutations = new RowMutations(row2); - put = new Put(row2); - put.addColumn(family, qualifier, value2); - mutations.add(put); - - // check row1 and put row2 - assertTrue(hTable.checkAndMutate(row1, family, qualifier, - CompareFilter.CompareOp.GREATER, value2, mutations)); - - result = hTable.get(new Get(row2)); - assertArrayEquals("the value of column q in row2 should be value2", - value2, result.getValue(family, qualifier)); - hTable.close(); - } - public static void testCheckAndPut(String tableName) throws Throwable { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); @@ -358,11 +326,6 @@ public void testCheckAndPut() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndPut); } - @Test - public void testCheckAndMutateDiffRow() throws Throwable { - FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateDiffRow); - } - @Test public void testCheckAndPutWithCompareOp() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndPutWithCompareOp); diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index 90d0e6cc..9d627fe7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -40,7 +40,15 @@ public enum TableType { SECONDARY_PARTITIONED_KEY_RANGE, // KEY-RANGE分区（使用K） SECONDARY_PARTITIONED_KEY_RANGE_GEN, // KEY-RANGE分区（使用生成列） SECONDARY_PARTITIONED_TIME_RANGE_KEY, // 时序表RANGE-KEY - SECONDARY_PARTITIONED_TIME_KEY_RANGE // 时序表KEY-RANGE + SECONDARY_PARTITIONED_TIME_KEY_RANGE, // 时序表KEY-RANGE + + /* ------------------ CELL TTL ----------------*/ + NON_PARTITIONED_REGULAR_CELL_TTL, + SINGLE_PARTITIONED_REGULAR_CELL_TTL, + SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, // RANGE-KEY分区（使用K） + SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, // RANGE-KEY分区（使用生成列） + SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, // KEY-RANGE分区（使用K） + SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, // KEY-RANGE分区（使用生成列） } public static List NORMAL_TABLES = Arrays.asList(NON_PARTITIONED_REGULAR, @@ -49,6 +57,13 @@ public enum TableType { SECONDARY_PARTITIONED_RANGE_KEY_GEN, SECONDARY_PARTITIONED_KEY_RANGE, SECONDARY_PARTITIONED_KEY_RANGE_GEN); + + public static List CELL_TTL_TABLES = Arrays.asList(NON_PARTITIONED_REGULAR_CELL_TTL, + SINGLE_PARTITIONED_REGULAR_CELL_TTL, + SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, + SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, + SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, + SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL); private static final Map SQL_TEMPLATES = new EnumMap<>(TableType.class); @@ -187,6 +202,93 @@ public enum TableType { " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE)"); + + /* ------------------ CELL TTL ----------------*/ + SQL_TEMPLATES.put(TableType.NON_PARTITIONED_REGULAR_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s"); + + SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); + + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE)"); + + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, + "CREATE TABLE IF NOT EXISTS `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); } public static String getCreateTableSQL(TableType type, String tableName, @@ -198,16 +300,22 @@ public static String getCreateTableSQL(TableType type, String tableName, switch (type) { case NON_PARTITIONED_REGULAR: case NON_PARTITIONED_TIME_SERIES: + case NON_PARTITIONED_REGULAR_CELL_TTL: params = new Object[]{tableName, tableGroup}; break; case SINGLE_PARTITIONED_REGULAR: case SINGLE_PARTITIONED_TIME_SERIES: // 合并相同处理逻辑 + case SINGLE_PARTITIONED_REGULAR_CELL_TTL: params = new Object[]{tableName, tableGroup, PART_NUM}; break; case SECONDARY_PARTITIONED_RANGE_KEY: case SECONDARY_PARTITIONED_RANGE_KEY_GEN: case SECONDARY_PARTITIONED_KEY_RANGE: case SECONDARY_PARTITIONED_KEY_RANGE_GEN: + case SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL: + case SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL: + case SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL: + case SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL: boolean isGen = type.name().contains("GEN"); params = new Object[]{ tableName, From 059ff3725ed4214cbf126b0197c8098d578707e1 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 19 Mar 2025 20:29:20 +0800 Subject: [PATCH 25/40] fix unstable case (cherry picked from commit f2b197ec64876bd135d17973e79cadf4ceddb5bb) --- .../secondary/OHTableSecondaryPartDeleteTest.java | 4 ++-- .../secondary/OHTableSecondaryPartPutTest.java | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java index af937fc9..81f41c0b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java @@ -42,7 +42,7 @@ public class OHTableSecondaryPartDeleteTest { @BeforeClass public static void before() throws Exception { openDistributedExecute(); - for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { createTables(type, tableNames, group2tableNames, true); } } @@ -163,7 +163,7 @@ public static void testDeleteColumnImpl(String tableName) throws Exception { get.addColumn(toBytes(family), toBytes(column)); get.setMaxVersions(); Result result = hTable.get(get); - Assert(tableName, ()->Assert.assertEquals(2, result.size())); // ts1, ts2, ts3 is deleted + Assert(tableName, ()->Assert.assertEquals(1, result.size())); // ts1, ts2, ts3 is deleted Assert.assertEquals(Arrays.toString((value + ts4).getBytes()), Arrays.toString(result.getValue(family.getBytes(), column.getBytes()))); for (Cell cell : result.rawCells()) { Assert.assertTrue(cell.getTimestamp() != ts1); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java index 89c3bd7d..94a81316 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -42,10 +42,8 @@ public class OHTableSecondaryPartPutTest { @BeforeClass public static void before() throws Exception { openDistributedExecute(); - for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { - if (!type.name().contains("TIME")) { - createTables(type, tableNames, group2tableNames, true); - } + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); } } @@ -84,11 +82,10 @@ public static void testPutImpl(String tableName) throws Exception { Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare((column1 + value).getBytes(), r.getValue(family.getBytes(), column1.getBytes())))); Assert(tableName, ()->Assert.assertTrue(ObHTableTestUtil.secureCompare((column1 + value).getBytes(), r.getValue(family.getBytes(), column1.getBytes())))); } - { // put exist key and get long timestamp = System.currentTimeMillis(); Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), toBytes(column1 + value + timestamp)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); hTable.put(put); Get get = new Get(toBytes(key)); @@ -112,7 +109,7 @@ public static void testPutImpl(String tableName) throws Exception { Assert(tableName, ()->Assert.assertEquals(timestamp, r.rawCells()[0].getTimestamp())); Put put1 = new Put(toBytes(key)); - put1.add(family.getBytes(), column1.getBytes(), toBytes(column1 + value)); + put1.add(family.getBytes(), column1.getBytes(), timestamp + 100, toBytes(column1 + value)); hTable.put(put1); Result r2 = hTable.get(get); @@ -236,6 +233,7 @@ public static void testMultiCFPutImpl(Map.Entry> entry) thr } for (String tableName : entry.getValue()) { String family = getColumnFamilyName(tableName); + // TODO: Get/Scan返回的结果Q 带了cf, 这里预期跑不过 Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column1 + value).getBytes(), r.getValue(family.getBytes(), column1.getBytes())))); Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column2 + value).getBytes(), r.getValue(family.getBytes(), column2.getBytes())))); } @@ -273,6 +271,7 @@ public static void testMltiCFPutBatchImpl(Map.Entry> entry) } for (String tableName : entry.getValue()) { String family = getColumnFamilyName(tableName); + // TODO: Get/Scan返回的结果Q 带了cf, 这里预期跑不过 Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column1 + value + timestamp).getBytes(), result.getValue(family.getBytes(), column1.getBytes())))); Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column2 + value + timestamp).getBytes(), result.getValue(family.getBytes(), column2.getBytes())))); } From 418551da28181b6ff0afd9b8aef8b4dec4c1500d Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 19 Mar 2025 21:11:04 +0800 Subject: [PATCH 26/40] fix del case (cherry picked from commit 15382be0d4f3a6d23aa62a2048f4e2cc82786f5b) --- .../hbase/secondary/OHTableSecondaryPartDeleteTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java index 81f41c0b..c62e8bfe 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java @@ -88,7 +88,7 @@ public static void testDeleteLastVersionImpl(String tableName) throws Exception get.setMaxVersions(); Result result = hTable.get(get); Assert.assertEquals(3, result.size()); // ts4 is deleted - Assert.assertEquals(value, result.getValue(family.getBytes(), column.getBytes())); + Assert(tableName, () -> Assert.assertTrue(secureCompare(value.getBytes(), result.getValue(family.getBytes(), column.getBytes())))); for (Cell cell : result.rawCells()) { Assert.assertTrue(cell.getTimestamp() != ts4); } @@ -440,8 +440,7 @@ public static void testMultiCFDeleteFamilyVersionImpl(Map.EntryAssert.assertEquals(0, result.size())); + Assert(entry.getValue(), ()->Assert.assertEquals(6, result.size())); } public static void testDeleteAllImpl(Map.Entry> entry) throws Exception { From 7eaf7fbc44f86c3493ead51f96c3c98fc31df08b Mon Sep 17 00:00:00 2001 From: Shen Yunlong <44497386+shenyunlong@users.noreply.github.com> Date: Thu, 20 Mar 2025 11:34:17 +0800 Subject: [PATCH 27/40] [Test] add hbase rowkey TTL test cases for seconary partition scenario (#175) (cherry picked from commit 41a745d3fd6fbdb7cba794e2479423767003a725) --- .../OHTableSecondaryPartGetTest.java | 27 +-- .../OHTableSecondaryPartTTLTest.java | 183 +++++++++++++++++- .../hbase/util/ObHTableSecondaryPartUtil.java | 16 +- 3 files changed, 201 insertions(+), 25 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java index 11616adf..6f1365d1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -216,8 +216,8 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr Result r = hTable.get(get); Assert.assertEquals(tableNames.size() * columns.length, r.raw().length); int cur = 0; - for (int i = 0; i < columns.length; i++) { - for (String tableName : tableNames) { + for (String tableName : tableNames) { + for (int i = 0; i < columns.length; i++) { AssertKeyValue(key, columns[i], lastTs, latestValue, r.raw()[cur]); cur++; } @@ -246,10 +246,11 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr Result r = hTable.get(get); Assert.assertEquals(tableNames.size() * columns.length * ts.length, r.raw().length); int cur = 0; - for (int i = 0; i < columns.length; i++) { - for (String tableName : tableNames) { - for (int k = ts.length-1; k >= 0; k--) { - AssertKeyValue(key, columns[i], ts[k], values[k], r.raw()[cur]); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < columns.length; i++) { + for (int k = ts.length - 1; k >= 0; k--) { + AssertKeyValue(key, family, columns[i], ts[k], values[k], r.raw()[cur]); cur++; } } @@ -264,9 +265,10 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr Result r = hTable.get(get); Assert.assertEquals(tableNames.size() * columns.length, r.raw().length); int cur = 0; - for (int i = 0; i < columns.length; i++) { - for (String tableName : tableNames) { - AssertKeyValue(key, columns[i], ts[1], values[1], r.raw()[cur]); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < columns.length; i++) { + AssertKeyValue(key, family, columns[i], ts[1], values[1], r.raw()[cur]); cur++; } } @@ -282,9 +284,10 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr Result r = hTable.get(get); Assert.assertEquals(tableNames.size() * columns.length, r.raw().length); int cur = 0; - for (int i = 0; i < columns.length; i++) { - for (String tableName : tableNames) { - AssertKeyValue(key, columns[i], ts[0], values[0], r.raw()[cur]); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < columns.length; i++) { + AssertKeyValue(key, family, columns[i], ts[0], values[0], r.raw()[cur]); cur++; } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java index 4914c802..c60e63d4 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java @@ -46,7 +46,7 @@ public void prepareCase() throws Exception { } public static void testTTLImpl(List tableNames) throws Exception { - closeTTLExecute(); + disableTTL(); // 0. prepare data String keys[] = {"putKey1", "putKey2", "putKey3"}; String endKey = "putKey4"; @@ -94,7 +94,8 @@ public static void testTTLImpl(List tableNames) throws Exception { } // 4. open ttl knob to delete expired hbase data - openTTLExecute(); + enableTTL(); + triggerTTL(); // 5. check util expired hbase data is deleted by ttl tasks checkUtilTimeout(()-> { @@ -110,11 +111,11 @@ public static void testTTLImpl(List tableNames) throws Exception { } // 6. close ttl knob - closeTTLExecute(); + disableTTL(); } public static void testMultiCFTTLImpl(Map> group2tableNames) throws Exception { - closeTTLExecute(); + disableTTL(); List allTableNames = group2tableNames.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); // 0. prepare data String keys[] = {"putKey1", "putKey2", "putKey3"}; @@ -127,7 +128,6 @@ public static void testMultiCFTTLImpl(Map> group2tableNames OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); hTable.init(); List tableNames = entry.getValue(); - for (String tableName : tableNames) { String family = getColumnFamilyName(tableName); for (String key : keys) { @@ -168,7 +168,8 @@ public static void testMultiCFTTLImpl(Map> group2tableNames } // 4. open ttl knob to delete expired hbase data - openTTLExecute(); + enableTTL(); + triggerTTL(); // 5. check util expired hbase data is deleted by ttl tasks checkUtilTimeout(()-> { @@ -184,7 +185,159 @@ public static void testMultiCFTTLImpl(Map> group2tableNames } // 6. close ttl knob - closeTTLExecute(); + disableTTL(); + } + + void testRowkeyTTL(List tableNames, Boolean useScan, Boolean isReversed) throws Exception { + disableTTL(); + // 0. prepare data + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String endKey = "putKey4"; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2", "putValue3", "putValue4"}; + for (String tableName : tableNames) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), values[i].getBytes()); + hTable.put(put); + } + } + } + } + + // 1. sleep util data expired + sleep(12000); + + // 2. enable kv ttl + enableTTL(); + + // 3. SQL can scan expired but not delete yet hbase data + for (String tableName : tableNames) { + Assert.assertEquals(keys.length * columns.length * values.length, + getSQLTableRowCnt(tableName)); + } + + // 4. use Hbase scan/get expired data to trigger ttl + for (String tableName : tableNames) { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + if (useScan) { + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.addFamily(family.getBytes()); + scan.setReversed(isReversed); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(0, cells.size()); + } else { + for (String key : keys) { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + assertEquals(0, result.rawCells().length); + } + } + } + + // 5. wait to disable + checkUtilTimeout(()-> { + try { + Boolean passed = true; + for (int i = 0; passed && i < tableNames.size(); i++) { + if (getSQLTableRowCnt(tableNames.get(i)) > 0) { + passed = false; + } + } + return passed; + } catch (Exception e) { + throw new RuntimeException(e); + } + }, 50000, 3000); + + // 6. disable ttl + disableTTL(); + } + + void testMultiCFRowkeyTTL(Map> group2tableNames, Boolean useScan, Boolean isReversed) throws Exception { + disableTTL(); + List allTableNames = group2tableNames.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + // 0. prepare data + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String endKey = "putKey4"; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2", "putValue3", "putValue4"}; + for (Map.Entry> entry : group2tableNames.entrySet()) { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + for (String tableName : entry.getValue()) { + String family = getColumnFamilyName(tableName); + for (String key : keys) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), values[i].getBytes()); + hTable.put(put); + } + } + } + } + } + + // 1. sleep util data expired + sleep(12000); + + // 2. enable kv ttl + enableTTL(); + + // 3. SQL can scan expired but not delete yet hbase data + for (String tableName : allTableNames) { + Assert.assertEquals(keys.length * columns.length * values.length, + getSQLTableRowCnt(tableName)); + } + + // 4. use Hbase scan expired data to trigger ttl + for (Map.Entry> entry : group2tableNames.entrySet()) { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + if (useScan) { + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.setReversed(isReversed); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + assertEquals(0, cells.size()); + } else { + for (String key : keys) { + Get get = new Get(key.getBytes()); + Result result = hTable.get(get); + assertEquals(0, result.rawCells().length); + } + } + } + + // 5. wait to disable + checkUtilTimeout(()-> { + try { + Boolean passed = true; + for (int i = 0; passed && i < allTableNames.size(); i++) { + if (getSQLTableRowCnt(allTableNames.get(i)) > 0) { + passed = false; + } + } + return passed; + } catch (Exception e) { + throw new RuntimeException(e); + } + }, 70000, 5000); + + // 6. disable ttl + disableTTL(); } @@ -197,4 +350,20 @@ public void testTTL() throws Throwable { public void testMultiCFTTL() throws Throwable { testMultiCFTTLImpl(group2tableNames); } + + @Test + public void testRowkeyTTL() throws Exception { + testRowkeyTTL(tableNames, true, false); + testRowkeyTTL(tableNames, false, false); + // TODO: open the test after reverse scan is ok + // testRowkeyTTL(tableNames, true); + } + + @Test + public void testMultiCFRowkeyTTL() throws Exception { + testMultiCFRowkeyTTL(group2tableNames, true, false); + testMultiCFRowkeyTTL(group2tableNames, false, false); + // TODO: open the test after reverse scan is ok + // testMultiCFRowkeyTTL(group2tableNames, true, true); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index 47ff5215..934170e6 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -273,15 +273,19 @@ public static int getRunningNormalTTLTaskCnt() throws Exception { return rowCnt; } - public static void openTTLExecute() throws Exception { + public static void enableTTL() throws Exception { Connection conn = ObHTableTestUtil.getConnection(); - String stmt1 = "ALTER SYSTEM set enable_kv_ttl = true;"; - String stmt2 = "ALTER SYSTEM trigger TTL;"; - conn.createStatement().execute(stmt1); - conn.createStatement().execute(stmt2); + String stmt = "ALTER SYSTEM set enable_kv_ttl = true;"; + conn.createStatement().execute(stmt); + } + + public static void triggerTTL() throws Exception { + Connection conn = ObHTableTestUtil.getConnection(); + String stmt = "ALTER SYSTEM trigger TTL;"; + conn.createStatement().execute(stmt); } - public static void closeTTLExecute() throws Exception { + public static void disableTTL() throws Exception { Connection conn = ObHTableTestUtil.getConnection(); String stmt = "ALTER SYSTEM set enable_kv_ttl = false;"; conn.createStatement().execute(stmt); From 58f85ccb908f87f534720510451cf5337e48182d Mon Sep 17 00:00:00 2001 From: maochongxin Date: Sat, 22 Mar 2025 19:54:35 +0800 Subject: [PATCH 28/40] fix del 4007 (cherry picked from commit a9002e716d75b9fa2e52407d07273c8bdefad783) --- .../com/alipay/oceanbase/hbase/OHTable.java | 57 ++++++++++++------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 252a6337..6215bb0b 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -2202,28 +2202,35 @@ private BatchOperation buildBatchOperation(String tableName, List } } } else if (row instanceof Delete) { + boolean disExec = obTableClient.getServerCapacity().isSupportDistributedExecute(); Delete delete = (Delete) row; if (delete.isEmpty()) { singleOpResultNum++; - KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), - KeyValue.Type.Maximum); - com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, DEL, isTableGroup, Long.MAX_VALUE); - ObNewRange range = new ObNewRange(); - ObTableQuery tableQuery = new ObTableQuery(); - ObHTableFilter filter = null; - tableQuery.setObKVParams(buildOBKVParams((Scan)null)); - range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); - range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); - if (kv.getTimestamp() == Long.MAX_VALUE) { - filter = buildObHTableFilter(null, null, Integer.MAX_VALUE); + if (disExec) { + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), + KeyValue.Type.Maximum); + com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, DEL, isTableGroup, Long.MAX_VALUE); + ObNewRange range = new ObNewRange(); + ObTableQuery tableQuery = new ObTableQuery(); + ObHTableFilter filter; + tableQuery.setObKVParams(buildOBKVParams((Scan) null)); + range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + if (kv.getTimestamp() == Long.MAX_VALUE) { + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE); + } else { + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE); + } + tableQuery.sethTableFilter(filter); + tableQuery.addKeyRange(range); + + tableMutation.setTable(tableName); + batch.addOperation(new QueryAndMutate(tableQuery, tableMutation)); } else { - filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE); + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), + KeyValue.Type.Maximum); + batch.addOperation(buildMutation(kv, DEL, isTableGroup, Long.MAX_VALUE)); } - tableQuery.sethTableFilter(filter); - tableQuery.addKeyRange(range); - - tableMutation.setTable(tableName); - batch.addOperation(new QueryAndMutate(tableQuery, tableMutation)); } else { for (Map.Entry> entry : delete.getFamilyCellMap().entrySet()) { byte[] family = entry.getKey(); @@ -2232,11 +2239,19 @@ private BatchOperation buildBatchOperation(String tableName, List singleOpResultNum++; if (isTableGroup) { KeyValue new_kv = modifyQualifier(kv, - (Bytes.toString(family) + "." + Bytes.toString(kv - .getQualifier())).getBytes()); - batch.addOperation(buildDeleteQueryAndMutate(new_kv, DEL, true, Long.MAX_VALUE)); + (Bytes.toString(family) + "." + Bytes.toString(kv + .getQualifier())).getBytes()); + if (disExec) { + batch.addOperation(buildDeleteQueryAndMutate(new_kv, DEL, true, Long.MAX_VALUE)); + } else { + batch.addOperation(buildMutation(new_kv, DEL, true, Long.MAX_VALUE)); + } } else { - batch.addOperation(buildDeleteQueryAndMutate(kv, DEL, false, Long.MAX_VALUE)); + if (disExec) { + batch.addOperation(buildDeleteQueryAndMutate(kv, DEL, false, Long.MAX_VALUE)); + } else { + batch.addOperation(buildMutation(kv, DEL, false, Long.MAX_VALUE)); + } } } } From a5d33eb30cd1d85e1ff0a24c36c32c175d80f561 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Tue, 25 Mar 2025 16:53:09 +0800 Subject: [PATCH 29/40] fix case stability (cherry picked from commit 8a4cf378cbeec741b251a6bc5a9eecb1f688f2d3) --- .../hbase/util/ObTableClientManager.java | 40 +-- .../OHTableSecondaryPartAppendTest.java | 281 +++++++++++------- .../OHTableSecondaryPartBatchGetTest.java | 8 +- .../OHTableSecondaryPartBatchTest.java | 4 - .../OHTableSecondaryPartCellTTLTest.java | 215 ++++++++++---- ...HTableSecondaryPartCheckAndMutateTest.java | 257 +++++++++++----- .../OHTableSecondaryPartDeleteTest.java | 11 + .../OHTableSecondaryPartGetTest.java | 38 ++- .../OHTableSecondaryPartIncrementTest.java | 211 ++++++++----- .../OHTableSecondaryPartPutTest.java | 10 +- .../OHTableSecondaryPartScanTest.java | 85 +++--- .../OHTableSecondaryPartTTLTest.java | 65 ++-- .../hbase/util/ObHTableSecondaryPartUtil.java | 69 +++-- 13 files changed, 845 insertions(+), 449 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java index 5db940aa..6f95aefe 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java @@ -22,6 +22,7 @@ import com.google.common.base.Objects; import org.apache.hadoop.classification.InterfaceAudience; + import java.io.IOException; import java.util.Map; import java.util.Properties; @@ -36,7 +37,7 @@ public class ObTableClientManager { public static final ConcurrentHashMap OB_TABLE_CLIENT_LOCK = new ConcurrentHashMap(); - public static final Map OB_TABLE_CLIENT_INSTANCE = new ConcurrentHashMap(); + public static final Map OB_TABLE_CLIENT_INSTANCE = new ConcurrentHashMap<>(); public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration connectionConfig) throws IllegalArgumentException, @@ -56,22 +57,27 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c obTableClientKey.setDatabase(connectionConfig.getDatabase()); } else { checkArgument(isNotBlank(connectionConfig.getParamUrl()), HBASE_OCEANBASE_PARAM_URL - + " is blank"); - obTableClientKey = new ObTableClientKey(); - String paramUrl = connectionConfig.getParamUrl(); - if (!paramUrl.contains("database")) { - paramUrl += "&database=default"; - } - obTableClientKey.setParamUrl(paramUrl); - obTableClientKey.setSysUserName(connectionConfig.getSysUsername()); - if (connectionConfig.getSysPassword() == null) { - obTableClientKey.setSysPassword(Constants.EMPTY_STRING); - } else { - obTableClientKey.setSysPassword(connectionConfig.getSysPassword()); - } + + " is blank"); + obTableClientKey = generateObTableClientKey(connectionConfig); + } + return getOrCreateObTableClient(obTableClientKey, connectionConfig.getRpcConnectTimeout()); + } + + public static ObTableClientKey generateObTableClientKey(OHConnectionConfiguration connectionConfig) { + ObTableClientKey obTableClientKey = new ObTableClientKey(); + String paramUrl = connectionConfig.getParamUrl(); + if (!paramUrl.contains("database")) { + paramUrl += "&database=default"; + } + obTableClientKey.setParamUrl(paramUrl); + obTableClientKey.setSysUserName(connectionConfig.getSysUsername()); + if (connectionConfig.getSysPassword() == null) { + obTableClientKey.setSysPassword(Constants.EMPTY_STRING); + } else { + obTableClientKey.setSysPassword(connectionConfig.getSysPassword()); } checkArgument(isNotBlank(connectionConfig.getFullUsername()), - HBASE_OCEANBASE_FULL_USER_NAME + " is blank"); + HBASE_OCEANBASE_FULL_USER_NAME + " is blank"); obTableClientKey.setFullUserName(connectionConfig.getFullUsername()); if (connectionConfig.getPassword() == null) { @@ -83,8 +89,8 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c for (Map.Entry property : connectionConfig.getProperties().entrySet()) { obTableClientKey.getProperties().put(property.getKey(), property.getValue()); } - - return getOrCreateObTableClient(obTableClientKey, connectionConfig.getRpcConnectTimeout()); + + return obTableClientKey; } public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableClientKey, diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java index 4156291e..122b4880 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java @@ -21,12 +21,10 @@ import com.alipay.oceanbase.hbase.OHTableClient; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import com.google.common.base.Strings; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.AfterClass; -import org.junit.Test; +import org.junit.*; import java.io.IOException; import java.util.*; @@ -37,13 +35,14 @@ import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.getConnection; import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; public class OHTableSecondaryPartAppendTest { private static List tableNames = new LinkedList(); - private static Map> group2tableNames = null; + private static List series_tables = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); @BeforeClass public static void before() throws Exception { @@ -56,7 +55,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - // dropTables(tableNames, group2tableNames); +// dropTables(tableNames, group2tableNames); } @Before @@ -71,131 +70,179 @@ private static void assertNullResult(Result result) throws Exception { private static void testAppend(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); + try { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + byte[] ROW = "appendKey".getBytes(); + byte[] v1 = Bytes.toBytes("42"); + byte[] v2 = Bytes.toBytes("23"); + byte[][] QUALIFIERS = new byte[][]{Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c")}; + Append a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[0], v1); + a.add(FAMILY, QUALIFIERS[1], v2); + a.setReturnResults(false); + assertNullResult(hTable.append(a)); - byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); - byte[] ROW = "appendKey".getBytes(); - byte[] v1 = Bytes.toBytes("42"); - byte[] v2 = Bytes.toBytes("23"); - byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), - Bytes.toBytes("c") }; - Append a = new Append(ROW); - a.add(FAMILY, QUALIFIERS[0], v1); - a.add(FAMILY, QUALIFIERS[1], v2); - a.setReturnResults(false); - assertNullResult(hTable.append(a)); - - a = new Append(ROW); - a.add(FAMILY, QUALIFIERS[0], v2); - a.add(FAMILY, QUALIFIERS[1], v1); - a.add(FAMILY, QUALIFIERS[2], v2); - Result r = hTable.append(a); - assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0]))); - assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1]))); - // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct - assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2]))); - assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(), - r.getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp()); - - Get get = new Get(ROW); - get.setMaxVersions(10); - get.addFamily(FAMILY); - Result result = hTable.get(get); - assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[0]).size()); - assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[1]).size()); - assertEquals(1, result.getColumnCells(FAMILY, QUALIFIERS[2]).size()); - assertEquals( - 0, - Bytes.compareTo(Bytes.add(v1, v2), result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0) - .getValue())); - assertEquals(0, - Bytes.compareTo(v2, result.getColumnCells(FAMILY, QUALIFIERS[2]).get(0).getValue())); + a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[0], v2); + a.add(FAMILY, QUALIFIERS[1], v1); + a.add(FAMILY, QUALIFIERS[2], v2); + Result r = hTable.append(a); + assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0]))); + assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1]))); + // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct + assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2]))); + assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(), + r.getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp()); - hTable.close(); + Get get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); + Result result = hTable.get(get); + assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[0]).size()); + assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[1]).size()); + assertEquals(1, result.getColumnCells(FAMILY, QUALIFIERS[2]).size()); + assertEquals( + 0, + Bytes.compareTo(Bytes.add(v1, v2), result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0) + .getValue())); + assertEquals(0, + Bytes.compareTo(v2, result.getColumnCells(FAMILY, QUALIFIERS[2]).get(0).getValue())); + } finally { + hTable.close(); + } } private static void testAppendBorder(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); - - byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); - byte[] ROW = "appendKey".getBytes(); - byte[] v1 = Bytes.toBytes("ab"); - byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), - Bytes.toBytes("c") }; - Put put = new Put(ROW); - put.addColumn(FAMILY, QUALIFIERS[1], v1); - hTable.put(put); - Append a = new Append(ROW); - a.add(FAMILY, QUALIFIERS[1], v1); - a.add(FAMILY, QUALIFIERS[2], "".getBytes()); - hTable.append(a); - Get get = new Get(ROW); - get.setMaxVersions(10); - get.addFamily(FAMILY); - Result result = hTable.get(get); - assertEquals(3, result.size()); - - a = new Append(ROW); - a.add(FAMILY, QUALIFIERS[2], v1); - a.add(FAMILY, QUALIFIERS[2], "".getBytes()); - hTable.append(a); - get = new Get(ROW); - get.setMaxVersions(10); - get.addFamily(FAMILY); - result = hTable.get(get); - assertEquals(4, result.size()); - - byte[] randomBytes = new byte[1025]; - Random random = new Random(); - random.nextBytes(randomBytes); - a = new Append(ROW); - a.add(FAMILY, QUALIFIERS[2], randomBytes); try { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + byte[] ROW = "appendKey".getBytes(); + byte[] v1 = Bytes.toBytes("ab"); + byte[][] QUALIFIERS = new byte[][]{Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c")}; + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIERS[1], v1); + hTable.put(put); + Append a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[1], v1); + a.add(FAMILY, QUALIFIERS[2], "".getBytes()); hTable.append(a); - } catch (IOException e) { - assertTrue(e.getCause().getMessage().contains("Data too long for column 'V'")); - } + Get get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); + Result result = hTable.get(get); + assertEquals(3, result.size()); - hTable.close(); + a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[2], v1); + a.add(FAMILY, QUALIFIERS[2], "".getBytes()); + hTable.append(a); + get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); + result = hTable.get(get); + assertEquals(4, result.size()); + + byte[] randomBytes = new byte[1025]; + Random random = new Random(); + random.nextBytes(randomBytes); + a = new Append(ROW); + a.add(FAMILY, QUALIFIERS[2], randomBytes); + try { + hTable.append(a); + fail("unexpect error, too long data should fail"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage().contains("Data too long for column 'V'")); + } + } finally { + hTable.close(); + } } private static void testAppendCon(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + try { + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + byte[] v = "a".getBytes(); + ThreadPoolExecutor threadPoolExecutor = OHTable.createDefaultThreadPoolExecutor(1, 100, 100); + AtomicInteger atomicInteger = new AtomicInteger(0); + CountDownLatch countDownLatch = new CountDownLatch(100); + + for (int i = 0; i < 100; i++) { + Append append = new Append(ROW); + append.add(FAMILY, column.getBytes(), v); + threadPoolExecutor.submit(() -> { + try { + hTable.append(append); + } catch (Exception e) { + if (!e.getCause().getMessage().contains("OB_TRY_LOCK_ROW_CONFLICT") + && !e.getCause().getMessage().contains("OB_TIMEOUT")) { + throw new RuntimeException(e); + } + } finally { + atomicInteger.incrementAndGet(); + countDownLatch.countDown(); + } + }); + } + threadPoolExecutor.shutdown(); + countDownLatch.await(100000, TimeUnit.MILLISECONDS); + final byte[] expect = Strings.repeat("a", atomicInteger.get()).getBytes(); + System.out.println("atomicInteger: " + atomicInteger.get()); + Get get = new Get(ROW); + get.setMaxVersions(1); + get.addColumn(FAMILY, column.getBytes()); + Result result = hTable.get(get); + ObHTableTestUtil.Assert(tableName, ()-> assertTrue(0 <= Bytes.compareTo(expect, result.getColumnCells(FAMILY, column.getBytes()).get(0).getValue()))); + } finally { + hTable.close(); + } + } + + private static void testAppendMultiCF(Map.Entry> entry) throws Exception { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); hTable.init(); - byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + List tableNames = entry.getValue(); String column = "appColumn"; byte[] ROW = "appendKey".getBytes(); byte[] v = "a".getBytes(); - byte[] expect = "a".getBytes(); - ThreadPoolExecutor threadPoolExecutor = OHTable.createDefaultThreadPoolExecutor(1, 100,100); - AtomicInteger atomicInteger = new AtomicInteger(0); - CountDownLatch countDownLatch = new CountDownLatch(100); - for (int i = 0; i < 100; i++) { - Append append = new Append(ROW); + Append append = new Append(ROW); + for (String tableName : tableNames) { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); append.add(FAMILY, column.getBytes(), v); - threadPoolExecutor.submit(() -> { - try { - hTable.append(append); - atomicInteger.incrementAndGet(); - } catch (Exception e) { - if (!e.getCause().getMessage().contains("OB_TRY_LOCK_ROW_CONFLICT") && !e.getCause().getMessage().contains("OB_TIMEOUT")) { - throw new RuntimeException(e); - } - } finally { - countDownLatch.countDown(); - } - }); } - countDownLatch.await(100000, TimeUnit.MILLISECONDS); - for (int i = 0; i < atomicInteger.get() - 1; i++) { - expect = Bytes.add(expect, v); + try { + hTable.append(append); + fail("unexpect error, append should not support multi cf"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("multi family is not supported")); + } + hTable.close(); + } + + private static void testAppendSeires(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + byte[] v = "a".getBytes(); + Append append = new Append(ROW); + append.add(FAMILY, column.getBytes(), v); + try { + hTable.append(append); + fail("unexpect error, append should not support series table"); + } catch (Exception e) { + assertTrue(e.getCause().getMessage() + .contains("query and mutate with hbase series type not supported")); } - Get get = new Get(ROW); - get.setMaxVersions(1); - get.addColumn(FAMILY, column.getBytes()); - Result result = hTable.get(get); - assertEquals(0, Bytes.compareTo(expect, result.getColumnCells(FAMILY, column.getBytes()).get(0).getValue())); hTable.close(); } @@ -213,4 +260,16 @@ public void testBorderAppend() throws Throwable { public void testAppendConcurrency() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartAppendTest::testAppendCon); } + + @Test + public void testAppendMultiCF() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartAppendTest::testAppendMultiCF); + } + + @Test + public void testAppendSeires() throws Throwable { + createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, null, true); + FOR_EACH(series_tables, OHTableSecondaryPartAppendTest::testAppendSeires); + dropTables(series_tables, null); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java index 93c4a459..77d573ec 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java @@ -35,16 +35,14 @@ import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; import static org.apache.hadoop.hbase.util.Bytes.toBytes; - public class OHTableSecondaryPartBatchGetTest { private static List tableNames = new LinkedList(); - private static Map> group2tableNames = new LinkedHashMap<>(); - + private static Map> group2tableNames = new LinkedHashMap>(); @BeforeClass public static void before() throws Exception { openDistributedExecute(); - for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { createTables(type, tableNames, group2tableNames, true); } } @@ -60,7 +58,6 @@ public void prepareCase() throws Exception { truncateTables(tableNames, group2tableNames); } - public static void testBatchGetImpl(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); @@ -97,6 +94,7 @@ public static void testBatchGetImpl(String tableName) throws Exception { System.out.println("Column: " + Q + ", Value: " + V); } } + hTable.close(); } @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java index ec431320..bdcccf4a 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java @@ -441,25 +441,21 @@ public static void testMultiCFBatchGetImpl(Map.Entry> entry @Test public void testBatchPut() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testBatchPutImpl); - truncateTables(ObHTableTestUtil.getConnection(), tableNames); } @Test public void testMultiCFPut() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchPutImpl); - truncateTables(ObHTableTestUtil.getConnection(), group2tableNames); } @Test public void testBatchGet() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testBatchGetImpl); - truncateTables(ObHTableTestUtil.getConnection(), tableNames); } @Test public void testMultiCFGet() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchGetImpl); - truncateTables(ObHTableTestUtil.getConnection(), group2tableNames); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java index b7012c3d..3a48d867 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java @@ -27,10 +27,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.junit.*; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; @@ -39,8 +36,8 @@ import static org.junit.Assert.assertTrue; public class OHTableSecondaryPartCellTTLTest { - private static List tableNames = new LinkedList(); - private static Map> group2tableNames = new LinkedHashMap<>(); + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); @BeforeClass public static void before() throws Exception { @@ -53,7 +50,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); -// dropTables(tableNames, group2tableNames); + dropTables(tableNames, group2tableNames); } @Before @@ -88,18 +85,11 @@ public static void testCellTTL(String tableName) throws Exception { Put put4 = new Put(key1.getBytes()); put4.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); put4.setTTL(0); - Put errorPut = new Put(key1.getBytes()); - errorPut.addColumn("family1".getBytes(), column1.getBytes(), toBytes(11L)); - errorPut.setTTL(10); Get get = new Get(key1.getBytes()); get.addFamily(family.getBytes()); get.setMaxVersions(10); - try { - hTable.put(errorPut); - } catch (Exception e) { - assertTrue(e.getCause().toString().contains("Unknown column 'TTL'")); - } + // test put and get hTable.put(put1); hTable.put(put2); @@ -131,11 +121,11 @@ public static void testCellTTL(String tableName) throws Exception { assertEquals(3, r.size()); assertEquals(23L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column1.getBytes()).get(0)))); + column1.getBytes()).get(0)))); assertEquals(35L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column2.getBytes()).get(0)))); + column2.getBytes()).get(0)))); assertEquals(5L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column3.getBytes()).get(0)))); + column3.getBytes()).get(0)))); Thread.sleep(10000); r = hTable.get(get); @@ -150,9 +140,9 @@ public static void testCellTTL(String tableName) throws Exception { assertEquals(2, r.size()); assertEquals(1L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column1.getBytes()).get(0)))); + column1.getBytes()).get(0)))); assertEquals(2L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column2.getBytes()).get(0)))); + column2.getBytes()).get(0)))); Thread.sleep(5000); r = hTable.get(get); @@ -165,9 +155,9 @@ public static void testCellTTL(String tableName) throws Exception { r = hTable.get(get); assertEquals(2, r.size()); assertEquals(26L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column1.getBytes()).get(0)))); + column1.getBytes()).get(0)))); assertEquals(35L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column2.getBytes()).get(0)))); + column2.getBytes()).get(0)))); // test append Thread.sleep(10000); @@ -181,7 +171,7 @@ public static void testCellTTL(String tableName) throws Exception { hTable.put(put3); Append append = new Append(key1.getBytes()); KeyValue kv = new KeyValue(key1.getBytes(), family.getBytes(), column1.getBytes(), - app.getBytes()); + app.getBytes()); append.add(kv); append.setTTL(-3000); hTable.append(append); @@ -191,17 +181,17 @@ public static void testCellTTL(String tableName) throws Exception { r = hTable.get(get); assertEquals(2, r.size()); assertEquals( - value1 + app, - Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column1.getBytes()).get(0)))); + value1 + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); Thread.sleep(3000); r = hTable.get(get); assertEquals(2, r.size()); assertEquals( - value1, - Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column1.getBytes()).get(0)))); + value1, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); Thread.sleep(7000); r = hTable.get(get); @@ -212,9 +202,9 @@ public static void testCellTTL(String tableName) throws Exception { r = hTable.get(get); assertEquals(1, r.size()); assertEquals( - app, - Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column1.getBytes()).get(0)))); + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); Thread.sleep(3000); append.add(family.getBytes(), column2.getBytes(), app.getBytes()); @@ -222,13 +212,13 @@ public static void testCellTTL(String tableName) throws Exception { r = hTable.get(get); assertEquals(2, r.size()); assertEquals( - app, - Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column1.getBytes()).get(0)))); + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column1.getBytes()).get(0)))); assertEquals( - app, - Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column2.getBytes()).get(0)))); + app, + Bytes.toString(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + column2.getBytes()).get(0)))); // test checkAndMutate Thread.sleep(3000); @@ -241,14 +231,14 @@ public static void testCellTTL(String tableName) throws Exception { delete.addColumn(family.getBytes(), column1.getBytes()); rowMutations.add(delete); boolean succ = hTable.checkAndMutate(key1.getBytes(), family.getBytes(), - column1.getBytes(), CompareFilter.CompareOp.EQUAL, toBytes(11L), rowMutations); + column1.getBytes(), CompareFilter.CompareOp.EQUAL, toBytes(11L), rowMutations); assertTrue(succ); r = hTable.get(get); assertEquals(r.size(), 2); assertEquals(22L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column1.getBytes()).get(0)))); + column1.getBytes()).get(0)))); assertEquals(33L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column2.getBytes()).get(0)))); + column2.getBytes()).get(0)))); Thread.sleep(10000); r = hTable.get(get); @@ -262,26 +252,26 @@ public static void testCellTTL(String tableName) throws Exception { put4.setTTL(10000); rowMutations.add(put4); succ = hTable.checkAndMutate(key1.getBytes(), family.getBytes(), column1.getBytes(), - CompareFilter.CompareOp.EQUAL, toBytes(1L), rowMutations); + CompareFilter.CompareOp.EQUAL, toBytes(1L), rowMutations); assertFalse(succ); succ = hTable.checkAndMutate(key1.getBytes(), family.getBytes(), column1.getBytes(), - CompareFilter.CompareOp.EQUAL, toBytes(11L), rowMutations); + CompareFilter.CompareOp.EQUAL, toBytes(11L), rowMutations); assertTrue(succ); r = hTable.get(get); assertEquals(r.size(), 2); assertEquals(22L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column1.getBytes()).get(0)))); + column1.getBytes()).get(0)))); assertEquals(33L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column2.getBytes()).get(0)))); + column2.getBytes()).get(0)))); Thread.sleep(5000); r = hTable.get(get); assertEquals(2, r.size()); assertEquals(22L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column1.getBytes()).get(0)))); + column1.getBytes()).get(0)))); assertEquals(33L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), - column2.getBytes()).get(0)))); + column2.getBytes()).get(0)))); Thread.sleep(5000); r = hTable.get(get); @@ -312,6 +302,52 @@ public static void testCellTTL(String tableName) throws Exception { hTable.close(); } + public static void testCellTTLWithRowkeyTTL(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + String key1 = "key1"; + String column1 = "cf1"; + String column2 = "cf2"; + String family = getColumnFamilyName(tableName); + Put put1 = new Put(key1.getBytes()); + put1.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put1.setTTL(5000); + Put put2 = new Put(key1.getBytes()); + put2.addColumn(family.getBytes(), column1.getBytes(), toBytes(22L)); + put2.addColumn(family.getBytes(), column2.getBytes(), toBytes(33L)); + put2.setTTL(10000); + Put put3 = new Put(key1.getBytes()); + put3.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put3.setTTL(-3000); + Put put4 = new Put(key1.getBytes()); + put4.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + put4.setTTL(0); + Put errorPut = new Put(key1.getBytes()); + errorPut.addColumn("family1".getBytes(), column1.getBytes(), toBytes(11L)); + errorPut.setTTL(10); + + // test put and get + hTable.put(put1); + hTable.put(put2); + hTable.put(put3); + hTable.put(put4); + + Assert.assertEquals(5, getSQLTableRowCnt(tableName)); + Thread.sleep(10000); + enableTTL(); + Get get = new Get(key1.getBytes()); + get.addFamily(family.getBytes()); + hTable.get(get); + Thread.sleep(10000); + + Assert.assertEquals(0, getSQLTableRowCnt(tableName)); + + // 6. close ttl knob + disableTTL(); + hTable.close(); + } + public static void testCellTTLSQL(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); @@ -319,11 +355,7 @@ public static void testCellTTLSQL(String tableName) throws Exception { String key1 = "key1"; String column1 = "cf1"; String column2 = "cf2"; - String column3 = "cf3"; String family = getColumnFamilyName(tableName); - String value1 = "v1"; - String value2 = "v2"; - String app = "app"; Result r; Put put1 = new Put(key1.getBytes()); @@ -351,9 +383,11 @@ public static void testCellTTLSQL(String tableName) throws Exception { Assert.assertEquals(5, getSQLTableRowCnt(tableName)); Thread.sleep(10000); - openTTLExecute(); - - checkUtilTimeout(()-> { + enableTTL(); + triggerTTL(); + List tableNames = new ArrayList<>(); + tableNames.add(tableName); + checkUtilTimeout(tableNames, ()-> { try { return getRunningNormalTTLTaskCnt() == 0; } catch (Exception e) { @@ -364,7 +398,60 @@ public static void testCellTTLSQL(String tableName) throws Exception { Assert.assertEquals(0, getSQLTableRowCnt(tableName)); // 6. close ttl knob - closeTTLExecute(); + disableTTL(); + hTable.close(); + } + + public static void testMultiCFCellTTL(Map.Entry> entry) throws Exception { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + List tableNames = entry.getValue(); + int familySize = tableNames.size(); + int ttl; + String key1 = "key1"; + String column1 = "cf1"; + for (int i = 0; i < familySize; i++) { + String tableName = tableNames.get(i); + String family = getColumnFamilyName(tableName); + Put put = new Put(key1.getBytes()); + put.addColumn(family.getBytes(), column1.getBytes(), toBytes("value")); + ttl = i * 5000; + put.setTTL(ttl); + hTable.put(put); + } + for (int i = 0; i < familySize; i++) { + int index = 0; + for (String tableName : tableNames) { + Get get = new Get(key1.getBytes()); + String family = getColumnFamilyName(tableName); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + int num = index <= i ? 0 : 1; + index++; + assertEquals(num, result.size()); + } + Thread.sleep(5000); + } + + hTable.close(); + } + + public static void testCellTTLWithNonTTLTable(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + String family = getColumnFamilyName(tableName); + String key1 = "key1"; + String column1 = "cf1"; + Put errorPut = new Put(key1.getBytes()); + errorPut.addColumn(family.getBytes(), column1.getBytes(), toBytes(11L)); + errorPut.setTTL(10); + try { + hTable.put(errorPut); + fail("unexpect error, put with ttl field and table not have ttl column should occur error"); + } catch (Exception e) { + assertTrue(e.getCause().getMessage().contains("Unknown column 'TTL'")); + } hTable.close(); } @@ -373,9 +460,27 @@ public void testCellTTL() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartCellTTLTest::testCellTTL); } + @Test + public void testMultiCFCellTTL() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartCellTTLTest::testMultiCFCellTTL); + } + @Test public void testCellTTLSQL() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartCellTTLTest::testCellTTLSQL); } + @Test + public void testCellTTLWithRowkeyTTL() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartCellTTLTest::testCellTTLWithRowkeyTTL); + } + + @Test + public void testCellTTLWithNonTTLTable() throws Throwable { + List NonTTLTable = new LinkedList(); + Map> NonTTLTableMultiCF = new LinkedHashMap<>(); + createTables(TableTemplateManager.TableType.NON_PARTITIONED_REGULAR, NonTTLTable, NonTTLTableMultiCF, true); + FOR_EACH(NonTTLTable, OHTableSecondaryPartCellTTLTest::testCellTTLWithNonTTLTable); + dropTables(NonTTLTable, NonTTLTableMultiCF); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java index 9b329c5d..f36898c1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java @@ -22,35 +22,34 @@ import com.alipay.oceanbase.hbase.util.TableTemplateManager; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.CompareFilter; -import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; import java.util.*; import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.COLUMN_FAMILY; import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES; import static org.junit.Assert.*; - public class OHTableSecondaryPartCheckAndMutateTest { private static List tableNames = new LinkedList(); - private static Map> group2tableNames = new LinkedHashMap<>(); - private static byte [] ROW = Bytes.toBytes("testRow"); - private static byte [] QUALIFIER = Bytes.toBytes("testQualifier"); - private static byte [] VALUE_1 = Bytes.toBytes("testValue"); - private static byte [] ROW_1 = Bytes.toBytes("testRow1"); - private static byte [] VALUE_2 = Bytes.toBytes("abcd"); - + private static Map> group2tableNames = new LinkedHashMap>(); + private static byte[] ROW = Bytes.toBytes("testRow"); + private static byte[] QUALIFIER = Bytes.toBytes("testQualifier"); + private static byte[] VALUE_1 = Bytes.toBytes("testValue"); + private static byte[] ROW_1 = Bytes.toBytes("testRow1"); + private static byte[] VALUE_2 = Bytes.toBytes("abcd"); @BeforeClass public static void before() throws Exception { openDistributedExecute(); - for (TableTemplateManager.TableType type : NORMAL_TABLES) { + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { createTables(type, tableNames, group2tableNames, true); } } @@ -58,7 +57,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); -// dropTables(tableNames, group2tableNames); + dropTables(tableNames, group2tableNames); } @Before @@ -82,11 +81,11 @@ public static void testCheckAndMutate(String tableName) throws Throwable { get.addFamily(FAMILY); Result result = hTable.get(get); assertTrue("Column A value should be a", - Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("A"))).equals("a")); + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("A"))).equals("a")); assertTrue("Column B value should be b", - Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))).equals("b")); + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))).equals("b")); assertTrue("Column C value should be c", - Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("C"))).equals("c")); + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("C"))).equals("c")); // put the same row again with C column deleted RowMutations rm = new RowMutations(ROW); @@ -97,8 +96,8 @@ public static void testCheckAndMutate(String tableName) throws Throwable { Delete del = new Delete(ROW); del.deleteColumn(FAMILY, Bytes.toBytes("C")); rm.add(del); - boolean res = hTable.checkAndMutate(ROW, FAMILY, Bytes.toBytes("A"), CompareFilter.CompareOp.EQUAL, - Bytes.toBytes("a"), rm); + boolean res = hTable.checkAndMutate(ROW, FAMILY, Bytes.toBytes("A"), + CompareFilter.CompareOp.EQUAL, Bytes.toBytes("a"), rm); assertTrue(res); // get row back and assert the values @@ -106,27 +105,23 @@ public static void testCheckAndMutate(String tableName) throws Throwable { get.addFamily(FAMILY); result = hTable.get(get); assertTrue("Column A value should be a", - Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("A"))).equals("a")); + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("A"))).equals("a")); assertTrue("Column B value should be b", - Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))).equals("b")); - assertTrue("Column C should not exist", - result.getValue(FAMILY, Bytes.toBytes("C")) == null); + Bytes.toString(result.getValue(FAMILY, Bytes.toBytes("B"))).equals("b")); + assertTrue("Column C should not exist", result.getValue(FAMILY, Bytes.toBytes("C")) == null); //Test that we get a hTable level exception try { Put p = new Put(ROW); - p.add(new byte[]{'b', 'o', 'g', 'u', 's'}, new byte[]{'A'}, new byte[0]); + p.add(new byte[] { 'b', 'o', 'g', 'u', 's' }, new byte[] { 'A' }, new byte[0]); rm = new RowMutations(ROW); rm.add(p); hTable.checkAndMutate(ROW, FAMILY, Bytes.toBytes("A"), CompareFilter.CompareOp.EQUAL, - Bytes.toBytes("a"), rm); + Bytes.toBytes("a"), rm); fail("Expected NoSuchColumnFamilyException"); - } catch (RetriesExhaustedWithDetailsException e) { - try { - throw e.getCause(0); - } catch (NoSuchColumnFamilyException e1) { - // expected - } + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("mutation family is not equal check family")); } hTable.close(); } @@ -134,7 +129,7 @@ public static void testCheckAndMutate(String tableName) throws Throwable { public static void testCheckAndPut(String tableName) throws Throwable { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); - byte [] FAMILY = getColumnFamilyName(tableName).getBytes(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); Put put1 = new Put(ROW); put1.add(FAMILY, QUALIFIER, VALUE_1); @@ -165,18 +160,19 @@ public static void testCheckAndPut(String tableName) throws Throwable { try { ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE_2, put3); fail("trying to check and modify different rows should have failed."); - } catch(Exception e) {} + } catch (Exception e) { + } hTable.close(); } public static void testCheckAndPutWithCompareOp(String tableName) throws Throwable { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); - final byte [] value1 = Bytes.toBytes("aaaa"); - final byte [] value2 = Bytes.toBytes("bbbb"); - final byte [] value3 = Bytes.toBytes("cccc"); - final byte [] value4 = Bytes.toBytes("dddd"); - byte [] FAMILY = getColumnFamilyName(tableName).getBytes(); + final byte[] value1 = Bytes.toBytes("aaaa"); + final byte[] value2 = Bytes.toBytes("bbbb"); + final byte[] value3 = Bytes.toBytes("cccc"); + final byte[] value4 = Bytes.toBytes("dddd"); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); Put put2 = new Put(ROW); put2.add(FAMILY, QUALIFIER, value2); @@ -190,58 +186,74 @@ public static void testCheckAndPutWithCompareOp(String tableName) throws Throwab // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL // turns out "match" - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value1, put2); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value1, + put2); assertEquals(ok, false); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value1, put2); + ok = hTable + .checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value1, put2); assertEquals(ok, false); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value1, put2); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, + value1, put2); assertEquals(ok, false); ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value1, put2); assertEquals(ok, true); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value1, put2); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value1, put2); assertEquals(ok, true); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value1, put3); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value1, + put3); assertEquals(ok, true); // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL // turns out "match" ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value4, put3); assertEquals(ok, false); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value4, put3); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value4, put3); assertEquals(ok, false); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value4, put3); + ok = hTable + .checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value4, put3); assertEquals(ok, false); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value4, put3); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value4, + put3); assertEquals(ok, true); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value4, put3); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, + value4, put3); assertEquals(ok, true); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value4, put2); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value4, + put2); assertEquals(ok, true); // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL // turns out "match" - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value2, put2); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value2, + put2); assertEquals(ok, false); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value2, put2); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value2, + put2); assertEquals(ok, false); ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value2, put2); assertEquals(ok, false); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value2, put2); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, + value2, put2); assertEquals(ok, true); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value2, put2); + ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value2, put2); assertEquals(ok, true); - ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value2, put3); + ok = hTable + .checkAndPut(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value2, put3); assertEquals(ok, true); + hTable.close(); } public static void testCheckAndDeleteWithCompareOp(String tableName) throws Throwable { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); - final byte [] value1 = Bytes.toBytes("aaaa"); - final byte [] value2 = Bytes.toBytes("bbbb"); - final byte [] value3 = Bytes.toBytes("cccc"); - final byte [] value4 = Bytes.toBytes("dddd"); - byte [] FAMILY = getColumnFamilyName(tableName).getBytes(); + final byte[] value1 = Bytes.toBytes("aaaa"); + final byte[] value2 = Bytes.toBytes("bbbb"); + final byte[] value3 = Bytes.toBytes("cccc"); + final byte[] value4 = Bytes.toBytes("dddd"); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); Put put2 = new Put(ROW); put2.add(FAMILY, QUALIFIER, value2); @@ -255,16 +267,21 @@ public static void testCheckAndDeleteWithCompareOp(String tableName) throws Thro // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL // turns out "match" - boolean ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value1, delete); + boolean ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, + value1, delete); assertEquals(ok, false); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value1, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value1, + delete); assertEquals(ok, false); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value1, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, + CompareFilter.CompareOp.GREATER_OR_EQUAL, value1, delete); assertEquals(ok, false); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value1, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value1, + delete); assertEquals(ok, true); hTable.put(put2); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value1, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value1, delete); assertEquals(ok, true); hTable.put(put2); @@ -273,44 +290,126 @@ public static void testCheckAndDeleteWithCompareOp(String tableName) throws Thro // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL // turns out "match" hTable.put(put3); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value4, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value4, + delete); assertEquals(ok, false); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value4, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value4, delete); assertEquals(ok, false); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value4, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value4, + delete); assertEquals(ok, false); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value4, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value4, + delete); assertEquals(ok, true); hTable.put(put3); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value4, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, + CompareFilter.CompareOp.GREATER_OR_EQUAL, value4, delete); assertEquals(ok, true); hTable.put(put3); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value4, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, + value4, delete); assertEquals(ok, true); // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL // turns out "match" hTable.put(put2); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value2, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, value2, + delete); assertEquals(ok, false); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, value2, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.NOT_EQUAL, + value2, delete); assertEquals(ok, false); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value2, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS, value2, + delete); assertEquals(ok, false); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER_OR_EQUAL, value2, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, + CompareFilter.CompareOp.GREATER_OR_EQUAL, value2, delete); assertEquals(ok, true); hTable.put(put2); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, value2, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.LESS_OR_EQUAL, + value2, delete); assertEquals(ok, true); hTable.put(put2); - ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value2, delete); + ok = hTable.checkAndDelete(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.EQUAL, value2, + delete); assertEquals(ok, true); + hTable.close(); + } + + private static void testCheckAndMutateMultiCF(Map.Entry> entry) + throws Exception { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + assertTrue(entry.getValue().size() > 1); + List tableNames = entry.getValue(); + byte[] FAMILY_1 = getColumnFamilyName(tableNames.get(0)).getBytes(); + Put put2 = new Put(ROW); + put2.add(FAMILY_1, QUALIFIER, VALUE_2); + hTable.put(put2); + RowMutations mutations = new RowMutations(ROW); + + Put put = new Put(ROW); + for (String tableName : tableNames) { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + put.addColumn(FAMILY, QUALIFIER, VALUE_1); + + } + mutations.add(put); + try { + hTable.checkAndMutate(ROW, FAMILY_1, QUALIFIER, CompareFilter.CompareOp.GREATER, + VALUE_2, mutations); + fail("unexpect error, check and mutate should not support multi cf"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("multi family is not supported")); + } + + mutations = new RowMutations(ROW); + byte[] FAMILY_2 = tableNames.get(1).getBytes(); + Put put3 = new Put(ROW); + put3.add(FAMILY_2, QUALIFIER, VALUE_1); + mutations.add(put2); + mutations.add(put3); + try { + hTable.checkAndMutate(ROW, FAMILY_1, QUALIFIER, CompareFilter.CompareOp.GREATER, + VALUE_2, mutations); + fail("unexpect error, check and mutate should not support multi cf"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("mutation family is not equal check family")); + } + hTable.close(); } + + private static void testCheckAndMutateSeires(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + RowMutations mutations = new RowMutations(ROW); + Put put2 = new Put(ROW); + put2.add(FAMILY, QUALIFIER, VALUE_2); + hTable.put(put2); + Put put3 = new Put(ROW); + put3.add(FAMILY, QUALIFIER, VALUE_1); + mutations.add(put3); + try { + hTable.checkAndMutate(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, VALUE_2, + mutations); + fail("unexpect error, check and mutate should not support series table"); + } catch (Exception e) { + assertTrue(e.getMessage().contains( + "query and mutate with hbase series type not supported")); + } + hTable.close(); + } + @Test public void testCheckAndMutate() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutate); @@ -330,4 +429,18 @@ public void testCheckAndPut() throws Throwable { public void testCheckAndPutWithCompareOp() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndPutWithCompareOp); } + + @Test + public void testCheckAndMutateMultiCF() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateMultiCF); + } + + @Test + public void testCheckAndMutateSeires() throws Throwable { + List seriesTables = new LinkedList(); + Map> seriesTablesMultiCF = new LinkedHashMap>(); + createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, seriesTables, seriesTablesMultiCF, true); + FOR_EACH(seriesTables, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateSeires); + dropTables(seriesTables, seriesTablesMultiCF); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java index c62e8bfe..1ecc3554 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java @@ -93,6 +93,7 @@ public static void testDeleteLastVersionImpl(String tableName) throws Exception Assert.assertTrue(cell.getTimestamp() != ts4); } } + hTable.close(); } public static void testDeleteSpecifiedImpl(String tableName) throws Exception { @@ -131,6 +132,7 @@ public static void testDeleteSpecifiedImpl(String tableName) throws Exception { Assert(tableName, ()->Assert.assertTrue(cell.getTimestamp() != ts1)); } } + hTable.close(); } public static void testDeleteColumnImpl(String tableName) throws Exception { @@ -169,6 +171,7 @@ public static void testDeleteColumnImpl(String tableName) throws Exception { Assert.assertTrue(cell.getTimestamp() != ts1); } } + hTable.close(); } public static void testDeleteFamilyImpl(String tableName) throws Exception { @@ -203,6 +206,7 @@ public static void testDeleteFamilyImpl(String tableName) throws Exception { Result result = hTable.get(get); Assert.assertEquals(0, result.size()); } + hTable.close(); } public static void testDeleteFamilyVersionImpl(String tableName) throws Exception { @@ -241,6 +245,7 @@ public static void testDeleteFamilyVersionImpl(String tableName) throws Exceptio Assert(tableName, ()->Assert.assertTrue(cell.getTimestamp() != ts2)); } } + hTable.close(); } public static void testMultiCFDeleteLastVersionImpl(Map.Entry> entry) throws Exception { @@ -286,6 +291,7 @@ public static void testMultiCFDeleteLastVersionImpl(Map.EntryAssert.assertTrue("should not found last version ts4", cell.getTimestamp() != ts4)); } } + hTable.close(); } public static void testMultiCFDeleteSpecifiedImpl(Map.Entry> entry) throws Exception { @@ -327,6 +333,7 @@ public static void testMultiCFDeleteSpecifiedImpl(Map.Entry Assert(tableName, ()->Assert.assertTrue("should not found last version ts2",cell.getTimestamp() != ts2)); } } + hTable.close(); } public static void testMultiCFDeleteColumnImpl(Map.Entry> entry) throws Exception { @@ -368,6 +375,7 @@ public static void testMultiCFDeleteColumnImpl(Map.Entry> e Assert(tableName, ()->Assert.assertTrue("should not found last version ts2",cell.getTimestamp() != ts2)); } } + hTable.close(); } @@ -405,6 +413,7 @@ public static void testMultiCFDeleteFamilyImpl(Map.Entry> e } Result result = hTable.get(get); Assert.assertEquals(0, result.size()); + hTable.close(); } public static void testMultiCFDeleteFamilyVersionImpl(Map.Entry> entry) throws Exception { @@ -441,6 +450,7 @@ public static void testMultiCFDeleteFamilyVersionImpl(Map.EntryAssert.assertEquals(6, result.size())); + hTable.close(); } public static void testDeleteAllImpl(Map.Entry> entry) throws Exception { @@ -476,6 +486,7 @@ public static void testDeleteAllImpl(Map.Entry> entry) thro } Result result = hTable.get(get); Assert(entry.getValue(), ()->Assert.assertEquals(0, result.size())); + hTable.close(); } @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java index 6f1365d1..b6eb6eb8 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -42,19 +42,15 @@ import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.assertEquals; - public class OHTableSecondaryPartGetTest { private static List tableNames = new LinkedList(); - private static Map> group2tableNames = new LinkedHashMap<>(); - + private static Map> group2tableNames = new LinkedHashMap>(); @BeforeClass public static void before() throws Exception { openDistributedExecute(); - for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { - if (!type.name().contains("TIME")) { - createTables(type, tableNames, group2tableNames, true); - } + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); } } @@ -69,7 +65,6 @@ public void prepareCase() throws Exception { truncateTables(tableNames, group2tableNames); } - public static void testGetImpl(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); @@ -77,10 +72,10 @@ public static void testGetImpl(String tableName) throws Exception { // 0. prepare data String family = getColumnFamilyName(tableName); String key = "putKey"; - String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; - String[] values = {"version1", "version2"}; // each column have two versions + String[] columns = { "putColumn1", "putColumn2", "putColumn3" }; + String[] values = { "version1", "version2" }; // each column have two versions long curTs = System.currentTimeMillis(); - long[] ts = {curTs, curTs+1}; // each column have two versions + long[] ts = { curTs, curTs + 1 }; // each column have two versions String latestValue = values[1]; long lastTs = ts[1]; for (int i = 0; i < values.length; i++) { @@ -109,7 +104,8 @@ public static void testGetImpl(String tableName) throws Exception { Cell[] cells = result.rawCells(); assertEquals(columns.length, cells.length); for (int i = 0; i < columns.length; i++) { - ObHTableSecondaryPartUtil.AssertKeyValue(key, columns[i], lastTs, latestValue, cells[i]); + ObHTableSecondaryPartUtil.AssertKeyValue(key, columns[i], lastTs, latestValue, + cells[i]); } } @@ -144,7 +140,7 @@ public static void testGetImpl(String tableName) throws Exception { get.addFamily(family.getBytes()); get.setMaxVersions(2); ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(values[0]))); + new BinaryComparator(toBytes(values[0]))); get.setFilter(valueFilter); Result r = hTable.get(get); Assert.assertEquals(columns.length, r.raw().length); @@ -155,19 +151,19 @@ public static void testGetImpl(String tableName) throws Exception { hTable.close(); } - + public static void testMultiCFGetImpl(Map.Entry> entry) throws Exception { // 0. prepare data String key = "putKey"; - String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; + String[] columns = { "putColumn1", "putColumn2", "putColumn3" }; String groupName = getTableName(entry.getKey()); - String[] values = {"version1", "version2"}; // each column have two versions + String[] values = { "version1", "version2" }; // each column have two versions String latestValue = values[1]; List tableNames = entry.getValue(); OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); long timestamp = System.currentTimeMillis(); - long[] ts = {timestamp, timestamp+1}; + long[] ts = { timestamp, timestamp + 1 }; long lastTs = ts[1]; hTable.init(); @@ -205,7 +201,8 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr Cell[] cells = result.rawCells(); assertEquals(columns.length, cells.length); for (int i = 0; i < columns.length; i++) { - ObHTableSecondaryPartUtil.AssertKeyValue(key, columns[i], lastTs, latestValue, cells[i]); + ObHTableSecondaryPartUtil.AssertKeyValue(key, columns[i], lastTs, latestValue, + cells[i]); } } } @@ -279,7 +276,7 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr Get get = new Get(key.getBytes()); get.setMaxVersions(2); ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(values[0]))); + new BinaryComparator(toBytes(values[0]))); get.setFilter(valueFilter); Result r = hTable.get(get); Assert.assertEquals(tableNames.size() * columns.length, r.raw().length); @@ -292,13 +289,14 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr } } } + hTable.close(); } @Test public void testGet() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartGetTest::testGetImpl); } - + @Test public void testMultiCFGet() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartGetTest::testMultiCFGetImpl); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java index 395d3799..04b65f02 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java @@ -37,12 +37,11 @@ import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; public class OHTableSecondaryPartIncrementTest { private static List tableNames = new LinkedList(); - private static Map> group2tableNames = null; + private static Map> group2tableNames = new LinkedHashMap>(); @BeforeClass public static void before() throws Exception { @@ -55,7 +54,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - // dropTables(tableNames, group2tableNames); + dropTables(tableNames, group2tableNames); } @Before @@ -103,87 +102,138 @@ public static void testIncrement(String tableName) throws Exception { private static void testIncBorder(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); - - byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); - byte[] ROW = "incKey".getBytes(); - byte[] v1 = Bytes.toBytes("ab"); - byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), - Bytes.toBytes("c") }; - Put put = new Put(ROW); - put.addColumn(FAMILY, QUALIFIERS[1], v1); - hTable.put(put); - Increment inc = new Increment(ROW); - inc.addColumn(FAMILY, QUALIFIERS[1], 2L); try { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + byte[] ROW = "incKey".getBytes(); + byte[] v1 = Bytes.toBytes("ab"); + byte[][] QUALIFIERS = new byte[][]{Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c")}; + Put put = new Put(ROW); + put.addColumn(FAMILY, QUALIFIERS[1], v1); + hTable.put(put); + Increment inc = new Increment(ROW); + inc.addColumn(FAMILY, QUALIFIERS[1], 2L); + try { + hTable.increment(inc); + fail("unexpect error, increment only support long value type"); + } catch (Exception e) { + assertTrue(e.getCause().getMessage().contains("OB_KV_HBASE_INCR_FIELD_IS_NOT_LONG")); + } + Get get = new Get(ROW); + get.setMaxVersions(10); + get.addFamily(FAMILY); + Result result = hTable.get(get); + assertEquals(1, result.size()); + byte[] ROW1 = "incKey1".getBytes(); + inc = new Increment(ROW1); + inc.addColumn(FAMILY, QUALIFIERS[1], 2L); hTable.increment(inc); - } catch (Exception e) { - assertTrue(e.getCause().getMessage().contains("OB_KV_HBASE_INCR_FIELD_IS_NOT_LONG")); + get = new Get(ROW1); + get.setMaxVersions(10); + get.addFamily(FAMILY); + result = hTable.get(get); + assertEquals(1, result.size()); + assertEquals(2L, Bytes.toLong(result.raw()[0].getValue())); + inc.addColumn(FAMILY, QUALIFIERS[0], 2L); + hTable.increment(inc); + get.setMaxVersions(10); + get.addFamily(FAMILY); + result = hTable.get(get); + assertEquals(3, result.size()); + assertEquals(4L, + Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(0).getValue())); + assertEquals(2L, + Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(1).getValue())); + assertEquals(2L, + Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0).getValue())); + } finally { + hTable.close(); } - Get get = new Get(ROW); - get.setMaxVersions(10); - get.addFamily(FAMILY); - Result result = hTable.get(get); - assertEquals(1, result.size()); - byte[] ROW1 = "incKey1".getBytes(); - inc = new Increment(ROW1); - inc.addColumn(FAMILY, QUALIFIERS[1], 2L); - hTable.increment(inc); - get = new Get(ROW1); - get.setMaxVersions(10); - get.addFamily(FAMILY); - result = hTable.get(get); - assertEquals(1, result.size()); - assertEquals(2L, Bytes.toLong(result.raw()[0].getValue())); - inc.addColumn(FAMILY, QUALIFIERS[0], 2L); - hTable.increment(inc); - get.setMaxVersions(10); - get.addFamily(FAMILY); - result = hTable.get(get); - assertEquals(3, result.size()); - assertEquals(4L, - Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(0).getValue())); - assertEquals(2L, - Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(1).getValue())); - assertEquals(2L, - Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0).getValue())); - hTable.close(); } private static void testIncCon(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); - byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); - String column = "incColumn"; - byte[] ROW = "incKey".getBytes(); - long expect = 0; - ThreadPoolExecutor threadPoolExecutor = OHTable.createDefaultThreadPoolExecutor(1, 100,100); - AtomicInteger atomicInteger = new AtomicInteger(0); - CountDownLatch countDownLatch = new CountDownLatch(100); - for (int i = 0; i < 100; i++) { - Increment inc = new Increment(ROW); - inc.addColumn(FAMILY, column.getBytes(), 2L); - threadPoolExecutor.submit(() -> { - try { - hTable.increment(inc); - atomicInteger.incrementAndGet(); - } catch (Exception e) { - if (!e.getCause().getMessage().contains("OB_TRY_LOCK_ROW_CONFLICT") && !e.getCause().getMessage().contains("OB_TIMEOUT")) { - throw new RuntimeException(e); + try { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "incColumn"; + byte[] ROW = "incKey".getBytes(); + long expect = 0; + ThreadPoolExecutor threadPoolExecutor = OHTable.createDefaultThreadPoolExecutor(1, 100, 100); + AtomicInteger atomicInteger = new AtomicInteger(0); + CountDownLatch countDownLatch = new CountDownLatch(100); + for (int i = 0; i < 100; i++) { + Increment inc = new Increment(ROW); + inc.addColumn(FAMILY, column.getBytes(), 2L); + threadPoolExecutor.submit(() -> { + try { + hTable.increment(inc); + atomicInteger.incrementAndGet(); + } catch (Exception e) { + if (!e.getCause().getMessage().contains("OB_TRY_LOCK_ROW_CONFLICT") && !e.getCause().getMessage().contains("OB_TIMEOUT")) { + throw new RuntimeException(e); + } + } finally { + countDownLatch.countDown(); } - } finally { - countDownLatch.countDown(); - } - }); + }); + } + countDownLatch.await(100000, TimeUnit.MILLISECONDS); + for (int i = 0; i < atomicInteger.get(); i++) { + expect += 2; + } + Get get = new Get(ROW); + get.setMaxVersions(1); + get.addColumn(FAMILY, column.getBytes()); + Result result = hTable.get(get); + assertEquals(expect, Bytes.toLong(result.getColumnCells(FAMILY, column.getBytes()).get(0).getValue())); + } finally { + hTable.close(); + } + } + + private static void testIncrementMultiCF(Map.Entry> entry) + throws Exception { + String groupName = getTableName(entry.getKey()); + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(groupName); + hTable.init(); + try { + List tableNames = entry.getValue(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + Long v = 11L; + Increment increment = new Increment(ROW); + for (String tableName : tableNames) { + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + increment.addColumn(FAMILY, column.getBytes(), v); + } + try { + hTable.increment(increment); + fail("unexpect error, increment should not support multi cf"); + } catch (Exception e) { + assertTrue(e.getMessage().contains("multi family is not supported")); + } + } finally { + hTable.close(); } - countDownLatch.await(100000, TimeUnit.MILLISECONDS); - for (int i = 0; i < atomicInteger.get(); i++) { - expect += 2; + } + + private static void testIncrementSeires(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + String column = "appColumn"; + byte[] ROW = "appendKey".getBytes(); + Long v = 11L; + Increment increment = new Increment(ROW); + increment.addColumn(FAMILY, column.getBytes(), v); + try { + hTable.increment(increment); + fail("unexpect error, increment should not support series table"); + } catch (Exception e) { + assertTrue(e.getCause().getMessage() + .contains("query and mutate with hbase series type not supported")); } - Get get = new Get(ROW); - get.setMaxVersions(1); - get.addColumn(FAMILY, column.getBytes()); - Result result = hTable.get(get); - assertEquals(expect, Bytes.toLong(result.getColumnCells(FAMILY, column.getBytes()).get(0).getValue())); hTable.close(); } @@ -201,4 +251,17 @@ public void testBorderInc() throws Throwable { public void testIncConcurrency() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartIncrementTest::testIncCon); } + + @Test + public void testIncrementMultiCF() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartIncrementTest::testIncrementMultiCF); + } + + @Test + public void testIncrementSeires() throws Throwable { + List series_tables = new LinkedList(); + createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, null, true); + FOR_EACH(series_tables, OHTableSecondaryPartIncrementTest::testIncrementSeires); + dropTables(series_tables, null); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java index 94a81316..a070e6a1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -69,9 +69,10 @@ public static void testPutImpl(String tableName) throws Exception { String value = "value"; { // put new key and get + long timestamp = System.currentTimeMillis(); Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), toBytes(column1 + value)); - put.add(family.getBytes(), column2.getBytes(), toBytes(column2 + value)); + put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value)); + put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value)); hTable.put(put); Get get = new Get(toBytes(key)); @@ -198,8 +199,8 @@ public static void testMultiCFPutImpl(Map.Entry> entry) thr Get get = new Get(toBytes(key)); for (String tableName : entry.getValue()) { String family = getColumnFamilyName(tableName); - put.add(family.getBytes(), column1.getBytes(), toBytes(column1 + value)); - put.add(family.getBytes(), column2.getBytes(), toBytes(column2 + value)); + put.add(family.getBytes(), column1.getBytes(), currentTime, toBytes(column1 + value)); + put.add(family.getBytes(), column2.getBytes(), currentTime, toBytes(column2 + value)); get.addColumn(family.getBytes(), column1.getBytes()); get.addColumn(family.getBytes(), column2.getBytes()); } @@ -233,7 +234,6 @@ public static void testMultiCFPutImpl(Map.Entry> entry) thr } for (String tableName : entry.getValue()) { String family = getColumnFamilyName(tableName); - // TODO: Get/Scan返回的结果Q 带了cf, 这里预期跑不过 Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column1 + value).getBytes(), r.getValue(family.getBytes(), column1.getBytes())))); Assert(entry.getValue(), () -> Assert.assertTrue(secureCompare((column2 + value).getBytes(), r.getValue(family.getBytes(), column2.getBytes())))); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java index 90045ec0..257793ec 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java @@ -36,15 +36,13 @@ public class OHTableSecondaryPartScanTest { private static List tableNames = new LinkedList(); - private static Map> group2tableNames = new LinkedHashMap<>(); + private static Map> group2tableNames = new LinkedHashMap>(); @BeforeClass public static void before() throws Exception { openDistributedExecute(); - for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { - if (!type.name().contains("TIME")) { - createTables(type, tableNames, group2tableNames, true); - } + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); } } @@ -72,10 +70,10 @@ public static void testScanImpl(String tableName) throws Exception { String family = getColumnFamilyName(tableName); long ts = System.currentTimeMillis(); - String keys[] = {"putKey1", "putKey2", "putKey3"}; - String columns[] = {"putColumn1", "putColumn2"}; - String values[] = {"putValue1", "putValue2"}; - long tss[] = {ts, ts + 1}; + String keys[] = { "putKey1", "putKey2", "putKey3" }; + String columns[] = { "putColumn1", "putColumn2" }; + String values[] = { "putValue1", "putValue2" }; + long tss[] = { ts, ts + 1 }; long lastTs = tss[1]; String latestValue = values[1]; @@ -163,7 +161,7 @@ public static void testScanImpl(String tableName) throws Exception { scan.setMaxVersions(2); scan.addFamily(family.getBytes()); ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(values[0]))); + new BinaryComparator(toBytes(values[0]))); scan.setFilter(valueFilter); ResultScanner scanner = hTable.getScanner(scan); List cells = getCellsFromScanner(scanner); @@ -226,21 +224,22 @@ public static void testScanImpl(String tableName) throws Exception { // 8. scan in reverse { -// Scan scan = new Scan(keys[2].getBytes(), keys[0].getBytes()); -// scan.addFamily(family.getBytes()); -// scan.setReversed(true); -// ResultScanner scanner = hTable.getScanner(scan); -// List cells = getCellsFromScanner(scanner); -// -// int cellIndex = 0; -// for (int i = 1; i >= 0; i--) { -// for (String column : columns) { -// AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); -// cellIndex++; -// } -// } -// assertEquals(columns.length * 2, cells.size()); + Scan scan = new Scan(keys[1].getBytes(), "putKey".getBytes()); + scan.addFamily(family.getBytes()); + scan.setReversed(true); + ResultScanner scanner = hTable.getScanner(scan); + List cells = getCellsFromScanner(scanner); + + int cellIndex = 0; + for (int i = 1; i >= 0; i--) { + for (String column : columns) { + AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); + cellIndex++; + } + } + assertEquals(columns.length * 2, cells.size()); } + hTable.close(); } public static void testMultiCFScanImpl(Map.Entry> entry) throws Exception { @@ -255,10 +254,10 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th hTable.init(); long ts = System.currentTimeMillis(); - String keys[] = {"putKey1", "putKey2", "putKey3"}; - String columns[] = {"putColumn1", "putColumn2"}; - String values[] = {"putValue1", "putValue2"}; - long tss[] = {ts, ts + 1}; + String keys[] = { "putKey1", "putKey2", "putKey3" }; + String columns[] = { "putColumn1", "putColumn2" }; + String values[] = { "putValue1", "putValue2" }; + long tss[] = { ts, ts + 1 }; long lastTs = tss[1]; String latestValue = values[1]; List tableNames = entry.getValue(); @@ -303,7 +302,8 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th int cellIndex = 0; for (int i = 0; i < 2; i++) { for (String column : columns) { - AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); cellIndex++; } } @@ -324,7 +324,8 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th String family = getColumnFamilyName(tableName); for (String column : columns) { Cell cell = cells.get(cellIndex); - AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); cellIndex++; } } @@ -349,7 +350,8 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th for (String tableName : tableNames) { String family = getColumnFamilyName(tableName); for (String column : columns) { - AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); cellIndex++; } } @@ -370,7 +372,8 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th String family = getColumnFamilyName(tableName); for (String column : columns) { for (int j = values.length - 1; j >= 0; j--) { - AssertKeyValue(keys[i], family, column, tss[j], values[j], cells.get(cellIndex)); + AssertKeyValue(keys[i], family, column, tss[j], values[j], + cells.get(cellIndex)); cellIndex++; } } @@ -391,7 +394,8 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th for (String tableName : tableNames) { String family = getColumnFamilyName(tableName); for (String column : columns) { - AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); cellIndex++; } } @@ -403,7 +407,7 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); scan.setMaxVersions(2); ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, - new BinaryComparator(toBytes(values[0]))); + new BinaryComparator(toBytes(values[0]))); scan.setFilter(valueFilter); ResultScanner scanner = hTable.getScanner(scan); List cells = getCellsFromScanner(scanner); @@ -413,7 +417,8 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th for (String tableName : tableNames) { String family = getColumnFamilyName(tableName); for (String column : columns) { - AssertKeyValue(keys[i], family, column, tss[0], values[0], cells.get(cellIndex)); + AssertKeyValue(keys[i], family, column, tss[0], values[0], + cells.get(cellIndex)); cellIndex++; } } @@ -434,7 +439,8 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th for (String tableName : tableNames) { String family = getColumnFamilyName(tableName); for (String column : columns) { - AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); cellIndex++; } } @@ -457,13 +463,12 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th Assert.assertEquals(null, result); } - // 10. multi cf scan with family scan and column-specific scan { Scan scan = new Scan(keys[0].getBytes(), keys[2].getBytes()); for (int i = 0; i < tableNames.size(); i++) { String family = getColumnFamilyName(tableNames.get(i)); - if (i % 2 == 0 ) { + if (i % 2 == 0) { scan.addFamily(family.getBytes()); } else { for (String column : columns) { @@ -479,12 +484,14 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th for (String tableName : tableNames) { String family = getColumnFamilyName(tableName); for (String column : columns) { - AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); + AssertKeyValue(keys[i], family, column, lastTs, latestValue, + cells.get(cellIndex)); cellIndex++; } } } } + hTable.close(); } @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java index c60e63d4..08d600df 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java @@ -15,22 +15,19 @@ import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.assertEquals; - public class OHTableSecondaryPartTTLTest { - private static List tableNames = new LinkedList(); - private static Map> group2tableNames = new LinkedHashMap<>(); + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); @BeforeClass public static void before() throws Exception { openDistributedExecute(); - for (TableTemplateManager.TableType type : TableTemplateManager.TableType.values()) { - if (!type.name().contains("TIME")) { - createTables(type, tableNames, group2tableNames, true); - alterTableTimeToLive(tableNames, true, 10); - for (List groupTableNames : group2tableNames.values()) { - alterTableTimeToLive(groupTableNames, true, 10); - } - } + for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + alterTableTimeToLive(tableNames, true, 10); + for (List groupTableNames : group2tableNames.values()) { + alterTableTimeToLive(groupTableNames, true, 10); } } @@ -65,6 +62,7 @@ public static void testTTLImpl(List tableNames) throws Exception { } } } + hTable.close(); } // 1. sleep util data expired @@ -85,6 +83,7 @@ public static void testTTLImpl(List tableNames) throws Exception { get.addFamily(family.getBytes()); Result result = hTable.get(get); Assert.assertEquals(0, result.rawCells().length); + hTable.close(); } // 3. using sql to scan expired but not delete yet hbase data @@ -98,7 +97,7 @@ public static void testTTLImpl(List tableNames) throws Exception { triggerTTL(); // 5. check util expired hbase data is deleted by ttl tasks - checkUtilTimeout(()-> { + checkUtilTimeout(tableNames, ()-> { try { return getRunningNormalTTLTaskCnt() == 0; } catch (Exception e) { @@ -140,7 +139,7 @@ public static void testMultiCFTTLImpl(Map> group2tableNames } } } - + hTable.close(); } // 1. sleep util data expired @@ -159,6 +158,7 @@ public static void testMultiCFTTLImpl(Map> group2tableNames Get get = new Get(keys[0].getBytes()); Result result = hTable.get(get); Assert.assertEquals(0, result.rawCells().length); + hTable.close(); } // 3. using sql to scan expired but not delete yet hbase data @@ -172,7 +172,7 @@ public static void testMultiCFTTLImpl(Map> group2tableNames triggerTTL(); // 5. check util expired hbase data is deleted by ttl tasks - checkUtilTimeout(()-> { + checkUtilTimeout(allTableNames, ()-> { try { return getRunningNormalTTLTaskCnt() == 0; } catch (Exception e) { @@ -193,6 +193,7 @@ void testRowkeyTTL(List tableNames, Boolean useScan, Boolean isReversed) // 0. prepare data String keys[] = {"putKey1", "putKey2", "putKey3"}; String endKey = "putKey4"; + String reversedEndKey = "putKey"; String columns[] = {"putColumn1", "putColumn2"}; String values[] = {"putValue1", "putValue2", "putValue3", "putValue4"}; for (String tableName : tableNames) { @@ -203,11 +204,12 @@ void testRowkeyTTL(List tableNames, Boolean useScan, Boolean isReversed) for (String column : columns) { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), values[i].getBytes()); + put.add(family.getBytes(), column.getBytes(), values[i].getBytes()); hTable.put(put); } } } + hTable.close(); } // 1. sleep util data expired @@ -218,8 +220,14 @@ void testRowkeyTTL(List tableNames, Boolean useScan, Boolean isReversed) // 3. SQL can scan expired but not delete yet hbase data for (String tableName : tableNames) { - Assert.assertEquals(keys.length * columns.length * values.length, - getSQLTableRowCnt(tableName)); + ObHTableTestUtil.Assert(tableName, ()-> { + try { + Assert.assertEquals(keys.length * columns.length * values.length, + getSQLTableRowCnt(tableName)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); } // 4. use Hbase scan/get expired data to trigger ttl @@ -228,9 +236,16 @@ void testRowkeyTTL(List tableNames, Boolean useScan, Boolean isReversed) hTable.init(); String family = getColumnFamilyName(tableName); if (useScan) { - Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + Scan scan = new Scan(); + if (isReversed) { + scan.setReversed(true); + scan.setStartRow(keys[2].getBytes()); + scan.setStopRow(reversedEndKey.getBytes()); + } else { + scan.setStartRow(keys[0].getBytes()); + scan.setStopRow(endKey.getBytes()); + } scan.addFamily(family.getBytes()); - scan.setReversed(isReversed); ResultScanner scanner = hTable.getScanner(scan); List cells = getCellsFromScanner(scanner); assertEquals(0, cells.size()); @@ -242,10 +257,11 @@ void testRowkeyTTL(List tableNames, Boolean useScan, Boolean isReversed) assertEquals(0, result.rawCells().length); } } + hTable.close(); } // 5. wait to disable - checkUtilTimeout(()-> { + checkUtilTimeout(tableNames, ()-> { try { Boolean passed = true; for (int i = 0; passed && i < tableNames.size(); i++) { @@ -287,6 +303,7 @@ void testMultiCFRowkeyTTL(Map> group2tableNames, Boolean us } } } + hTable.close(); } // 1. sleep util data expired @@ -319,10 +336,11 @@ void testMultiCFRowkeyTTL(Map> group2tableNames, Boolean us assertEquals(0, result.rawCells().length); } } + hTable.close(); } // 5. wait to disable - checkUtilTimeout(()-> { + checkUtilTimeout(allTableNames, ()-> { try { Boolean passed = true; for (int i = 0; passed && i < allTableNames.size(); i++) { @@ -340,10 +358,9 @@ void testMultiCFRowkeyTTL(Map> group2tableNames, Boolean us disableTTL(); } - @Test public void testTTL() throws Throwable { - testTTLImpl(tableNames); + testTTLImpl(tableNames); } @Test @@ -356,7 +373,7 @@ public void testRowkeyTTL() throws Exception { testRowkeyTTL(tableNames, true, false); testRowkeyTTL(tableNames, false, false); // TODO: open the test after reverse scan is ok - // testRowkeyTTL(tableNames, true); +// testRowkeyTTL(tableNames, true, true); } @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index 934170e6..fc24e3f6 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -92,7 +92,16 @@ public static void createTables(Connection conn, TableTemplateManager.TableType TimeGenerator.TimeRange timeRange = TimeGenerator.generateTestTimeRange(); String tableGroup = TableTemplateManager.getTableGroupName(type, true); String tableGroupSql = TableTemplateManager.generateTableGroupSQL(tableGroup); - conn.createStatement().execute(tableGroupSql); + try { + conn.createStatement().execute(tableGroupSql); + System.out.println("============= create table_group: " + getTableName(tableGroup) + " =============\n" + (printSql ? tableGroupSql : "") + " \n============= done =============\n"); + } catch (SQLSyntaxErrorException e) { + if (!e.getMessage().contains("already exists")) { + throw e; + } else { + System.out.println("============= table_group: " + getTableName(tableGroup) + " already exist ============="); + } + } group2tableNames.put(tableGroup, new LinkedList<>()); for (int i = 1; i <= 3; ++i) { String tableName = TableTemplateManager.generateTableName(tableGroup, true, i); @@ -211,19 +220,23 @@ public static String getColumnFamilyName(String input) throws Exception { return result; } - public static void alterTableTimeToLive(List tableNames, boolean printSql, long timeToLive) - throws Exception { + public static void alterTableTimeToLive(List tableNames, boolean printSql, + long timeToLive) throws Exception { Connection conn = ObHTableTestUtil.getConnection(); if (tableNames != null) { for (String tableName : tableNames) { - String alterTableTTLSQL = "ALTER TABLE " + tableName + - String.format(" kv_attributes ='{\"Hbase\": {\"TimeToLive\": %d}}';", timeToLive); + String alterTableTTLSQL = "ALTER TABLE " + + tableName + + String + .format( + " kv_attributes ='{\"Hbase\": {\"TimeToLive\": %d}}';", + timeToLive); try { conn.createStatement().execute(alterTableTTLSQL); - System.out.println("============= alter table ttl: " + tableName + " table_group: " - + getTableName(tableName) + " =============\n" - + (printSql ? alterTableTTLSQL : "") - + " \n============= done =============\n"); + System.out.println("============= alter table ttl: " + tableName + + " table_group: " + getTableName(tableName) + + " =============\n" + (printSql ? alterTableTTLSQL : "") + + " \n============= done =============\n"); } catch (SQLSyntaxErrorException e) { throw e; } @@ -231,19 +244,23 @@ public static void alterTableTimeToLive(List tableNames, boolean printSq } } - public static void alterTableMaxVersion(List tableNames, boolean printSql, long maxVersion) - throws Exception { + public static void alterTableMaxVersion(List tableNames, boolean printSql, + long maxVersion) throws Exception { Connection conn = ObHTableTestUtil.getConnection(); if (tableNames != null) { for (String tableName : tableNames) { - String alterTableTTLSQL = "ALTER TABLE " + tableName + - String.format(" kv_attributes ='{\"Hbase\": {\"MaxVersions\": %d}}';", maxVersion); + String alterTableTTLSQL = "ALTER TABLE " + + tableName + + String + .format( + " kv_attributes ='{\"Hbase\": {\"MaxVersions\": %d}}';", + maxVersion); try { conn.createStatement().execute(alterTableTTLSQL); - System.out.println("============= alter table ttl: " + tableName + " table_group: " - + getTableName(tableName) + " =============\n" - + (printSql ? alterTableTTLSQL : "") - + " \n============= done =============\n"); + System.out.println("============= alter table ttl: " + tableName + + " table_group: " + getTableName(tableName) + + " =============\n" + (printSql ? alterTableTTLSQL : "") + + " \n============= done =============\n"); } catch (SQLSyntaxErrorException e) { throw e; } @@ -259,12 +276,14 @@ public static int getSQLTableRowCnt(String tableName) throws Exception { if (resultSet.next()) { rowCnt = resultSet.getInt(1); } + System.out.println("============= rowCnt: " + rowCnt + " ============="); return rowCnt; } public static int getRunningNormalTTLTaskCnt() throws Exception { Connection conn = ObHTableTestUtil.getConnection(); - String RowCountSQL = "SELECT COUNT(*) FROM " + "OCEANBASE.DBA_OB_KV_TTL_TASKS where TASK_TYPE = 'NORMAL'"; + String RowCountSQL = "SELECT COUNT(*) FROM " + + "OCEANBASE.DBA_OB_KV_TTL_TASKS where TASK_TYPE = 'NORMAL'"; ResultSet resultSet = conn.createStatement().executeQuery(RowCountSQL); int rowCnt = 0; if (resultSet.next()) { @@ -291,7 +310,8 @@ public static void disableTTL() throws Exception { conn.createStatement().execute(stmt); } - public static void AssertKeyValue(String key, String qualifier, long timestamp, String value, Cell cell) { + public static void AssertKeyValue(String key, String qualifier, long timestamp, String value, + Cell cell) { Assert.assertEquals(key, Bytes.toString(cell.getRow())); Assert.assertEquals(qualifier, Bytes.toString(cell.getQualifier())); Assert.assertEquals(timestamp, cell.getTimestamp()); @@ -304,7 +324,8 @@ public static void AssertKeyValue(String key, String qualifier, String value, Ce Assert.assertEquals(value, Bytes.toString(cell.getValue())); } - public static void AssertKeyValue(String key, String family, String qualifier, long timestamp, String value, Cell cell) { + public static void AssertKeyValue(String key, String family, String qualifier, long timestamp, + String value, Cell cell) { Assert.assertEquals(key, Bytes.toString(cell.getRow())); Assert.assertEquals(family, Bytes.toString(cell.getFamily())); Assert.assertEquals(qualifier, Bytes.toString(cell.getQualifier())); @@ -323,12 +344,14 @@ public static List getCellsFromScanner(ResultScanner scanner) { } - public static void checkUtilTimeout(Supplier function, long timeout, long interval) throws Exception { + public static void checkUtilTimeout(List tableNames, Supplier function, long timeout, long interval) throws Exception { long startTime = System.currentTimeMillis(); while (System.currentTimeMillis() - startTime < timeout) { - if (function.get()) { return; } + if (function.get()) { + return; + } Thread.sleep(interval); } - Assert.assertTrue("Timeout while waiting for the function to return expected result", false); + ObHTableTestUtil.Assert(tableNames, ()-> Assert.fail("Timeout while waiting for the function to return expected result")); } } From 075be677d6d12e01d2e5427d44e052198dbf6147 Mon Sep 17 00:00:00 2001 From: shen Date: Wed, 26 Mar 2025 17:17:53 +0800 Subject: [PATCH 30/40] [Test] add testcases for hbase timeseries model get/scan (cherry picked from commit 8bbb915ffcede207a7090b93fc613003d1164b05) --- .../secondary/OHTableTimeSeriesGetTest.java | 149 ++++++ .../secondary/OHTableTimeSeriesScanTest.java | 227 +++++++++ .../hbase/util/ObHTableSecondaryPartUtil.java | 28 +- .../hbase/util/TableTemplateManager.java | 466 ++++++++---------- 4 files changed, 614 insertions(+), 256 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java new file mode 100644 index 00000000..b117d630 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java @@ -0,0 +1,149 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.junit.*; + +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TIMESERIES_TABLES; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + + +public class OHTableTimeSeriesGetTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TIMESERIES_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + + public static void testGetImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data + String family = getColumnFamilyName(tableName); + String key = "putKey"; + String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; + String[] columns1 = {"putColumn1", "putColumn2"}; + String[] columns2 = {"putColumn3"}; + String[] values = {"version1", "version2"}; // each column have two versions + long curTs = System.currentTimeMillis(); + long[] ts = {curTs, curTs+1}; // each column have two versions + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns1.length; j++) { + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), columns1[j].getBytes(), ts[i], toBytes(values[i])); + hTable.put(put); + } + } + + Put put = new Put(toBytes(key)); + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns2.length; j++) { + put.add(family.getBytes(), columns2[j].getBytes(), ts[i], toBytes(values[i])); + } + } + hTable.put(put); + + + // 1. get specify column + { + int index = 0; + Get get = new Get(key.getBytes()); + get.addColumn(family.getBytes(), columns[index].getBytes()); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + assertEquals(2, cells.length); + sortCells(cells); + for (int i = values.length - 1; i >= 0; i--) { + AssertKeyValue(key, columns[index], ts[i], values[i], cells[values.length - 1-i]); + } + } + + // 2. get do not specify column + { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + Result result = hTable.get(get); + Cell[] cells = result.rawCells(); + assertEquals(columns.length * values.length, cells.length); + sortCells(cells); + int idx = 0; + for (int i = 0; i < columns.length; i++) { + for (int j = values.length - 1; j >= 0; j--) { + ObHTableSecondaryPartUtil.AssertKeyValue(key, columns[i], ts[j], values[j], cells[idx]); + idx++; + } + } + } + + // 3. get specify time range + { + Get get = new Get(key.getBytes()); + get.addFamily(family.getBytes()); + get.setTimeStamp(ts[1]); + Result r = hTable.get(get); + Cell cells[] = r.rawCells(); + assertEquals(columns.length, cells.length); + sortCells(cells); + for (int i = 0; i < columns.length; i++) { + AssertKeyValue(key, columns[i], values[1],cells[i]); + } + } + + hTable.close(); + } + + @Test + public void testGet() throws Throwable { + FOR_EACH(tableNames, OHTableTimeSeriesGetTest::testGetImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java new file mode 100644 index 00000000..41ac2380 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java @@ -0,0 +1,227 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.*; +import org.junit.*; + +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TIMESERIES_TABLES; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY; +import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; + +public class OHTableTimeSeriesScanTest { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap<>(); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : TIMESERIES_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + public static void testScanImpl(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + + // 0. prepare data + // putKey1 putColumn1 putValue1,ts + // putKey1 putColumn1 putValue2,ts+1 + // putKey1 putColumn2 putValue1,ts + // putKey1 putColumn2 putValue2,ts+1 + // ... + String family = getColumnFamilyName(tableName); + long ts = System.currentTimeMillis(); + + String keys[] = {"putKey1", "putKey2", "putKey3"}; + String keys1[] = {"putKey1", "putKey2"}; + String keys2[] = {"putKey3"}; + String endKey = "putKey4"; + String columns[] = {"putColumn1", "putColumn2"}; + String values[] = {"putValue1", "putValue2"}; + long tss[] = {ts, ts + 1}; + + for (String key : keys1) { + Put put = new Put(toBytes(key)); + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + put.add(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); + } + } + hTable.put(put); + } + + for (String key : keys2) { + for (String column : columns) { + for (int i = 0; i < values.length; i++) { + Put put = new Put(toBytes(key)); + put.add(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); + hTable.put(put); + } + } + } + + // 1. scan specify column + { + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.addColumn(family.getBytes(), columns[0].getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + Cell cells[] = getCellsFromScanner(scanner).toArray(new Cell[0]); + Assert.assertEquals(keys.length*values.length, cells.length); + sortCells(cells); + int idx = 0; + for (String key : keys) { + for (int i = values.length-1; i >= 0; i--) { + AssertKeyValue(key, columns[0], tss[i], values[i], cells[idx++]); + } + } + } + + // 2. scan do not specify column + { + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + Cell cells[] = getCellsFromScanner(scanner).toArray(new Cell[0]); + Assert.assertEquals(keys.length*columns.length*values.length, cells.length); + sortCells(cells); + int idx = 0; + for (String key : keys) { + for (String column : columns) { + for (int i = values.length-1; i >= 0; i--) { + AssertKeyValue(key, column, tss[i], values[i], cells[idx++]); + } + } + } + } + + // 3. scan specify time range + { + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.setTimeStamp(tss[1]); + scan.addFamily(family.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + Cell cells[] = getCellsFromScanner(scanner).toArray(new Cell[0]); + assertEquals(keys.length * columns.length, cells.length); + sortCells(cells); + int idx = 0; + for (String key : keys) { + for (String column : columns) { + AssertKeyValue(key, column, tss[1], values[1], cells[idx++]); + } + } + } + + + // 4. scan using setStartRow/setEndRow + { + Scan scan = new Scan(); + scan.setStartRow(keys[0].getBytes()); + scan.setStopRow(endKey.getBytes()); + ResultScanner scanner = hTable.getScanner(scan); + Cell cells[] = getCellsFromScanner(scanner).toArray(new Cell[0]); + assertEquals(keys.length * columns.length * values.length, cells.length); + sortCells(cells); + int idx = 0; + for (String key : keys) { + for (String column : columns) { + for (int i = values.length-1; i >= 0; i--) { + AssertKeyValue(key, column, tss[i], values[i], cells[idx++]); + } + } + } + } + + // 5. scan using batch + { + int batchSize = 2; + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.addFamily(family.getBytes()); + scan.setBatch(batchSize); + ResultScanner scanner = hTable.getScanner(scan); + Result result = null; + int resultSize = (keys.length * columns.length * values.length) / batchSize; + for (int i = 0; i < resultSize; i++) { + result = scanner.next(); + Assert.assertEquals(2, result.size()); + } + result = scanner.next(); + Assert.assertEquals(null, result); + } + + // 7. scan using setAllowPartialResults/setAllowPartialResults + { + Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); + scan.addFamily(family.getBytes()); + scan.setMaxResultSize(10); + scan.setAllowPartialResults(true); + ResultScanner scanner = hTable.getScanner(scan); + int resultSize = keys.length * columns.length * values.length; + for (int i = 0; i < resultSize; i++) { + Result result = scanner.next(); + Assert.assertEquals(1, result.size()); + } + Result result = scanner.next(); + Assert.assertEquals(null, result); + } + + // 8. scan in reverse + { +// Scan scan = new Scan(keys[2].getBytes(), keys[0].getBytes()); +// scan.addFamily(family.getBytes()); +// scan.setReversed(true); +// ResultScanner scanner = hTable.getScanner(scan); +// List cells = getCellsFromScanner(scanner); +// +// int cellIndex = 0; +// for (int i = 1; i >= 0; i--) { +// for (String column : columns) { +// AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); +// cellIndex++; +// } +// } +// assertEquals(columns.length * 2, cells.size()); + } + } + + @Test + public void testScan() throws Throwable { + FOR_EACH(tableNames, OHTableTimeSeriesScanTest::testScanImpl); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index fc24e3f6..2bee5685 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -26,10 +26,7 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLSyntaxErrorException; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Supplier; public class ObHTableSecondaryPartUtil { @@ -354,4 +351,27 @@ public static void checkUtilTimeout(List tableNames, Supplier f } ObHTableTestUtil.Assert(tableNames, ()-> Assert.fail("Timeout while waiting for the function to return expected result")); } + + public static void sortCells(Cell[] cells) { + if (cells == null || cells.length <= 1) { return; } + + Arrays.sort(cells, new Comparator() { + @Override + public int compare(Cell c1, Cell c2) { + if (c1 == null) return 1; + if (c2 == null) return -1; + int cmpRet = Bytes.compareTo(c1.getRow(), c2.getRow()); + if (cmpRet != 0) { return cmpRet; } + + cmpRet = Bytes.compareTo(c1.getFamily(), c2.getFamily()); + if (cmpRet != 0) { return cmpRet; } + + cmpRet = Bytes.compareTo(c1.getQualifier(), c2.getQualifier()); + if (cmpRet != 0) { return cmpRet; } + + cmpRet = Long.compare(c2.getTimestamp(), c1.getTimestamp()); + return cmpRet; + } + }); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index 9d627fe7..64fad088 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -17,7 +17,6 @@ package com.alipay.oceanbase.hbase.util; - import java.util.Arrays; import java.util.EnumMap; import java.util.List; @@ -27,272 +26,250 @@ import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN; public class TableTemplateManager { - public static final long PART_NUM = 3; + public static final long PART_NUM = 3; public static final String TABLE_GROUP_PREFIX = "test_group_"; - public static final String COLUMN_FAMILY = "cf"; + public static final String COLUMN_FAMILY = "cf"; + public enum TableType { - NON_PARTITIONED_REGULAR, - NON_PARTITIONED_TIME_SERIES, - SINGLE_PARTITIONED_REGULAR, - SINGLE_PARTITIONED_TIME_SERIES, - SECONDARY_PARTITIONED_RANGE_KEY, // RANGE-KEY分区（使用K） - SECONDARY_PARTITIONED_RANGE_KEY_GEN, // RANGE-KEY分区（使用生成列） - SECONDARY_PARTITIONED_KEY_RANGE, // KEY-RANGE分区（使用K） - SECONDARY_PARTITIONED_KEY_RANGE_GEN, // KEY-RANGE分区（使用生成列） - SECONDARY_PARTITIONED_TIME_RANGE_KEY, // 时序表RANGE-KEY - SECONDARY_PARTITIONED_TIME_KEY_RANGE, // 时序表KEY-RANGE + NON_PARTITIONED_REGULAR, NON_PARTITIONED_TIME_SERIES, SINGLE_PARTITIONED_REGULAR, SINGLE_PARTITIONED_TIME_SERIES, SECONDARY_PARTITIONED_RANGE_KEY, // RANGE-KEY分区（使用K） + SECONDARY_PARTITIONED_RANGE_KEY_GEN, // RANGE-KEY分区（使用生成列） + SECONDARY_PARTITIONED_KEY_RANGE, // KEY-RANGE分区（使用K） + SECONDARY_PARTITIONED_KEY_RANGE_GEN, // KEY-RANGE分区（使用生成列） + SECONDARY_PARTITIONED_TIME_RANGE_KEY, // 时序表RANGE-KEY + SECONDARY_PARTITIONED_TIME_KEY_RANGE, // 时序表KEY-RANGE /* ------------------ CELL TTL ----------------*/ - NON_PARTITIONED_REGULAR_CELL_TTL, - SINGLE_PARTITIONED_REGULAR_CELL_TTL, - SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, // RANGE-KEY分区（使用K） - SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, // RANGE-KEY分区（使用生成列） - SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, // KEY-RANGE分区（使用K） - SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, // KEY-RANGE分区（使用生成列） + NON_PARTITIONED_REGULAR_CELL_TTL, SINGLE_PARTITIONED_REGULAR_CELL_TTL, SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, // RANGE-KEY分区（使用K） + SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, // RANGE-KEY分区（使用生成列） + SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, // KEY-RANGE分区（使用K） + SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, // KEY-RANGE分区（使用生成列） } - public static List NORMAL_TABLES = Arrays.asList(NON_PARTITIONED_REGULAR, - SINGLE_PARTITIONED_REGULAR, - SECONDARY_PARTITIONED_RANGE_KEY, - SECONDARY_PARTITIONED_RANGE_KEY_GEN, - SECONDARY_PARTITIONED_KEY_RANGE, - SECONDARY_PARTITIONED_KEY_RANGE_GEN); - - public static List CELL_TTL_TABLES = Arrays.asList(NON_PARTITIONED_REGULAR_CELL_TTL, - SINGLE_PARTITIONED_REGULAR_CELL_TTL, - SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, - SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, - SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, - SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL); - + public static List NORMAL_AND_SERIES_TABLES = Arrays + .asList( + NON_PARTITIONED_REGULAR, + NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_REGULAR, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_RANGE_KEY, + SECONDARY_PARTITIONED_RANGE_KEY_GEN, + SECONDARY_PARTITIONED_KEY_RANGE, + SECONDARY_PARTITIONED_KEY_RANGE_GEN, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + + public static List NORMAL_TABLES = Arrays + .asList( + NON_PARTITIONED_REGULAR, + SINGLE_PARTITIONED_REGULAR, + SECONDARY_PARTITIONED_RANGE_KEY, + SECONDARY_PARTITIONED_RANGE_KEY_GEN, + SECONDARY_PARTITIONED_KEY_RANGE, + SECONDARY_PARTITIONED_KEY_RANGE_GEN); + + public static List CELL_TTL_TABLES = Arrays + .asList( + NON_PARTITIONED_REGULAR_CELL_TTL, + SINGLE_PARTITIONED_REGULAR_CELL_TTL, + SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, + SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, + SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, + SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL); + + public static List TIMESERIES_TABLES = Arrays.asList(NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + + private static final Map SQL_TEMPLATES = new EnumMap( + TableType.class); + private static final Map SQL_TEMPLATES = new EnumMap<>(TableType.class); static { // 普通表非分区表模版 SQL_TEMPLATES.put(TableType.NON_PARTITIONED_REGULAR, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s"); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s"); // 时序表非分区表模版 SQL_TEMPLATES.put(TableType.NON_PARTITIONED_TIME_SERIES, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `S` bigint(20) NOT NULL,\n" + - " `V` json NOT NULL,\n" + - " PRIMARY KEY (`K`, `T`, `S`)\n" + - ") TABLEGROUP = %s"); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s"); // 普通表一级分区模板 SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); // 时序表一级分区模板 SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_TIME_SERIES, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `S` bigint(20) NOT NULL,\n" + - " `V` json NOT NULL,\n" + - " PRIMARY KEY (`K`, `T`, `S`)\n" + - ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); // 普通表RANGE-KEY分区（使用K） SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + - "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + - "(PARTITION `p0` VALUES LESS THAN (%d),\n" + - " PARTITION `p1` VALUES LESS THAN (%d),\n" + - " PARTITION `p2` VALUES LESS THAN (%d),\n" + - " PARTITION `p3` VALUES LESS THAN MAXVALUE)"); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE)"); // 合并GEN类型的注释处理 SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + - "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + - "(PARTITION `p0` VALUES LESS THAN (%d),\n" + - " PARTITION `p1` VALUES LESS THAN (%d),\n" + - " PARTITION `p2` VALUES LESS THAN (%d),\n" + - " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); // 普通表KEY-RANGE分区（使用K） SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + - "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + - "SUBPARTITION TEMPLATE (\n" + - " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); // 普通表KEY-RANGE分区（使用生成列） SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + - "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + - "SUBPARTITION TEMPLATE (\n" + - " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); // 时序表RANGE-KEY分区 SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `S` bigint(20) NOT NULL,\n" + - " `V` json NOT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + - " PRIMARY KEY (`K`, `T`, `S`)\n" + - ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + - "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + - "(PARTITION `p0` VALUES LESS THAN (%d),\n" + - " PARTITION `p1` VALUES LESS THAN (%d),\n" + - " PARTITION `p2` VALUES LESS THAN (%d),\n" + - " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); // 时序表KEY-RANGE分区 SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_TIME_KEY_RANGE, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `S` bigint(20) NOT NULL,\n" + - " `V` json NOT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + - " PRIMARY KEY (`K`, `T`, `S`)\n" + - ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + - "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + - "SUBPARTITION TEMPLATE (\n" + - " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE)"); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE)"); /* ------------------ CELL TTL ----------------*/ SQL_TEMPLATES.put(TableType.NON_PARTITIONED_REGULAR_CELL_TTL, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " `TTL` bigint(20) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s"); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s"); SQL_TEMPLATES.put(TableType.SINGLE_PARTITIONED_REGULAR_CELL_TTL, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " `TTL` bigint(20) DEFAULT NULL,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`K`) PARTITIONS %d "); SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_CELL_TTL, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " `TTL` bigint(20) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + - "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + - "(PARTITION `p0` VALUES LESS THAN (%d),\n" + - " PARTITION `p1` VALUES LESS THAN (%d),\n" + - " PARTITION `p2` VALUES LESS THAN (%d),\n" + - " PARTITION `p3` VALUES LESS THAN MAXVALUE)"); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE)"); SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " `TTL` bigint(20) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + - "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + - "(PARTITION `p0` VALUES LESS THAN (%d),\n" + - " PARTITION `p1` VALUES LESS THAN (%d),\n" + - " PARTITION `p2` VALUES LESS THAN (%d),\n" + - " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY RANGE COLUMNS(`G`) \n" + + "SUBPARTITION BY KEY(`%s`) SUBPARTITIONS %d \n" + + "(PARTITION `p0` VALUES LESS THAN (%d),\n" + + " PARTITION `p1` VALUES LESS THAN (%d),\n" + + " PARTITION `p2` VALUES LESS THAN (%d),\n" + + " PARTITION `p3` VALUES LESS THAN MAXVALUE) "); SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " `TTL` bigint(20) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + - "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + - "SUBPARTITION TEMPLATE (\n" + - " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); SQL_TEMPLATES.put(TableType.SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, - "CREATE TABLE IF NOT EXISTS `%s` (\n" + - " `K` varbinary(1024) NOT NULL,\n" + - " `Q` varbinary(256) NOT NULL,\n" + - " `T` bigint(20) NOT NULL,\n" + - " `V` varbinary(1024) DEFAULT NULL,\n" + - " `TTL` bigint(20) DEFAULT NULL,\n" + - " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + - " PRIMARY KEY (`K`, `Q`, `T`)\n" + - ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + - "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + - "SUBPARTITION TEMPLATE (\n" + - " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + - " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + "CREATE TABLE IF NOT EXISTS `%s` (\n" + " `K` varbinary(1024) NOT NULL,\n" + + " `Q` varbinary(256) NOT NULL,\n" + " `T` bigint(20) NOT NULL,\n" + + " `V` varbinary(1024) DEFAULT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " `G` bigint(20) GENERATED ALWAYS AS (ABS(`T`))%s,\n" + + " PRIMARY KEY (`K`, `Q`, `T`)\n" + + ") TABLEGROUP = %s PARTITION BY KEY(`%s`) PARTITIONS %d \n" + + "SUBPARTITION BY RANGE COLUMNS(`G`) \n" + "SUBPARTITION TEMPLATE (\n" + + " SUBPARTITION `p0` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); } public static String getCreateTableSQL(TableType type, String tableName, TimeGenerator.TimeRange timeRange) { + System.out.println(tableName); String template = SQL_TEMPLATES.get(type); Object[] params; String tableGroup = extractTableGroup(tableName); @@ -301,12 +278,12 @@ public static String getCreateTableSQL(TableType type, String tableName, case NON_PARTITIONED_REGULAR: case NON_PARTITIONED_TIME_SERIES: case NON_PARTITIONED_REGULAR_CELL_TTL: - params = new Object[]{tableName, tableGroup}; + params = new Object[] { tableName, tableGroup }; break; case SINGLE_PARTITIONED_REGULAR: - case SINGLE_PARTITIONED_TIME_SERIES: // 合并相同处理逻辑 + case SINGLE_PARTITIONED_TIME_SERIES: // 合并相同处理逻辑 case SINGLE_PARTITIONED_REGULAR_CELL_TTL: - params = new Object[]{tableName, tableGroup, PART_NUM}; + params = new Object[] { tableName, tableGroup, PART_NUM }; break; case SECONDARY_PARTITIONED_RANGE_KEY: case SECONDARY_PARTITIONED_RANGE_KEY_GEN: @@ -317,29 +294,15 @@ public static String getCreateTableSQL(TableType type, String tableName, case SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL: case SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL: boolean isGen = type.name().contains("GEN"); - params = new Object[]{ - tableName, - getGeneratedColumn(type), - tableGroup, - isGen ? "K_PREFIX" : "K", - PART_NUM, - timeRange.lowerBound1(), - timeRange.lowerBound1() + 86400000, - timeRange.lowerBound1() + 172800000 - }; + params = new Object[] { tableName, getGeneratedColumn(type), tableGroup, + isGen ? "K_PREFIX" : "K", PART_NUM, timeRange.lowerBound1(), + timeRange.lowerBound1() + 86400000, timeRange.lowerBound1() + 172800000 }; break; case SECONDARY_PARTITIONED_TIME_RANGE_KEY: - case SECONDARY_PARTITIONED_TIME_KEY_RANGE: // 合并时序表处理 - params = new Object[]{ - tableName, - "", - tableGroup, - "K", - PART_NUM, - timeRange.lowerBound1(), - timeRange.lowerBound1() + 86400000, - timeRange.lowerBound1() + 172800000 - }; + case SECONDARY_PARTITIONED_TIME_KEY_RANGE: // 合并时序表处理 + params = new Object[] { tableName, "", tableGroup, "K", PART_NUM, + timeRange.lowerBound1(), timeRange.lowerBound1() + 86400000, + timeRange.lowerBound1() + 172800000 }; break; default: throw new IllegalArgumentException("Unsupported table type"); @@ -351,8 +314,7 @@ public static String getCreateTableSQL(TableType type, String tableName, private static String getGeneratedColumn(TableType type) { StringBuilder sb = new StringBuilder(); boolean needsKPrefix = type.name().startsWith("SECONDARY_PARTITIONED") - && !type.name().contains("TIME") - && type.name().contains("GEN"); + && !type.name().contains("TIME") && type.name().contains("GEN"); if (needsKPrefix) { sb.append(",\n K_PREFIX varbinary(1024) GENERATED ALWAYS AS (substring(`K`, 1, 4))"); @@ -362,20 +324,21 @@ private static String getGeneratedColumn(TableType type) { private static String getPartitionStrategy(TableType type) { if (type.name().contains("RANGE_KEY")) { - return type.name().contains("GEN") - ? "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS " + PART_NUM - : "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS "+ PART_NUM; + return type.name().contains("GEN") ? "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K_PREFIX`) SUBPARTITIONS " + + PART_NUM + : "RANGE COLUMNS(`G`) SUBPARTITION BY KEY(`K`) SUBPARTITIONS " + PART_NUM; } if (type.name().contains("KEY_RANGE")) { - return type.name().contains("GEN") - ? "KEY(`K_PREFIX`) PARTITIONS "+ PART_NUM +" SUBPARTITION BY RANGE COLUMNS(`G`)" - : "KEY(`K`) PARTITIONS "+ PART_NUM +" SUBPARTITION BY RANGE COLUMNS(`G`)"; + return type.name().contains("GEN") ? "KEY(`K_PREFIX`) PARTITIONS " + PART_NUM + + " SUBPARTITION BY RANGE COLUMNS(`G`)" + : "KEY(`K`) PARTITIONS " + PART_NUM + " SUBPARTITION BY RANGE COLUMNS(`G`)"; } return ""; } public static String generateTableGroupSQL(String tableGroup) { - return String.format("CREATE TABLEGROUP IF NOT EXISTS %s SHARDING = 'ADAPTIVE'", tableGroup); + return String + .format("CREATE TABLEGROUP IF NOT EXISTS %s SHARDING = 'ADAPTIVE'", tableGroup); } public static String getTableGroupName(TableTemplateManager.TableType type, boolean multiCf) { @@ -383,9 +346,8 @@ public static String getTableGroupName(TableTemplateManager.TableType type, bool } public static String generateTableName(String tableGroup, boolean multiCf, int cfIndex) { - return String.format("%s$%s", - tableGroup, - multiCf ? COLUMN_FAMILY + cfIndex : COLUMN_FAMILY); + return String + .format("%s$%s", tableGroup, multiCf ? COLUMN_FAMILY + cfIndex : COLUMN_FAMILY); } public static String extractTableGroup(String tableName) { From 566c9a97f5454d014037c9b3daa62baad671418e Mon Sep 17 00:00:00 2001 From: shen Date: Wed, 26 Mar 2025 18:39:02 +0800 Subject: [PATCH 31/40] [Test] improve test cases stablility (cherry picked from commit f5d91c6d11c1fed9581f2517f8c3971901d1dd5b) --- .../hbase/secondary/OHTableSecondaryPartAppendTest.java | 2 -- .../hbase/secondary/OHTableSecondaryPartBatchGetTest.java | 1 - .../hbase/secondary/OHTableSecondaryPartBatchTest.java | 1 - .../hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java | 2 -- .../hbase/secondary/OHTableSecondaryPartDeleteTest.java | 1 - .../oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java | 1 - .../hbase/secondary/OHTableSecondaryPartIncrementTest.java | 2 -- .../oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java | 1 - .../oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java | 1 - .../oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java | 1 - .../oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java | 1 - .../com/alipay/oceanbase/hbase/util/TableTemplateManager.java | 2 -- 12 files changed, 16 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java index 122b4880..475f1c81 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java @@ -55,7 +55,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); -// dropTables(tableNames, group2tableNames); } @Before @@ -270,6 +269,5 @@ public void testAppendMultiCF() throws Throwable { public void testAppendSeires() throws Throwable { createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, null, true); FOR_EACH(series_tables, OHTableSecondaryPartAppendTest::testAppendSeires); - dropTables(series_tables, null); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java index 77d573ec..2240c0c8 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java @@ -50,7 +50,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java index bdcccf4a..c8e3b04c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java @@ -54,7 +54,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java index f36898c1..faa5b55c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java @@ -57,7 +57,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before @@ -441,6 +440,5 @@ public void testCheckAndMutateSeires() throws Throwable { Map> seriesTablesMultiCF = new LinkedHashMap>(); createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, seriesTables, seriesTablesMultiCF, true); FOR_EACH(seriesTables, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateSeires); - dropTables(seriesTables, seriesTablesMultiCF); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java index 1ecc3554..226d4fa0 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java @@ -50,7 +50,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java index b6eb6eb8..edda1e18 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -57,7 +57,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java index 04b65f02..2ff68b04 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java @@ -54,7 +54,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before @@ -262,6 +261,5 @@ public void testIncrementSeires() throws Throwable { List series_tables = new LinkedList(); createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, null, true); FOR_EACH(series_tables, OHTableSecondaryPartIncrementTest::testIncrementSeires); - dropTables(series_tables, null); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java index a070e6a1..36247fe9 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -50,7 +50,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java index 257793ec..d248ccce 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java @@ -49,7 +49,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java index b117d630..fc025e8b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java @@ -55,7 +55,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java index 41ac2380..4ef33070 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java @@ -48,7 +48,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index 64fad088..059a29d3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -84,8 +84,6 @@ public enum TableType { private static final Map SQL_TEMPLATES = new EnumMap( TableType.class); - private static final Map SQL_TEMPLATES = new EnumMap<>(TableType.class); - static { // 普通表非分区表模版 SQL_TEMPLATES.put(TableType.NON_PARTITIONED_REGULAR, From 3336857d8c2405b0c5f7bc1a4db57022d12319af Mon Sep 17 00:00:00 2001 From: stuBirdFly <1065492934@qq.com> Date: Wed, 26 Mar 2025 19:04:00 +0800 Subject: [PATCH 32/40] add hbase abnormal test case (cherry picked from commit 684d407560635b6ee33bcd5a97585bc51ecd0b44) --- .../OHTableSecondaryPartAbnormal.java | 229 ++++++++++++++++++ .../OHTableSecondaryPartAppendTest.java | 24 +- .../OHTableSecondaryPartBatchGetTest.java | 2 +- .../OHTableSecondaryPartCellTTLTest.java | 7 +- ...HTableSecondaryPartCheckAndMutateTest.java | 10 +- .../OHTableSecondaryPartIncrementTest.java | 10 +- .../OHTableSecondaryPartTTLTest.java | 2 +- .../hbase/util/ObHTableSecondaryPartUtil.java | 1 - .../hbase/util/TableTemplateManager.java | 18 ++ 9 files changed, 275 insertions(+), 28 deletions(-) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAbnormal.java diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAbnormal.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAbnormal.java new file mode 100644 index 00000000..aaee798f --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAbnormal.java @@ -0,0 +1,229 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2025 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.secondary; + +import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; +import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; + +import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; +import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.SERIES_TABLES; +import static com.alipay.oceanbase.hbase.util.TableTemplateManager.TableType.*; +import static org.junit.Assert.*; + +public class OHTableSecondaryPartAbnormal { + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); + private static byte[] ROW = Bytes.toBytes("testRow"); + private static byte[] QUALIFIER = Bytes.toBytes("testQualifier"); + private static byte[] VALUE_2 = Bytes.toBytes("abcd"); + + @BeforeClass + public static void before() throws Exception { + openDistributedExecute(); + for (TableTemplateManager.TableType type : SERIES_TABLES) { + createTables(type, tableNames, group2tableNames, true); + } + } + + @AfterClass + public static void finish() throws Exception { + closeDistributedExecute(); + // dropTables(tableNames, group2tableNames); + } + + @Before + public void prepareCase() throws Exception { + truncateTables(tableNames, group2tableNames); + } + + private static void testSeriesLimit(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + Put put2 = new Put(ROW); + put2.add(FAMILY, QUALIFIER, VALUE_2); + hTable.put(put2); + Get get = new Get(ROW); + get.addFamily(FAMILY); + get.setFilter(new PageFilter(10)); + try { + hTable.get(get); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with filter query not supported")); + } + get = new Get(ROW); + get.addFamily(FAMILY); + get.setCheckExistenceOnly(true); + try { + hTable.get(get); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with check existence only query not supported")); + } + get = new Get(ROW); + get.addFamily(FAMILY); + get.setClosestRowBefore(true); + try { + hTable.get(get); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with reverse query not supported")); + } + Scan scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setReversed(true); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with reverse query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setCaching(1); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with caching query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setBatch(1); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with batch query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setAllowPartialResults(true); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with allow partial results query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setMaxResultsPerColumnFamily(1); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e + .getCause() + .getMessage() + .contains( + "timeseries hbase table with max results per column family query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setRowOffsetPerColumnFamily(1); + try { + hTable.getScanner(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with row offset query not supported")); + } + scan = new Scan(ROW); + scan.addFamily(FAMILY); + ResultScanner scanner = hTable.getScanner(scan); + assertEquals(1, scanner.next().size()); + hTable.close(); + } + + private static void testSeriesWithCellTTL(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + Put put2 = new Put(ROW); + put2.add(FAMILY, QUALIFIER, VALUE_2); + try { + hTable.put(put2); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("series table not support cell ttl not supported")); + } + hTable.close(); + } + + private static void testSecondaryPartReverseScan(String tableName) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); + Put put2 = new Put(ROW); + put2.add(FAMILY, QUALIFIER, VALUE_2); + hTable.put(put2); + Scan scan = new Scan(ROW); + scan.addFamily(FAMILY); + scan.setReversed(true); + try { + hTable.getScanners(scan); + fail("unexpected, should failed before"); + } catch (IOException e) { + assertTrue(e.getCause().getMessage() + .contains("secondary partitioned hbase table with reverse query not supported")); + } + hTable.close(); + } + + @Test + public void testSeriesLimit() throws Throwable { + FOR_EACH(tableNames, com.alipay.oceanbase.hbase.secondary.OHTableSecondaryPartAbnormal::testSeriesLimit); + } + + @Test + public void testSeriesWithCellTTL() throws Throwable { + List tmpTable = new ArrayList<>(); + createTables(NON_PARTITIONED_TIME_CELL_TTL, tmpTable, null, true); + FOR_EACH(tmpTable, OHTableSecondaryPartAbnormal::testSeriesWithCellTTL); + dropTables(tmpTable, null); + } + + @Test + public void testSecondaryPartReverseScan() throws Throwable { + List tmpTable = new ArrayList<>(); + createTables(SECONDARY_PARTITIONED_KEY_RANGE_GEN, tmpTable, null, true); + FOR_EACH(tmpTable, OHTableSecondaryPartAbnormal::testSecondaryPartReverseScan); + dropTables(tmpTable, null); + } +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java index 475f1c81..1a2e2863 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java @@ -41,7 +41,6 @@ public class OHTableSecondaryPartAppendTest { private static List tableNames = new LinkedList(); - private static List series_tables = new LinkedList(); private static Map> group2tableNames = new LinkedHashMap>(); @BeforeClass @@ -55,6 +54,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); + dropTables(tableNames, group2tableNames); } @Before @@ -74,8 +74,8 @@ private static void testAppend(String tableName) throws Exception { byte[] ROW = "appendKey".getBytes(); byte[] v1 = Bytes.toBytes("42"); byte[] v2 = Bytes.toBytes("23"); - byte[][] QUALIFIERS = new byte[][]{Bytes.toBytes("b"), Bytes.toBytes("a"), - Bytes.toBytes("c")}; + byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c") }; Append a = new Append(ROW); a.add(FAMILY, QUALIFIERS[0], v1); a.add(FAMILY, QUALIFIERS[1], v2); @@ -91,8 +91,8 @@ private static void testAppend(String tableName) throws Exception { assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1]))); // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2]))); - assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(), - r.getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp()); + assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(), r + .getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp()); Get get = new Get(ROW); get.setMaxVersions(10); @@ -102,11 +102,11 @@ private static void testAppend(String tableName) throws Exception { assertEquals(2, result.getColumnCells(FAMILY, QUALIFIERS[1]).size()); assertEquals(1, result.getColumnCells(FAMILY, QUALIFIERS[2]).size()); assertEquals( - 0, - Bytes.compareTo(Bytes.add(v1, v2), result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0) - .getValue())); + 0, + Bytes.compareTo(Bytes.add(v1, v2), result.getColumnCells(FAMILY, QUALIFIERS[0]) + .get(0).getValue())); assertEquals(0, - Bytes.compareTo(v2, result.getColumnCells(FAMILY, QUALIFIERS[2]).get(0).getValue())); + Bytes.compareTo(v2, result.getColumnCells(FAMILY, QUALIFIERS[2]).get(0).getValue())); } finally { hTable.close(); } @@ -119,8 +119,8 @@ private static void testAppendBorder(String tableName) throws Exception { byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); byte[] ROW = "appendKey".getBytes(); byte[] v1 = Bytes.toBytes("ab"); - byte[][] QUALIFIERS = new byte[][]{Bytes.toBytes("b"), Bytes.toBytes("a"), - Bytes.toBytes("c")}; + byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c") }; Put put = new Put(ROW); put.addColumn(FAMILY, QUALIFIERS[1], v1); hTable.put(put); @@ -267,7 +267,9 @@ public void testAppendMultiCF() throws Throwable { @Test public void testAppendSeires() throws Throwable { + List series_tables = new LinkedList(); createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, null, true); FOR_EACH(series_tables, OHTableSecondaryPartAppendTest::testAppendSeires); + dropTables(series_tables, null); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java index 2240c0c8..f7fc441d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java @@ -95,7 +95,7 @@ public static void testBatchGetImpl(String tableName) throws Exception { } hTable.close(); } - + @Test public void testBatchGet() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartBatchGetTest::testBatchGetImpl); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java index 3a48d867..c674878c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java @@ -477,10 +477,9 @@ public void testCellTTLWithRowkeyTTL() throws Throwable { @Test public void testCellTTLWithNonTTLTable() throws Throwable { - List NonTTLTable = new LinkedList(); - Map> NonTTLTableMultiCF = new LinkedHashMap<>(); - createTables(TableTemplateManager.TableType.NON_PARTITIONED_REGULAR, NonTTLTable, NonTTLTableMultiCF, true); + List NonTTLTable = new LinkedList(); + createTables(TableTemplateManager.TableType.NON_PARTITIONED_REGULAR, NonTTLTable, null, true); FOR_EACH(NonTTLTable, OHTableSecondaryPartCellTTLTest::testCellTTLWithNonTTLTable); - dropTables(NonTTLTable, NonTTLTableMultiCF); + dropTables(NonTTLTable, null); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java index faa5b55c..d2f6d3a3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java @@ -33,7 +33,6 @@ import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; -import static com.alipay.oceanbase.hbase.util.TableTemplateManager.COLUMN_FAMILY; import static com.alipay.oceanbase.hbase.util.TableTemplateManager.NORMAL_TABLES; import static org.junit.Assert.*; @@ -57,6 +56,7 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); + dropTables(tableNames, group2tableNames); } @Before @@ -436,9 +436,9 @@ public void testCheckAndMutateMultiCF() throws Throwable { @Test public void testCheckAndMutateSeires() throws Throwable { - List seriesTables = new LinkedList(); - Map> seriesTablesMultiCF = new LinkedHashMap>(); - createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, seriesTables, seriesTablesMultiCF, true); - FOR_EACH(seriesTables, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateSeires); + List series_tables = new LinkedList(); + createTables(TableTemplateManager.TableType.SECONDARY_PARTITIONED_TIME_RANGE_KEY, series_tables, null, true); + FOR_EACH(series_tables, OHTableSecondaryPartCheckAndMutateTest::testCheckAndMutateSeires); + dropTables(series_tables, null); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java index 2ff68b04..4204ddf7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java @@ -105,8 +105,8 @@ private static void testIncBorder(String tableName) throws Exception { byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); byte[] ROW = "incKey".getBytes(); byte[] v1 = Bytes.toBytes("ab"); - byte[][] QUALIFIERS = new byte[][]{Bytes.toBytes("b"), Bytes.toBytes("a"), - Bytes.toBytes("c")}; + byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("b"), Bytes.toBytes("a"), + Bytes.toBytes("c") }; Put put = new Put(ROW); put.addColumn(FAMILY, QUALIFIERS[1], v1); hTable.put(put); @@ -140,11 +140,11 @@ private static void testIncBorder(String tableName) throws Exception { result = hTable.get(get); assertEquals(3, result.size()); assertEquals(4L, - Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(0).getValue())); + Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(0).getValue())); assertEquals(2L, - Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(1).getValue())); + Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(1).getValue())); assertEquals(2L, - Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0).getValue())); + Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0).getValue())); } finally { hTable.close(); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java index 08d600df..4950ca89 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java @@ -373,7 +373,7 @@ public void testRowkeyTTL() throws Exception { testRowkeyTTL(tableNames, true, false); testRowkeyTTL(tableNames, false, false); // TODO: open the test after reverse scan is ok -// testRowkeyTTL(tableNames, true, true); + // testRowkeyTTL(tableNames, true, true); } @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index 2bee5685..e7195e55 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -340,7 +340,6 @@ public static List getCellsFromScanner(ResultScanner scanner) { return cells; } - public static void checkUtilTimeout(List tableNames, Supplier function, long timeout, long interval) throws Exception { long startTime = System.currentTimeMillis(); while (System.currentTimeMillis() - startTime < timeout) { diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index 059a29d3..a0d85231 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -43,6 +43,7 @@ public enum TableType { SECONDARY_PARTITIONED_RANGE_KEY_GEN_CELL_TTL, // RANGE-KEY分区（使用生成列） SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, // KEY-RANGE分区（使用K） SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL, // KEY-RANGE分区（使用生成列） + NON_PARTITIONED_TIME_CELL_TTL, // 时序表带CELL TTL列 } public static List NORMAL_AND_SERIES_TABLES = Arrays @@ -58,6 +59,13 @@ public enum TableType { SECONDARY_PARTITIONED_TIME_RANGE_KEY, SECONDARY_PARTITIONED_TIME_KEY_RANGE); + public static List SERIES_TABLES = Arrays + .asList( + NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); + public static List NORMAL_TABLES = Arrays .asList( NON_PARTITIONED_REGULAR, @@ -263,6 +271,15 @@ public enum TableType { + " SUBPARTITION `p1` VALUES LESS THAN (%d),\n" + " SUBPARTITION `p2` VALUES LESS THAN (%d),\n" + " SUBPARTITION `p3` VALUES LESS THAN MAXVALUE) "); + + SQL_TEMPLATES.put(NON_PARTITIONED_TIME_CELL_TTL, "CREATE TABLE IF NOT EXISTS `%s` (\n" + + " `K` varbinary(1024) NOT NULL,\n" + + " `T` bigint(20) NOT NULL,\n" + + " `S` bigint(20) NOT NULL,\n" + + " `V` json NOT NULL,\n" + + " `TTL` bigint(20) DEFAULT NULL,\n" + + " PRIMARY KEY (`K`, `T`, `S`)\n" + + ") TABLEGROUP = %s"); } public static String getCreateTableSQL(TableType type, String tableName, @@ -276,6 +293,7 @@ public static String getCreateTableSQL(TableType type, String tableName, case NON_PARTITIONED_REGULAR: case NON_PARTITIONED_TIME_SERIES: case NON_PARTITIONED_REGULAR_CELL_TTL: + case NON_PARTITIONED_TIME_CELL_TTL: params = new Object[] { tableName, tableGroup }; break; case SINGLE_PARTITIONED_REGULAR: From 855211cec85f524e7ade14b9b0dfb9040f3c71f2 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Thu, 27 Mar 2025 14:43:55 +0800 Subject: [PATCH 33/40] revert table client manager (cherry picked from commit 7230d783aaf1335351b2f4f03241a91c8b753f50) --- .../hbase/util/ObTableClientManager.java | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java index 6f95aefe..c5e77d29 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java @@ -22,7 +22,6 @@ import com.google.common.base.Objects; import org.apache.hadoop.classification.InterfaceAudience; - import java.io.IOException; import java.util.Map; import java.util.Properties; @@ -37,7 +36,7 @@ public class ObTableClientManager { public static final ConcurrentHashMap OB_TABLE_CLIENT_LOCK = new ConcurrentHashMap(); - public static final Map OB_TABLE_CLIENT_INSTANCE = new ConcurrentHashMap<>(); + public static final Map OB_TABLE_CLIENT_INSTANCE = new ConcurrentHashMap(); public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration connectionConfig) throws IllegalArgumentException, @@ -58,23 +57,18 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c } else { checkArgument(isNotBlank(connectionConfig.getParamUrl()), HBASE_OCEANBASE_PARAM_URL + " is blank"); - obTableClientKey = generateObTableClientKey(connectionConfig); - } - return getOrCreateObTableClient(obTableClientKey, connectionConfig.getRpcConnectTimeout()); - } - - public static ObTableClientKey generateObTableClientKey(OHConnectionConfiguration connectionConfig) { - ObTableClientKey obTableClientKey = new ObTableClientKey(); - String paramUrl = connectionConfig.getParamUrl(); - if (!paramUrl.contains("database")) { - paramUrl += "&database=default"; - } - obTableClientKey.setParamUrl(paramUrl); - obTableClientKey.setSysUserName(connectionConfig.getSysUsername()); - if (connectionConfig.getSysPassword() == null) { - obTableClientKey.setSysPassword(Constants.EMPTY_STRING); - } else { - obTableClientKey.setSysPassword(connectionConfig.getSysPassword()); + obTableClientKey = new ObTableClientKey(); + String paramUrl = connectionConfig.getParamUrl(); + if (!paramUrl.contains("database")) { + paramUrl += "&database=default"; + } + obTableClientKey.setParamUrl(paramUrl); + obTableClientKey.setSysUserName(connectionConfig.getSysUsername()); + if (connectionConfig.getSysPassword() == null) { + obTableClientKey.setSysPassword(Constants.EMPTY_STRING); + } else { + obTableClientKey.setSysPassword(connectionConfig.getSysPassword()); + } } checkArgument(isNotBlank(connectionConfig.getFullUsername()), HBASE_OCEANBASE_FULL_USER_NAME + " is blank"); @@ -90,7 +84,7 @@ public static ObTableClientKey generateObTableClientKey(OHConnectionConfiguratio obTableClientKey.getProperties().put(property.getKey(), property.getValue()); } - return obTableClientKey; + return getOrCreateObTableClient(obTableClientKey, connectionConfig.getRpcConnectTimeout()); } public static ObTableClient getOrCreateObTableClient(ObTableClientKey obTableClientKey, From 39d23880ac404ec30f94b7a13bb7d83d685c6a5c Mon Sep 17 00:00:00 2001 From: GroundWu <1175416256@qq.com> Date: Fri, 28 Mar 2025 16:19:09 +0800 Subject: [PATCH 34/40] add hbase batch test (cherry picked from commit 6b91a93cda3fba2bf0fe83f8f9aaaf0edf615985) --- .../OHTableSecondaryPartBatchTest.java | 540 +++++++++++++++++- .../OHTableSecondaryPartCellTTLTest.java | 4 +- .../OHTableSecondaryPartTTLTest.java | 1 - 3 files changed, 531 insertions(+), 14 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java index c8e3b04c..43cba9eb 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java @@ -15,39 +15,38 @@ * #L% */ - package com.alipay.oceanbase.hbase.secondary; import com.alipay.oceanbase.hbase.OHTableClient; +import com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.util.TableTemplateManager; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.filter.ValueFilter; import org.apache.hadoop.hbase.util.Bytes; import org.junit.*; +import java.io.IOException; import java.util.*; import static com.alipay.oceanbase.hbase.util.ObHTableSecondaryPartUtil.*; import static com.alipay.oceanbase.hbase.util.ObHTableTestUtil.FOR_EACH; import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.assertEquals; public class OHTableSecondaryPartBatchTest { - private static List tableNames = new LinkedList(); - private static Map> group2tableNames = new LinkedHashMap<>(); - + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); @BeforeClass public static void before() throws Exception { openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { - createTables(type, tableNames, group2tableNames, true); + createTables(type, tableNames, group2tableNames, true); } } @@ -430,11 +429,521 @@ public static void testBatchGetImpl(String tableName) throws Exception { hTable.close(); } + public static void testMixBatchImpl(String tableName) throws Exception { + byte[] family = getColumnFamilyName(tableName).getBytes(); + byte[][] keys = new byte[][]{"key1".getBytes(), "key2".getBytes()}; + byte[][] qualifiers = new byte[][]{"col1".getBytes(), "col2".getBytes()}; + byte[][] values = new byte[][]{"value1".getBytes(), "value2".getBytes()}; + long curTs = System.currentTimeMillis(); + long[] ts = {curTs, curTs - 1000}; + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); + hTable.init(); + List batchLsit = new LinkedList<>(); + // 0. load data + for (int i = 0; i < keys.length; i++) { + Put put = new Put(keys[i]); + for (int j = 0; j < qualifiers.length; j++) { + for (int k = 0; k < values.length; k++) { + put.add(family, qualifiers[j], ts[k], values[k]); + } + } + batchLsit.add(put); + } + hTable.batch(batchLsit); + // 1. get + put + get + batchLsit.clear(); + { + // get old result + Get get1 = new Get(keys[0]); + get1.setMaxVersions(); + get1.addFamily(family); + // get old result + Get get2 = new Get(keys[1]); + get2.setMaxVersions(1); + get2.addColumn(family, qualifiers[1]); + + // put new value + byte[] newValue = "new_value".getBytes(); + Put put1 = new Put(keys[0]); + put1.add(family, qualifiers[0], newValue); + // put new value + Put put2 = new Put(keys[1]); + put2.add(family, qualifiers[1], newValue); + // get new result + Get get3 = new Get(keys[0]); + get3.setMaxVersions(1); + get3.addColumn(family, qualifiers[0]); + + // get new result + Get get4 = new Get(keys[1]); + get4.setMaxVersions(1); + get4.addColumn(family, qualifiers[1]); + + // execute + batchLsit.addAll(Arrays.asList(get1 ,get2, put1, put2, get3, get4)); + Object[] results = hTable.batch(batchLsit); + // verify result + Assert.assertEquals(6, results.length); + // get1 + Result get1Result = (Result) results[0]; + Assert.assertEquals(4, get1Result.raw().length); + List k0cq0 = get1Result.getColumnCells(family, qualifiers[0]); + Assert.assertEquals(2, k0cq0.size()); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[0]), ts[0], Bytes.toString(values[0]), k0cq0.get(0)); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[0]), ts[1], Bytes.toString(values[1]), k0cq0.get(1)); + List k0cq1 = get1Result.getColumnCells(family, qualifiers[1]); + Assert.assertEquals(2, k0cq1.size()); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[1]), ts[0], Bytes.toString(values[0]), k0cq1.get(0)); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[1]), ts[1], Bytes.toString(values[1]), k0cq1.get(1)); + // get2 + Result get2Result = (Result) results[1]; + Assert.assertEquals(1, get2Result.raw().length); + List k1cq1 = get2Result.getColumnCells(family, qualifiers[1]); + Assert.assertEquals(1, k1cq1.size()); + AssertKeyValue(Bytes.toString(keys[1]), Bytes.toString(qualifiers[1]), ts[0], Bytes.toString(values[0]), k1cq1.get(0)); + // get3 + Result get3Result = (Result) results[4]; + Assert.assertEquals(1, get3Result.raw().length); + k0cq0 = get3Result.getColumnCells(family, qualifiers[0]); + Assert.assertEquals(1, k0cq0.size()); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[0]), Bytes.toString(newValue), k0cq0.get(0)); + // get4 + Result get4Result = (Result) results[5]; + Assert.assertEquals(1, get4Result.raw().length); + k1cq1 = get4Result.getColumnCells(family, qualifiers[1]); + Assert.assertEquals(1, k1cq1.size()); + AssertKeyValue(Bytes.toString(keys[1]), Bytes.toString(qualifiers[1]), Bytes.toString(newValue), k1cq1.get(0)); + } + // 2. delete + get + delete + get + batchLsit.clear(); + { + // delete all version keys[0]-qualifiers[0] + Delete delete1 = new Delete(keys[0]); + delete1.addColumns(family, qualifiers[0]); + + // get family: keys[0]-qualifier[1] may have two versions and keys[0]-qualifier[0] have been deleted + Get get1 = new Get(keys[0]); + get1.addFamily(family); + get1.setMaxVersions(); + // delete all version keys[0]-qualifier[1] + Delete delete2 = new Delete(keys[0]); + delete2.addColumns(family, qualifiers[1]); + // keys[0] may not have results + Get get2 = new Get(keys[0]); + get2.addFamily(family); + get2.setMaxVersions(); + // execute + batchLsit.addAll(Arrays.asList(delete1, get1, delete2, get2)); + Object[] results = hTable.batch(batchLsit); + // verify result + Assert.assertEquals(4, results.length); + Result get1Result = (Result) results[1]; + ObHTableTestUtil.Assert(tableName, ()->Assert.assertEquals(2, get1Result.size())); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(family), Bytes.toString(qualifiers[1]), + ts[0], Bytes.toString(values[0]), get1Result.listCells().get(0)); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(family), Bytes.toString(qualifiers[1]), + ts[1], Bytes.toString(values[1]), get1Result.listCells().get(1)); + Result get2Result = (Result) results[3]; + Assert.assertEquals(0, get2Result.size()); + } + // 3. put + delete + get + batchLsit.clear(); + { + Put put1 = new Put(keys[0]); + put1.add(family, qualifiers[0], "new_new_value".getBytes()); + put1.add(family, qualifiers[1], "new_new_value".getBytes()); + Put put2 = new Put(keys[1]); + put2.add(family, qualifiers[0], "new_new_value".getBytes()); + put2.add(family, qualifiers[1], "new_new_value".getBytes()); + Delete delete = new Delete(keys[0]); + delete.addColumns(family, qualifiers[0]); + Get get1 = new Get(keys[0]); + get1.setMaxVersions(1); + get1.addFamily(family); + + Get get2 = new Get(keys[1]); + get2.setMaxVersions(1); + get2.addFamily(family); + + batchLsit.addAll(Arrays.asList(put1, put2, delete, get1, get2)); + Object[] results = hTable.batch(batchLsit); + Assert.assertEquals(5, results.length); + Result getResult1 = (Result) results[3]; + Assert.assertEquals(1, getResult1.size()); + AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[1]), + "new_new_value", getResult1.listCells().get(0)); + Result getResult2 = (Result) results[4]; + Assert.assertEquals(2, getResult2.size()); + AssertKeyValue(Bytes.toString(keys[1]), Bytes.toString(qualifiers[0]), + "new_new_value", getResult2.listCells().get(0)); + AssertKeyValue(Bytes.toString(keys[1]), Bytes.toString(qualifiers[1]), + "new_new_value", getResult2.listCells().get(1)); + } + hTable.close(); + } public static void testMultiCFBatchGetImpl(Map.Entry> entry) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); hTable.init(); - // prepare data + String keyPrefix = "putKey_"; + String[] columns = {"putColumn1", "putColumn2", "putColumn3"}; + String[] values = {"version1_", "version2_"}; // each column have two versions + String latestValue = values[1]; + List tableNames = entry.getValue(); + long timestamp = System.currentTimeMillis(); + long[] ts = {timestamp, timestamp+1}; + long lastTs = ts[1]; + int batchSize = 10; + // 0. prepare data + List puts = new ArrayList<>(); + for (int k = 0; k < batchSize; k++) { + String key = keyPrefix+k; + Put put = new Put(toBytes(key)); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns.length; j++) { + put.add(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i]+k)); + } + } + } + puts.add(put); + } + hTable.put(puts); + // 1. get specify column + { + List gets = new ArrayList<>(); + int columnIndex = 0; + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + for (String tableName : tableNames) { + Get get = new Get(key.getBytes()); + String family = getColumnFamilyName(tableName); + get.addColumn(family.getBytes(), columns[columnIndex].getBytes()); + gets.add(get); + } + } + Result[] results = hTable.get(gets); + Assert.assertEquals(gets.size(), results.length); + for (int i = 0; i < gets.size(); i++) { + int idx = i / tableNames.size(); + String key = keyPrefix + idx; + Result res = results[i]; + Assert.assertEquals(1, res.size()); + AssertKeyValue(key, columns[columnIndex], lastTs, latestValue + idx, res.rawCells()[0]); + } + } + // 2. get do not specify column + { + List gets = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addFamily(family.getBytes()); + } + gets.add(get); + } + Result[] results = hTable.get(gets); + Assert.assertEquals(batchSize, results.length); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result result = results[i]; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int j = 0; j < columns.length; j++) { + Cell cell = result.getColumnCells(family.getBytes(), columns[j].getBytes()).get(0); + AssertKeyValue(key, columns[j], lastTs, latestValue + i, cell); + } + } + } + } + // 3. get do not specify column family + { + List gets = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + gets.add(get); + } + Result[] results = hTable.get(gets); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result result = results[i]; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int j = 0; j < columns.length; j++) { + Cell cell = result.getColumnCells(family.getBytes(), columns[j].getBytes()).get(0); + AssertKeyValue(key, columns[j], lastTs, latestValue + i, cell); + } + } + } + } + // 4. get specify multi cf and column + { + List gets = new ArrayList<>(); + int columnIndex = 0; + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addColumn(family.getBytes(), columns[columnIndex].getBytes()); + } + gets.add(get); + } + Result[] results = hTable.get(gets); + Assert.assertEquals(batchSize, results.length); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result res = results[i]; + Assert.assertEquals(tableNames.size(), res.size()); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + AssertKeyValue(key, columns[columnIndex], lastTs, latestValue + i, + res.getColumnCells(family.getBytes(), columns[columnIndex].getBytes()).get(0)); + + } + } + } + // 5. get specify multi cf and versions + { + List gets = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + get.setMaxVersions(2); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addFamily(family.getBytes()); + } + gets.add(get); + } + Result[] results = hTable.get(gets); + Assert.assertEquals(batchSize, results.length); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result res = results[i]; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int j = 0; j < columns.length; j++) { + List cells = res.getColumnCells(family.getBytes(), columns[j].getBytes()); + Assert.assertEquals(2, cells.size()); + for (int k = ts.length - 1; k >= 0; k--) { + AssertKeyValue(key, family, columns[j], ts[k], values[k] + i, cells.get(ts.length - k - 1)); + } + } + } + } + } + // 6. get specify multi cf and set timestamp + { + List gets = new ArrayList<>(); + int columnIndex = 0; + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + get.setTimeStamp(ts[1]); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addColumn(family.getBytes(), columns[columnIndex].getBytes()); + } + gets.add(get); + } + Result[] results = hTable.get(gets); + Assert.assertEquals(batchSize, results.length); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result res = results[i]; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + AssertKeyValue(key, columns[columnIndex], ts[1], latestValue + i, + res.getColumnCells(family.getBytes(), columns[columnIndex].getBytes()).get(0)); + } + } + } + // 7. get specify multi cf and filter + { + List gets = new ArrayList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Get get = new Get(key.getBytes()); + get.setMaxVersions(2); + ValueFilter valueFilter = new ValueFilter(CompareFilter.CompareOp.EQUAL, + new BinaryComparator(toBytes(values[0] + i))); + get.setFilter(valueFilter); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + get.addFamily(family.getBytes()); + } + gets.add(get); + } + Result[] results = hTable.get(gets); + Assert.assertEquals(batchSize, results.length); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + Result res = results[i]; + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int j = 0; j < columns.length; j++) { + AssertKeyValue(key, columns[j], ts[0], values[0] + i, + res.getColumnCells(family.getBytes(), columns[j].getBytes()).get(0)); + } + } + } + } + } + + public static void testMultiCFMixBatchImpl(Map.Entry> entry) throws Exception { + OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(entry.getKey())); + hTable.init(); + String keyPrefix = "Key_"; + String[] columns = {"Column1", "Column2"}; + String[] values = {"version1_", "version2_"}; // each column have two versions + String latestValue = values[1]; + long timestamp = System.currentTimeMillis(); + long[] ts = {timestamp, timestamp+1}; + long lastTs = ts[1]; + int batchSize = 10; + List tableNames = entry.getValue(); + + // 0. prepare data + { + List puts = new ArrayList<>(); + for (int k = 0; k < batchSize; k++) { + String key = keyPrefix + k; + Put put = new Put(toBytes(key)); + for (String tableName : tableNames) { + String family = getColumnFamilyName(tableName); + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < columns.length; j++) { + put.add(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i] + k)); + } + } + } + puts.add(put); + } + hTable.put(puts); + } + String[] families = getAllFamilies(tableNames); + // 1. get + put + get + { + List batchList = new LinkedList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + batchList.add(createGetOp(key, families, columns[0], ts[1])); + batchList.add(createPutOp(key, families, columns[0], ts[1], "new_value_"+i)); + batchList.add(createGetOp(key, families, columns[0], ts[1])); + } + Object[] results = hTable.batch(batchList); + Assert.assertEquals(batchSize*3, results.length); + for (int i = 0; i < batchSize; i++) { + Result get1Result = (Result) results[3*i+0]; + checkResult(keyPrefix+i, families, columns[0], ts[1], values[1]+i, get1Result); + Result get2Result = (Result) results[3*i+2]; + checkResult(keyPrefix+i, families, columns[0], ts[1], "new_value_"+i, get2Result); + } + } + + // 2. delete + get + delete + get + { + List batchList = new LinkedList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + batchList.add(createDeleteOp(key, families, columns[1], ts[0])); + batchList.add(createGetOp(key, families, columns[1], ts[0])); + batchList.add(createGetOp(key, families, columns[0], ts[1])); + batchList.add(createDeleteOp(key, families, columns[0], ts[1])); + batchList.add(createGetOp(key, families, columns[0], ts[1])); + } + Object[] results = hTable.batch(batchList); + Assert.assertEquals(batchSize*5, results.length); + for (int i = 0; i < batchSize; i++) { + Result get1Result = (Result) results[5*i+1]; + Assert.assertEquals(0, get1Result.size()); + Result get2Result = (Result) results[5*i+2]; + checkResult(keyPrefix+i, families, columns[0], ts[1], "new_value_"+i, get2Result); + Result get3Result = (Result) results[5*i+4]; + Assert.assertEquals(0, get3Result.size()); + } + } + + // 3. put + delete + get + { + List batchList = new LinkedList<>(); + for (int i = 0; i < batchSize; i++) { + String key = keyPrefix + i; + batchList.add(createPutOp(key, families, columns[0], ts[1], "new_new_value"+i)); + batchList.add(createGetOp(key, families, columns[0], ts[1])); + batchList.add(createDeleteOp(key, families, columns[0], ts[1])); + batchList.add(createGetOp(key, families, columns[0], ts[1])); + } + Object[] results = hTable.batch(batchList); + Assert.assertEquals(batchSize*4, results.length); + for (int i = 0; i < batchSize; i++) { + Result get1Result = (Result) results[4*i+1]; + checkResult(keyPrefix+i, families, columns[0], ts[1], "new_new_value"+i, get1Result); + Result get2Result = (Result) results[4*i+3]; + Assert.assertEquals(0, get2Result.size()); + } + } + hTable.close(); + } + + private static String[] getAllFamilies(List tableNames) throws Exception { + String[] families = new String[tableNames.size()]; + for (int i = 0; i < tableNames.size(); i++) { + families[i] = getColumnFamilyName(tableNames.get(i)); + } + return families; + } + + private static void checkResult(String key, String[] families, String qualifier, + long ts, String value, Result result) { + Assert.assertEquals(families.length, result.size()); + for (String family: families) { + Cell cell = result.getColumnCells(family.getBytes(), qualifier.getBytes()).get(0); + AssertKeyValue(key, family, qualifier, ts, value, cell); + } + } + + + private static Put createPutOp(String key, String[] families, String qualifier, + long ts, String value) { + if (families == null || qualifier == null || value == null) { + throw new IllegalArgumentException("input parameters is illegal"); + } + Put put = new Put(key.getBytes()); + for (int i = 0; i < families.length; i++) { + put.add(families[i].getBytes(), qualifier.getBytes(), ts, value.getBytes()); + } + return put; + } + + private static Delete createDeleteOp(String key, String[] families, String qualifier, + long ts) { + if (families == null || qualifier == null) { + throw new IllegalArgumentException("input parameters is illegal"); + } + Delete delete = new Delete(key.getBytes()); + for (int i = 0; i < families.length; i++) { + delete.addColumn(families[i].getBytes(), qualifier.getBytes(), ts); + } + return delete; + } + + private static Get createGetOp(String key, String[] families, String qualifier, + long ts) throws IOException { + if (families == null || qualifier == null) { + throw new IllegalArgumentException("input parameters is illegal"); + } + Get get = new Get(key.getBytes()); + for (int i = 0; i < families.length; i++) { + get.addColumn(families[i].getBytes(), qualifier.getBytes()); + get.setTimeStamp(ts); + } + return get; } @Test @@ -442,6 +951,18 @@ public void testBatchPut() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testBatchPutImpl); } + @Test + public void testMixBatch() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartBatchTest::testMixBatchImpl); + truncateTables(ObHTableTestUtil.getConnection(), tableNames); + } + + @Test + public void testMultiCFMixBatch() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFMixBatchImpl); + truncateTables(ObHTableTestUtil.getConnection(), group2tableNames); + } + @Test public void testMultiCFPut() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchPutImpl); @@ -456,5 +977,4 @@ public void testBatchGet() throws Throwable { public void testMultiCFGet() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartBatchTest::testMultiCFBatchGetImpl); } - } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java index c674878c..bf9b178a 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java @@ -50,7 +50,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before @@ -235,7 +234,7 @@ public static void testCellTTL(String tableName) throws Exception { assertTrue(succ); r = hTable.get(get); assertEquals(r.size(), 2); - assertEquals(22L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), + assertEquals(11L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), column1.getBytes()).get(0)))); assertEquals(33L, Bytes.toLong(CellUtil.cloneValue(r.getColumnCells(family.getBytes(), column2.getBytes()).get(0)))); @@ -480,6 +479,5 @@ public void testCellTTLWithNonTTLTable() throws Throwable { List NonTTLTable = new LinkedList(); createTables(TableTemplateManager.TableType.NON_PARTITIONED_REGULAR, NonTTLTable, null, true); FOR_EACH(NonTTLTable, OHTableSecondaryPartCellTTLTest::testCellTTLWithNonTTLTable); - dropTables(NonTTLTable, null); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java index 4950ca89..8b0c0109 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java @@ -34,7 +34,6 @@ public static void before() throws Exception { @AfterClass public static void finish() throws Exception { closeDistributedExecute(); - dropTables(tableNames, group2tableNames); } @Before From d77526a9f515b6b80095ea687d7f87e9ffe97a28 Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 31 Mar 2025 19:36:56 +0800 Subject: [PATCH 35/40] add K Q T to scan range column (cherry picked from commit 687a2cbd26feab0eb522422606ce61445882a1f9) --- src/main/java/com/alipay/oceanbase/hbase/OHTable.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 6215bb0b..bad322f6 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1916,6 +1916,7 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, obTableQuery.setIndexName("PRIMARY"); obTableQuery.sethTableFilter(filter); obTableQuery.addKeyRange(obNewRange); + obTableQuery.setScanRangeColumns("K", "Q", "T"); return obTableQuery; } @@ -1941,6 +1942,7 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { : configuration.getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE)); obTableQuery.setObKVParams(buildOBKVParams(scan)); + obTableQuery.setScanRangeColumns("K", "Q", "T"); return obTableQuery; } @@ -1957,6 +1959,7 @@ private ObTableQuery buildObTableQuery(final Get get, Collection columnQ get.getTimeRange()); } obTableQuery.setObKVParams(buildOBKVParams(get)); + obTableQuery.setScanRangeColumns("K", "Q", "T"); return obTableQuery; } From a616f2a102a80100b38507c2d0395101fef0a88b Mon Sep 17 00:00:00 2001 From: maochongxin Date: Mon, 31 Mar 2025 20:08:03 +0800 Subject: [PATCH 36/40] fix (cherry picked from commit 50af7ab8bb625d5dd0acc1df4f0527e3346931b8) --- .../java/com/alipay/oceanbase/hbase/OHTable.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index bad322f6..6d4524af 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1916,7 +1916,9 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, obTableQuery.setIndexName("PRIMARY"); obTableQuery.sethTableFilter(filter); obTableQuery.addKeyRange(obNewRange); - obTableQuery.setScanRangeColumns("K", "Q", "T"); + if (obTableClient.isOdpMode()) { + obTableQuery.setScanRangeColumns("K", "Q", "T"); + } return obTableQuery; } @@ -1942,7 +1944,9 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { : configuration.getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE)); obTableQuery.setObKVParams(buildOBKVParams(scan)); - obTableQuery.setScanRangeColumns("K", "Q", "T"); + if (obTableClient.isOdpMode()) { + obTableQuery.setScanRangeColumns("K", "Q", "T"); + } return obTableQuery; } @@ -1959,7 +1963,9 @@ private ObTableQuery buildObTableQuery(final Get get, Collection columnQ get.getTimeRange()); } obTableQuery.setObKVParams(buildOBKVParams(get)); - obTableQuery.setScanRangeColumns("K", "Q", "T"); + if (obTableClient.isOdpMode()) { + obTableQuery.setScanRangeColumns("K", "Q", "T"); + } return obTableQuery; } From cff129c3115cb303c9d9a3afb1e1ad93778234a7 Mon Sep 17 00:00:00 2001 From: shen Date: Tue, 1 Apr 2025 15:38:07 +0800 Subject: [PATCH 37/40] [Fix] hbase multi partition scan return repetitive data (cherry picked from commit ed7b5cd6ea24c84520671380f955001cdf87efad) --- src/main/java/com/alipay/oceanbase/hbase/OHTable.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 6d4524af..ffdd07e1 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1128,7 +1128,8 @@ public List call() throws IOException { List partitions = obTableClient.getPartition(phyTableName, false); for (Partition partition : partitions) { request.getObTableQueryRequest().setTableQueryPartId( - partition.getPartId()); + partition.getPartId()); + request.setAllowDistributeScan(false); clientQueryAsyncStreamResult = (ObTableClientQueryAsyncStreamResult) obTableClient .execute(request); ClientStreamScanner clientScanner = new ClientStreamScanner( From 216d65ff1ef1b9ea463b4348e3dc33a31a4a792c Mon Sep 17 00:00:00 2001 From: maochongxin Date: Tue, 1 Apr 2025 19:33:49 +0800 Subject: [PATCH 38/40] check not support cases (cherry picked from commit 826e980815ef573d25e1cde076404852d7292d10) --- .../OHTableSecondaryPartScanTest.java | 14 ++---- .../secondary/OHTableTimeSeriesScanTest.java | 50 ++++++++----------- 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java index d248ccce..34a966ad 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java @@ -226,17 +226,11 @@ public static void testScanImpl(String tableName) throws Exception { Scan scan = new Scan(keys[1].getBytes(), "putKey".getBytes()); scan.addFamily(family.getBytes()); scan.setReversed(true); - ResultScanner scanner = hTable.getScanner(scan); - List cells = getCellsFromScanner(scanner); - - int cellIndex = 0; - for (int i = 1; i >= 0; i--) { - for (String column : columns) { - AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); - cellIndex++; - } + try { + hTable.getScanner(scan); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getMessage().contains("secondary partitioned hbase table with reverse query not supported")); } - assertEquals(columns.length * 2, cells.size()); } hTable.close(); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java index 4ef33070..d0765c2f 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java @@ -151,6 +151,7 @@ public static void testScanImpl(String tableName) throws Exception { // 4. scan using setStartRow/setEndRow { Scan scan = new Scan(); + scan.addFamily(family.getBytes()); scan.setStartRow(keys[0].getBytes()); scan.setStopRow(endKey.getBytes()); ResultScanner scanner = hTable.getScanner(scan); @@ -173,15 +174,13 @@ public static void testScanImpl(String tableName) throws Exception { Scan scan = new Scan(keys[0].getBytes(), endKey.getBytes()); scan.addFamily(family.getBytes()); scan.setBatch(batchSize); - ResultScanner scanner = hTable.getScanner(scan); - Result result = null; - int resultSize = (keys.length * columns.length * values.length) / batchSize; - for (int i = 0; i < resultSize; i++) { - result = scanner.next(); - Assert.assertEquals(2, result.size()); + try { + ResultScanner scanner = hTable.getScanner(scan); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with batch query not supported")); } - result = scanner.next(); - Assert.assertEquals(null, result); + } // 7. scan using setAllowPartialResults/setAllowPartialResults @@ -190,32 +189,25 @@ public static void testScanImpl(String tableName) throws Exception { scan.addFamily(family.getBytes()); scan.setMaxResultSize(10); scan.setAllowPartialResults(true); - ResultScanner scanner = hTable.getScanner(scan); - int resultSize = keys.length * columns.length * values.length; - for (int i = 0; i < resultSize; i++) { - Result result = scanner.next(); - Assert.assertEquals(1, result.size()); + try { + ResultScanner scanner = hTable.getScanner(scan); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with allow partial results query not supported")); } - Result result = scanner.next(); - Assert.assertEquals(null, result); } // 8. scan in reverse { -// Scan scan = new Scan(keys[2].getBytes(), keys[0].getBytes()); -// scan.addFamily(family.getBytes()); -// scan.setReversed(true); -// ResultScanner scanner = hTable.getScanner(scan); -// List cells = getCellsFromScanner(scanner); -// -// int cellIndex = 0; -// for (int i = 1; i >= 0; i--) { -// for (String column : columns) { -// AssertKeyValue(keys[i], column, lastTs, latestValue, cells.get(cellIndex)); -// cellIndex++; -// } -// } -// assertEquals(columns.length * 2, cells.size()); + Scan scan = new Scan(keys[2].getBytes(), keys[0].getBytes()); + scan.addFamily(family.getBytes()); + scan.setReversed(true); + try { + ResultScanner scanner = hTable.getScanner(scan); + } catch (Exception e) { + Assert.assertTrue(e.getCause().getMessage() + .contains("timeseries hbase table with reverse query not supported")); + } } } From cc9c1ffbe7753c1225bab0694547d65d5350f74d Mon Sep 17 00:00:00 2001 From: maochongxin Date: Wed, 2 Apr 2025 11:48:02 +0800 Subject: [PATCH 39/40] always add scan range columns (cherry picked from commit dd0e7814803af2a08ffa7922b1db431d824ae3ad) --- .../java/com/alipay/oceanbase/hbase/OHTable.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index ffdd07e1..82256271 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1917,9 +1917,7 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, obTableQuery.setIndexName("PRIMARY"); obTableQuery.sethTableFilter(filter); obTableQuery.addKeyRange(obNewRange); - if (obTableClient.isOdpMode()) { - obTableQuery.setScanRangeColumns("K", "Q", "T"); - } + obTableQuery.setScanRangeColumns("K", "Q", "T"); return obTableQuery; } @@ -1945,9 +1943,7 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, final Scan scan) { : configuration.getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE)); obTableQuery.setObKVParams(buildOBKVParams(scan)); - if (obTableClient.isOdpMode()) { - obTableQuery.setScanRangeColumns("K", "Q", "T"); - } + obTableQuery.setScanRangeColumns("K", "Q", "T"); return obTableQuery; } @@ -1964,9 +1960,7 @@ private ObTableQuery buildObTableQuery(final Get get, Collection columnQ get.getTimeRange()); } obTableQuery.setObKVParams(buildOBKVParams(get)); - if (obTableClient.isOdpMode()) { - obTableQuery.setScanRangeColumns("K", "Q", "T"); - } + obTableQuery.setScanRangeColumns("K", "Q", "T"); return obTableQuery; } From 74e8ca78e21763ea6f0f0faac103bdccce351454 Mon Sep 17 00:00:00 2001 From: stuBirdFly <1065492934@qq.com> Date: Thu, 10 Apr 2025 10:00:23 +0800 Subject: [PATCH 40/40] fix merge secondary_part conflict --- .../com/alipay/oceanbase/hbase/OHTable.java | 126 ++++++++++-------- .../hbase/util/ObTableClientManager.java | 6 +- .../oceanbase/hbase/HTableTestBase.java | 54 ++++---- .../OHTableSecondaryPartAbnormal.java | 16 +-- .../OHTableSecondaryPartAppendTest.java | 17 ++- .../OHTableSecondaryPartBatchGetTest.java | 4 +- .../OHTableSecondaryPartBatchTest.java | 125 ++++++++--------- .../OHTableSecondaryPartCellTTLTest.java | 4 +- ...HTableSecondaryPartCheckAndMutateTest.java | 38 +++--- .../OHTableSecondaryPartDeleteTest.java | 94 +++++++------ .../OHTableSecondaryPartGetTest.java | 39 +++--- .../OHTableSecondaryPartIncrementTest.java | 27 ++-- .../OHTableSecondaryPartPutTest.java | 65 +++++---- .../OHTableSecondaryPartScanTest.java | 26 ++-- .../OHTableSecondaryPartTTLTest.java | 8 +- .../secondary/OHTableTimeSeriesGetTest.java | 4 +- .../secondary/OHTableTimeSeriesScanTest.java | 4 +- .../hbase/util/ObHTableSecondaryPartUtil.java | 49 ++++--- .../hbase/util/ObHTableTestUtil.java | 76 +++++------ .../hbase/util/ResultSetPrinter.java | 85 ++++++------ .../hbase/util/TableTemplateManager.java | 10 +- 21 files changed, 455 insertions(+), 422 deletions(-) diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 82256271..f6a1a4bf 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -154,7 +154,7 @@ public class OHTable implements Table { /** * the buffer of put request */ - private final ArrayList writeBuffer = new ArrayList<>(); + private final ArrayList writeBuffer = new ArrayList(); /** * when the put request reach the write buffer size the do put will * flush commits automatically @@ -1423,7 +1423,7 @@ private boolean checkAndMutation(byte[] row, byte[] family, byte[] qualifier, "mutation family is not equal check family"); } byte[] filterString = buildCheckAndMutateFilterString(family, qualifier, compareOp, value); - ObHTableFilter filter = buildObHTableFilter(filterString, null, 1, qualifier); + ObHTableFilter filter = buildObHTableFilter(filterString, timeRange, 1, qualifier); ObTableQuery obTableQuery = buildObTableQuery(filter, row, true, row, true, false, new TimeRange()); ObTableBatchOperation batch = buildObTableBatchOperation(mutations, null); @@ -2004,69 +2004,98 @@ public static ObTableBatchOperation buildObTableBatchOperation(List ro batch.setSamePropertiesNames(true); return batch; } + private QueryAndMutate buildDeleteQueryAndMutate(KeyValue kv, ObTableOperationType operationType, - boolean isTableGroup, Long TTL) { - KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType()); - com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, operationType, isTableGroup, TTL); + boolean isTableGroup, byte[] family, Long TTL) { + KeyValue.Type kvType = KeyValue.Type.codeToType(kv.getType().getCode()); + com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, operationType, + isTableGroup, family, TTL); ObNewRange range = new ObNewRange(); ObTableQuery tableQuery = new ObTableQuery(); - tableQuery.setObKVParams(buildOBKVParams((Scan)null)); + tableQuery.setObKVParams(buildOBKVParams((Scan) null)); ObHTableFilter filter = null; switch (kvType) { case Delete: if (kv.getTimestamp() == Long.MAX_VALUE) { - range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); - range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); - filter = buildObHTableFilter(null, null, 1, kv.getQualifier()); - } else { - range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(),ObObj.getMin())); - range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); - filter = buildObHTableFilter(null, new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), 1, kv.getQualifier()); + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + filter = buildObHTableFilter(null, null, 1, CellUtil.cloneQualifier(kv)); + } else { + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + filter = buildObHTableFilter(null, + new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), 1, + CellUtil.cloneQualifier(kv)); } break; case DeleteColumn: if (kv.getTimestamp() == Long.MAX_VALUE) { - range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); - range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); - filter = buildObHTableFilter(null, null, Integer.MAX_VALUE, kv.getQualifier()); + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE, + CellUtil.cloneQualifier(kv)); } else { - range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); - range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); - filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE, kv.getQualifier()); + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), + Integer.MAX_VALUE, CellUtil.cloneQualifier(kv)); } break; case DeleteFamily: if (kv.getTimestamp() == Long.MAX_VALUE) { - range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); - range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); if (!isTableGroup) { filter = buildObHTableFilter(null, null, Integer.MAX_VALUE); } else { - filter = buildObHTableFilter(null, null, Integer.MAX_VALUE, kv.getQualifier()); + filter = buildObHTableFilter(null, null, Integer.MAX_VALUE, + CellUtil.cloneQualifier(kv)); } } else { - range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); - range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); if (!isTableGroup) { - filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE); + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), + Integer.MAX_VALUE); } else { - filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), Integer.MAX_VALUE, kv.getQualifier()); + filter = buildObHTableFilter(null, new TimeRange(0, kv.getTimestamp() + 1), + Integer.MAX_VALUE, CellUtil.cloneQualifier(kv)); } } break; case DeleteFamilyVersion: if (kv.getTimestamp() == Long.MAX_VALUE) { - range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); - range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); filter = buildObHTableFilter(null, null, Integer.MAX_VALUE); } else { - range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); - range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), + ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), + ObObj.getMax())); if (!isTableGroup) { - filter = buildObHTableFilter(null, new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), Integer.MAX_VALUE); + filter = buildObHTableFilter(null, + new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), + Integer.MAX_VALUE); } else { - filter = buildObHTableFilter(null, new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), Integer.MAX_VALUE, kv.getQualifier()); + filter = buildObHTableFilter(null, + new TimeRange(kv.getTimestamp(), kv.getTimestamp() + 1), + Integer.MAX_VALUE, CellUtil.cloneQualifier(kv)); } } break; @@ -2078,7 +2107,6 @@ private QueryAndMutate buildDeleteQueryAndMutate(KeyValue kv, return new QueryAndMutate(tableQuery, tableMutation); } - private com.alipay.oceanbase.rpc.mutation.Mutation buildMutation(Cell kv, ObTableOperationType operationType, boolean isTableGroup, @@ -2212,14 +2240,14 @@ private BatchOperation buildBatchOperation(String tableName, List singleOpResultNum++; if (disExec) { KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), - KeyValue.Type.Maximum); - com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, DEL, isTableGroup, Long.MAX_VALUE); + KeyValue.Type.DeleteFamily); + com.alipay.oceanbase.rpc.mutation.Mutation tableMutation = buildMutation(kv, DEL, isTableGroup, null, Long.MAX_VALUE); ObNewRange range = new ObNewRange(); ObTableQuery tableQuery = new ObTableQuery(); ObHTableFilter filter; tableQuery.setObKVParams(buildOBKVParams((Scan) null)); - range.setStartKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMin(), ObObj.getMin())); - range.setEndKey(ObRowKey.getInstance(kv.getRow(), ObObj.getMax(), ObObj.getMax())); + range.setStartKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMin(), ObObj.getMin())); + range.setEndKey(ObRowKey.getInstance(CellUtil.cloneRow(kv), ObObj.getMax(), ObObj.getMax())); if (kv.getTimestamp() == Long.MAX_VALUE) { filter = buildObHTableFilter(null, null, Integer.MAX_VALUE); } else { @@ -2231,9 +2259,10 @@ private BatchOperation buildBatchOperation(String tableName, List tableMutation.setTable(tableName); batch.addOperation(new QueryAndMutate(tableQuery, tableMutation)); } else { - KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp(), - KeyValue.Type.Maximum); - batch.addOperation(buildMutation(kv, DEL, isTableGroup, Long.MAX_VALUE)); + KeyValue kv = new KeyValue(delete.getRow(), delete.getTimeStamp()); + batch.addOperation(com.alipay.oceanbase.rpc.mutation.Mutation.getInstance(DEL, ROW_KEY_COLUMNS, + new Object[] { CellUtil.cloneRow(kv), null, -kv.getTimestamp() }, + null, null)); } } else { for (Map.Entry> entry : delete.getFamilyCellMap().entrySet()) { @@ -2241,21 +2270,10 @@ private BatchOperation buildBatchOperation(String tableName, List List keyValueList = entry.getValue(); for (Cell kv : keyValueList) { singleOpResultNum++; - if (isTableGroup) { - KeyValue new_kv = modifyQualifier(kv, - (Bytes.toString(family) + "." + Bytes.toString(kv - .getQualifier())).getBytes()); - if (disExec) { - batch.addOperation(buildDeleteQueryAndMutate(new_kv, DEL, true, Long.MAX_VALUE)); - } else { - batch.addOperation(buildMutation(new_kv, DEL, true, Long.MAX_VALUE)); - } + if (disExec) { + batch.addOperation(buildDeleteQueryAndMutate((KeyValue) kv, DEL, false, family, Long.MAX_VALUE)); } else { - if (disExec) { - batch.addOperation(buildDeleteQueryAndMutate(kv, DEL, false, Long.MAX_VALUE)); - } else { - batch.addOperation(buildMutation(kv, DEL, false, Long.MAX_VALUE)); - } + batch.addOperation(buildMutation(kv, DEL, isTableGroup, family, Long.MAX_VALUE)); } } } diff --git a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java index c5e77d29..5db940aa 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java +++ b/src/main/java/com/alipay/oceanbase/hbase/util/ObTableClientManager.java @@ -56,7 +56,7 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c obTableClientKey.setDatabase(connectionConfig.getDatabase()); } else { checkArgument(isNotBlank(connectionConfig.getParamUrl()), HBASE_OCEANBASE_PARAM_URL - + " is blank"); + + " is blank"); obTableClientKey = new ObTableClientKey(); String paramUrl = connectionConfig.getParamUrl(); if (!paramUrl.contains("database")) { @@ -71,7 +71,7 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c } } checkArgument(isNotBlank(connectionConfig.getFullUsername()), - HBASE_OCEANBASE_FULL_USER_NAME + " is blank"); + HBASE_OCEANBASE_FULL_USER_NAME + " is blank"); obTableClientKey.setFullUserName(connectionConfig.getFullUsername()); if (connectionConfig.getPassword() == null) { @@ -83,7 +83,7 @@ public static ObTableClient getOrCreateObTableClient(OHConnectionConfiguration c for (Map.Entry property : connectionConfig.getProperties().entrySet()) { obTableClientKey.getProperties().put(property.getKey(), property.getValue()); } - + return getOrCreateObTableClient(obTableClientKey, connectionConfig.getRpcConnectTimeout()); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 6823b2be..4dd3e0f7 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -2219,7 +2219,7 @@ public void testColumnValueFilter() throws Exception { scan.addFamily(family.getBytes()); scan.setMaxVersions(10); filter = new ColumnValueFilter(Bytes.toBytes(family), Bytes.toBytes(column2), - CompareOperator.GREATER, Bytes.toBytes(value1)); + CompareOperator.LESS, Bytes.toBytes(value1)); scan.setFilter(filter); scanner = hTable.getScanner(scan); @@ -2890,7 +2890,7 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - valueFilter = new ValueFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator( + valueFilter = new ValueFilter(CompareFilter.CompareOp.LESS, new BinaryComparator( toBytes(value1))); get.setFilter(valueFilter); r = hTable.get(get); @@ -2908,7 +2908,7 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - valueFilter = new ValueFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator( + valueFilter = new ValueFilter(CompareFilter.CompareOp.LESS, new BinaryComparator( toBytes(value3))); get.setFilter(valueFilter); r = hTable.get(get); @@ -2948,7 +2948,7 @@ public void testGetFilter() throws Exception { get = new Get(toBytes(key1)); get.setMaxVersions(10); get.addFamily(toBytes(family)); - qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.GREATER, + qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.LESS, new BinaryComparator(toBytes(column1))); get.setFilter(qualifierFilter); r = hTable.get(get); @@ -3073,7 +3073,7 @@ public void testGetFilter() throws Exception { filterList = new FilterList(); filterList.addFilter(new ColumnCountGetFilter(1)); - filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.GREATER, + filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.LESS, new BinaryComparator(toBytes(column2)))); get = new Get(toBytes(key1)); get.setMaxVersions(10); @@ -3274,7 +3274,7 @@ public void testGetFilter() throws Exception { Assert.assertEquals(7, r.rawCells().length); singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes(family), - Bytes.toBytes(column2), CompareFilter.CompareOp.GREATER, new BinaryComparator( + Bytes.toBytes(column2), CompareFilter.CompareOp.LESS, new BinaryComparator( toBytes(value2))); singleColumnValueFilter.setLatestVersionOnly(false); get = new Get(toBytes(key1)); @@ -5014,16 +5014,16 @@ public void testCheckAndPut() throws IOException, InterruptedException { Assert.assertTrue(ret); ret = hTable.checkAndPut(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.GREATER, "value1".getBytes(), put); + CompareFilter.CompareOp.LESS, "value1".getBytes(), put); Assert.assertFalse(ret); ret = hTable.checkAndPut(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.GREATER_OR_EQUAL, "value1".getBytes(), put); + CompareFilter.CompareOp.LESS_OR_EQUAL, "value1".getBytes(), put); Assert.assertTrue(ret); ret = hTable.checkAndPut(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.LESS, "".getBytes(), put); + CompareFilter.CompareOp.GREATER, "".getBytes(), put); Assert.assertFalse(ret); ret = hTable.checkAndPut(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.LESS_OR_EQUAL, "".getBytes(), put); + CompareFilter.CompareOp.GREATER_OR_EQUAL, "".getBytes(), put); Assert.assertFalse(ret); get = new Get(key.getBytes()); @@ -5061,16 +5061,16 @@ public void testCheckAndPut() throws IOException, InterruptedException { }); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.GREATER, toBytes("value1")).thenPut(put); + .ifMatches(CompareOperator.LESS, toBytes("value1")).thenPut(put); Assert.assertFalse(ret); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("value1")).thenPut(put); + .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("value1")).thenPut(put); Assert.assertTrue(ret); - ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.LESS, toBytes("")) + ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.GREATER, toBytes("")) .thenPut(put); Assert.assertFalse(ret); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("")).thenPut(put); + .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("")).thenPut(put); Assert.assertFalse(ret); get = new Get(key.getBytes()); @@ -5121,22 +5121,22 @@ public void testCheckAndDelete() throws IOException { put.addColumn(family.getBytes(), column.getBytes(), "value6".getBytes()); hTable.put(put); ret = hTable.checkAndDelete(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.GREATER, "value5".getBytes(), delete); + CompareFilter.CompareOp.LESS, "value5".getBytes(), delete); Assert.assertTrue(ret); put = new Put(key.getBytes()); put.addColumn(family.getBytes(), column.getBytes(), "value5".getBytes()); hTable.put(put); ret = hTable.checkAndDelete(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.GREATER_OR_EQUAL, "value5".getBytes(), delete); + CompareFilter.CompareOp.LESS_OR_EQUAL, "value5".getBytes(), delete); Assert.assertTrue(ret); put = new Put(key.getBytes()); put.addColumn(family.getBytes(), column.getBytes(), "value1".getBytes()); hTable.put(put); ret = hTable.checkAndDelete(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.LESS, "value1".getBytes(), delete); + CompareFilter.CompareOp.GREATER, "value1".getBytes(), delete); Assert.assertFalse(ret); ret = hTable.checkAndDelete(key.getBytes(), "family1".getBytes(), column.getBytes(), - CompareFilter.CompareOp.LESS_OR_EQUAL, "value1".getBytes(), delete); + CompareFilter.CompareOp.GREATER_OR_EQUAL, "value1".getBytes(), delete); Assert.assertTrue(ret); Get get = new Get(key.getBytes()); @@ -5209,22 +5209,22 @@ public void testCheckAndDelete() throws IOException { put.addColumn(family.getBytes(), column.getBytes(), "value6".getBytes()); hTable.put(put); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.GREATER, toBytes("value5")).thenDelete(delete); + .ifMatches(CompareOperator.LESS, toBytes("value5")).thenDelete(delete); Assert.assertTrue(ret); put = new Put(key.getBytes()); put.addColumn(family.getBytes(), column.getBytes(), "value5".getBytes()); hTable.put(put); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("value5")).thenDelete(delete); + .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("value5")).thenDelete(delete); Assert.assertTrue(ret); put = new Put(key.getBytes()); put.addColumn(family.getBytes(), column.getBytes(), "value1".getBytes()); hTable.put(put); - ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.LESS, toBytes("value1")) + ret = builder.qualifier(toBytes(column)).ifMatches(CompareOperator.GREATER, toBytes("value1")) .thenDelete(delete); Assert.assertFalse(ret); ret = builder.qualifier(toBytes(column)) - .ifMatches(CompareOperator.LESS_OR_EQUAL, toBytes("value1")).thenDelete(delete); + .ifMatches(CompareOperator.GREATER_OR_EQUAL, toBytes("value1")).thenDelete(delete); Assert.assertTrue(ret); get = new Get(key.getBytes()); @@ -5349,9 +5349,9 @@ public void testCheckAndMutate() throws IOException { rowMutations.add(put1); rowMutations.add(put2); rowMutations.add(put3); - // test greater op + // test GREATER op ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), - CompareFilter.CompareOp.LESS, value1.getBytes(), rowMutations); + CompareFilter.CompareOp.GREATER, value1.getBytes(), rowMutations); Assert.assertFalse(ret); get = new Get(key.getBytes()); get.addFamily(family.getBytes()); @@ -5361,7 +5361,7 @@ public void testCheckAndMutate() throws IOException { // test less op ret = hTable.checkAndMutate(key.getBytes(), family.getBytes(), column1.getBytes(), - CompareFilter.CompareOp.LESS, value2.getBytes(), rowMutations); + CompareFilter.CompareOp.GREATER, value2.getBytes(), rowMutations); Assert.assertTrue(ret); get = new Get(key.getBytes()); get.addFamily(family.getBytes()); @@ -5487,7 +5487,7 @@ public void testCheckAndMutate() throws IOException { rowMutations.add(put1); rowMutations.add(put2); rowMutations.add(put3); - // test greater op + // test LESS op ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.LESS, toBytes(value1)) .thenMutate(rowMutations); Assert.assertFalse(ret); @@ -5498,7 +5498,7 @@ public void testCheckAndMutate() throws IOException { Assert.assertEquals(6, r.rawCells().length); // test less op - ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.LESS, toBytes(value2)) + ret = builder.qualifier(toBytes(column1)).ifMatches(CompareOperator.GREATER, toBytes(value2)) .thenMutate(rowMutations); Assert.assertTrue(ret); get = new Get(key.getBytes()); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAbnormal.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAbnormal.java index aaee798f..0bbe8bce 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAbnormal.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAbnormal.java @@ -68,7 +68,7 @@ private static void testSeriesLimit(String tableName) throws Exception { hTable.init(); byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); Put put2 = new Put(ROW); - put2.add(FAMILY, QUALIFIER, VALUE_2); + put2.addColumn(FAMILY, QUALIFIER, VALUE_2); hTable.put(put2); Get get = new Get(ROW); get.addFamily(FAMILY); @@ -90,16 +90,6 @@ private static void testSeriesLimit(String tableName) throws Exception { assertTrue(e.getCause().getMessage() .contains("timeseries hbase table with check existence only query not supported")); } - get = new Get(ROW); - get.addFamily(FAMILY); - get.setClosestRowBefore(true); - try { - hTable.get(get); - fail("unexpected, should failed before"); - } catch (IOException e) { - assertTrue(e.getCause().getMessage() - .contains("timeseries hbase table with reverse query not supported")); - } Scan scan = new Scan(ROW); scan.addFamily(FAMILY); scan.setReversed(true); @@ -175,7 +165,7 @@ private static void testSeriesWithCellTTL(String tableName) throws Exception { hTable.init(); byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); Put put2 = new Put(ROW); - put2.add(FAMILY, QUALIFIER, VALUE_2); + put2.addColumn(FAMILY, QUALIFIER, VALUE_2); try { hTable.put(put2); fail("unexpected, should failed before"); @@ -191,7 +181,7 @@ private static void testSecondaryPartReverseScan(String tableName) throws Except hTable.init(); byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); Put put2 = new Put(ROW); - put2.add(FAMILY, QUALIFIER, VALUE_2); + put2.addColumn(FAMILY, QUALIFIER, VALUE_2); hTable.put(put2); Scan scan = new Scan(ROW); scan.addFamily(FAMILY); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java index 1a2e2863..2c41fc67 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartAppendTest.java @@ -22,6 +22,7 @@ import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.util.TableTemplateManager; import com.google.common.base.Strings; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.junit.*; @@ -91,8 +92,8 @@ private static void testAppend(String tableName) throws Exception { assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1]))); // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2]))); - assertEquals(r.getColumnLatest(FAMILY, QUALIFIERS[0]).getTimestamp(), r - .getColumnLatest(FAMILY, QUALIFIERS[2]).getTimestamp()); + assertEquals(r.getColumnLatestCell(FAMILY, QUALIFIERS[0]).getTimestamp(), r + .getColumnLatestCell(FAMILY, QUALIFIERS[2]).getTimestamp()); Get get = new Get(ROW); get.setMaxVersions(10); @@ -103,10 +104,12 @@ private static void testAppend(String tableName) throws Exception { assertEquals(1, result.getColumnCells(FAMILY, QUALIFIERS[2]).size()); assertEquals( 0, - Bytes.compareTo(Bytes.add(v1, v2), result.getColumnCells(FAMILY, QUALIFIERS[0]) - .get(0).getValue())); - assertEquals(0, - Bytes.compareTo(v2, result.getColumnCells(FAMILY, QUALIFIERS[2]).get(0).getValue())); + Bytes.compareTo(Bytes.add(v1, v2), + CellUtil.cloneValue(result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0)))); + assertEquals( + 0, + Bytes.compareTo(v2, + CellUtil.cloneValue(result.getColumnCells(FAMILY, QUALIFIERS[2]).get(0)))); } finally { hTable.close(); } @@ -198,7 +201,7 @@ private static void testAppendCon(String tableName) throws Exception { get.setMaxVersions(1); get.addColumn(FAMILY, column.getBytes()); Result result = hTable.get(get); - ObHTableTestUtil.Assert(tableName, ()-> assertTrue(0 <= Bytes.compareTo(expect, result.getColumnCells(FAMILY, column.getBytes()).get(0).getValue()))); + ObHTableTestUtil.Assert(tableName, ()-> assertTrue(0 <= Bytes.compareTo(expect, CellUtil.cloneValue(result.getColumnCells(FAMILY, column.getBytes()).get(0))))); } finally { hTable.close(); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java index f7fc441d..0445b6a3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchGetTest.java @@ -67,8 +67,8 @@ public static void testBatchGetImpl(String tableName) throws Exception { String column2 = "putColumn2"; long timestamp = System.currentTimeMillis(); Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes("1")); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes("1")); hTable.put(put); List gets = new ArrayList<>(); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java index 43cba9eb..4e345d28 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartBatchTest.java @@ -46,7 +46,7 @@ public class OHTableSecondaryPartBatchTest { public static void before() throws Exception { openDistributedExecute(); for (TableTemplateManager.TableType type : TableTemplateManager.NORMAL_TABLES) { - createTables(type, tableNames, group2tableNames, true); + createTables(type, tableNames, group2tableNames, true); } } @@ -177,10 +177,10 @@ public static void testMultiCFBatchPutImpl(Map.Entry> entry Put put = new Put(toBytes("row-" + i)); for (String tableName : entry.getValue()) { String family = getColumnFamilyName(tableName); - put.add(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(value[0])); - put.add(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(value[1])); - put.add(toBytes(family), toBytes(qualifier[2]), toBytes(value[2])); - put.add(toBytes(family), toBytes(qualifier[3]), toBytes(value[3])); + put.addColumn(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(value[0])); + put.addColumn(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(value[1])); + put.addColumn(toBytes(family), toBytes(qualifier[2]), toBytes(value[2])); + put.addColumn(toBytes(family), toBytes(qualifier[3]), toBytes(value[3])); } puts.add(put); } @@ -216,10 +216,10 @@ public static void testMultiCFBatchPutImpl(Map.Entry> entry Put put = new Put(toBytes("row-" + i)); for (String tableName : entry.getValue()) { String family = getColumnFamilyName(tableName); - put.add(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(updateValue[0])); - put.add(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(updateValue[1])); - put.add(toBytes(family), toBytes(qualifier[2]), toBytes(updateValue[2])); - put.add(toBytes(family), toBytes(qualifier[3]), toBytes(updateValue[3])); + put.addColumn(toBytes(family), toBytes(qualifier[0]), timestamp, toBytes(updateValue[0])); + put.addColumn(toBytes(family), toBytes(qualifier[1]), timestamp, toBytes(updateValue[1])); + put.addColumn(toBytes(family), toBytes(qualifier[2]), toBytes(updateValue[2])); + put.addColumn(toBytes(family), toBytes(qualifier[3]), toBytes(updateValue[3])); } puts.add(put); } @@ -286,8 +286,8 @@ public static void testBatchGetImpl(String tableName) throws Exception { for (int i = 0; i < batchSize; i++) { Put put = new Put(toBytes(keyPrefix + i)); for (int j = 0; j < columns.length; j++) { - put.add(toBytes(family), toBytes(columns[j]), ts[0], toBytes(valuePrefix + "1_" +i)); - put.add(toBytes(family), toBytes(columns[j]), ts[1], toBytes(valuePrefix + "2_" +i)); + put.addColumn(toBytes(family), toBytes(columns[j]), ts[0], toBytes(valuePrefix + "1_" +i)); + put.addColumn(toBytes(family), toBytes(columns[j]), ts[1], toBytes(valuePrefix + "2_" +i)); } puts.add(put); } @@ -311,13 +311,13 @@ public static void testBatchGetImpl(String tableName) throws Exception { List cells = r.listCells(); for (int j = 0; j < r.size(); j++) { Cell cell = cells.get(j); - Assert.assertEquals(keyPrefix+i, Bytes.toString(cell.getRow())); - Assert.assertEquals(columns[index], Bytes.toString(cell.getQualifier())); + Assert.assertEquals(keyPrefix+i, Bytes.toString(CellUtil.cloneRow(cell))); + Assert.assertEquals(columns[index], Bytes.toString(CellUtil.cloneQualifier(cell))); Assert.assertEquals(ts[j], cell.getTimestamp()); if (j == 0) { - Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cell.getValue())); + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(CellUtil.cloneValue(cell))); } else { - Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cell.getValue())); + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(CellUtil.cloneValue(cell))); } } } @@ -342,9 +342,9 @@ public static void testBatchGetImpl(String tableName) throws Exception { for (int k = 0; k < cells.size(); k++) { Assert.assertEquals(ts[k], cells.get(k).getTimestamp()); if (k == 0) { - Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cells.get(k).getValue())); + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(k)))); } else { - Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(k).getValue())); + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(k)))); } } } @@ -370,9 +370,9 @@ public static void testBatchGetImpl(String tableName) throws Exception { for (int k = 0; k < cells.size(); k++) { Assert.assertEquals(ts[k], cells.get(k).getTimestamp()); if (k == 0) { - Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(cells.get(k).getValue())); + Assert.assertEquals(valuePrefix+"1_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(k)))); } else { - Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(k).getValue())); + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(k)))); } } } @@ -397,7 +397,7 @@ public static void testBatchGetImpl(String tableName) throws Exception { List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); Assert.assertEquals(1, cells.size()); Assert.assertEquals(ts[1], cells.get(0).getTimestamp()); - Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(0).getValue())); + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(0)))); } } } @@ -422,7 +422,7 @@ public static void testBatchGetImpl(String tableName) throws Exception { List cells = r.getColumnCells(family.getBytes(), toBytes(columns[j])); Assert.assertEquals(1, cells.size()); Assert.assertEquals(ts[1], cells.get(0).getTimestamp()); - Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(cells.get(0).getValue())); + Assert.assertEquals(valuePrefix+"2_"+i, Bytes.toString(CellUtil.cloneValue(cells.get(0)))); } } } @@ -438,20 +438,21 @@ public static void testMixBatchImpl(String tableName) throws Exception { long[] ts = {curTs, curTs - 1000}; OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); - List batchLsit = new LinkedList<>(); + List batchList = new LinkedList<>(); // 0. load data for (int i = 0; i < keys.length; i++) { Put put = new Put(keys[i]); for (int j = 0; j < qualifiers.length; j++) { for (int k = 0; k < values.length; k++) { - put.add(family, qualifiers[j], ts[k], values[k]); + put.addColumn(family, qualifiers[j], ts[k], values[k]); } } - batchLsit.add(put); + batchList.add(put); } - hTable.batch(batchLsit); + Result[] results = new Result[batchList.size()]; + hTable.batch(batchList, results); // 1. get + put + get - batchLsit.clear(); + batchList.clear(); { // get old result Get get1 = new Get(keys[0]); @@ -465,10 +466,10 @@ public static void testMixBatchImpl(String tableName) throws Exception { // put new value byte[] newValue = "new_value".getBytes(); Put put1 = new Put(keys[0]); - put1.add(family, qualifiers[0], newValue); + put1.addColumn(family, qualifiers[0], newValue); // put new value Put put2 = new Put(keys[1]); - put2.add(family, qualifiers[1], newValue); + put2.addColumn(family, qualifiers[1], newValue); // get new result Get get3 = new Get(keys[0]); get3.setMaxVersions(1); @@ -480,13 +481,14 @@ public static void testMixBatchImpl(String tableName) throws Exception { get4.addColumn(family, qualifiers[1]); // execute - batchLsit.addAll(Arrays.asList(get1 ,get2, put1, put2, get3, get4)); - Object[] results = hTable.batch(batchLsit); + batchList.addAll(Arrays.asList(get1 ,get2, put1, put2, get3, get4)); + results = new Result[batchList.size()]; + hTable.batch(batchList, results); // verify result Assert.assertEquals(6, results.length); // get1 Result get1Result = (Result) results[0]; - Assert.assertEquals(4, get1Result.raw().length); + Assert.assertEquals(4, get1Result.size()); List k0cq0 = get1Result.getColumnCells(family, qualifiers[0]); Assert.assertEquals(2, k0cq0.size()); AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[0]), ts[0], Bytes.toString(values[0]), k0cq0.get(0)); @@ -497,25 +499,25 @@ public static void testMixBatchImpl(String tableName) throws Exception { AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[1]), ts[1], Bytes.toString(values[1]), k0cq1.get(1)); // get2 Result get2Result = (Result) results[1]; - Assert.assertEquals(1, get2Result.raw().length); + Assert.assertEquals(1, get2Result.size()); List k1cq1 = get2Result.getColumnCells(family, qualifiers[1]); Assert.assertEquals(1, k1cq1.size()); AssertKeyValue(Bytes.toString(keys[1]), Bytes.toString(qualifiers[1]), ts[0], Bytes.toString(values[0]), k1cq1.get(0)); // get3 Result get3Result = (Result) results[4]; - Assert.assertEquals(1, get3Result.raw().length); + Assert.assertEquals(1, get3Result.size()); k0cq0 = get3Result.getColumnCells(family, qualifiers[0]); Assert.assertEquals(1, k0cq0.size()); AssertKeyValue(Bytes.toString(keys[0]), Bytes.toString(qualifiers[0]), Bytes.toString(newValue), k0cq0.get(0)); // get4 Result get4Result = (Result) results[5]; - Assert.assertEquals(1, get4Result.raw().length); + Assert.assertEquals(1, get4Result.size()); k1cq1 = get4Result.getColumnCells(family, qualifiers[1]); Assert.assertEquals(1, k1cq1.size()); AssertKeyValue(Bytes.toString(keys[1]), Bytes.toString(qualifiers[1]), Bytes.toString(newValue), k1cq1.get(0)); } // 2. delete + get + delete + get - batchLsit.clear(); + batchList.clear(); { // delete all version keys[0]-qualifiers[0] Delete delete1 = new Delete(keys[0]); @@ -533,8 +535,9 @@ public static void testMixBatchImpl(String tableName) throws Exception { get2.addFamily(family); get2.setMaxVersions(); // execute - batchLsit.addAll(Arrays.asList(delete1, get1, delete2, get2)); - Object[] results = hTable.batch(batchLsit); + batchList.addAll(Arrays.asList(delete1, get1, delete2, get2)); + results = new Result[batchList.size()]; + hTable.batch(batchList, results); // verify result Assert.assertEquals(4, results.length); Result get1Result = (Result) results[1]; @@ -547,14 +550,14 @@ public static void testMixBatchImpl(String tableName) throws Exception { Assert.assertEquals(0, get2Result.size()); } // 3. put + delete + get - batchLsit.clear(); + batchList.clear(); { Put put1 = new Put(keys[0]); - put1.add(family, qualifiers[0], "new_new_value".getBytes()); - put1.add(family, qualifiers[1], "new_new_value".getBytes()); + put1.addColumn(family, qualifiers[0], "new_new_value".getBytes()); + put1.addColumn(family, qualifiers[1], "new_new_value".getBytes()); Put put2 = new Put(keys[1]); - put2.add(family, qualifiers[0], "new_new_value".getBytes()); - put2.add(family, qualifiers[1], "new_new_value".getBytes()); + put2.addColumn(family, qualifiers[0], "new_new_value".getBytes()); + put2.addColumn(family, qualifiers[1], "new_new_value".getBytes()); Delete delete = new Delete(keys[0]); delete.addColumns(family, qualifiers[0]); Get get1 = new Get(keys[0]); @@ -565,8 +568,9 @@ public static void testMixBatchImpl(String tableName) throws Exception { get2.setMaxVersions(1); get2.addFamily(family); - batchLsit.addAll(Arrays.asList(put1, put2, delete, get1, get2)); - Object[] results = hTable.batch(batchLsit); + batchList.addAll(Arrays.asList(put1, put2, delete, get1, get2)); + results = new Result[batchList.size()]; + hTable.batch(batchList, results); Assert.assertEquals(5, results.length); Result getResult1 = (Result) results[3]; Assert.assertEquals(1, getResult1.size()); @@ -603,7 +607,7 @@ public static void testMultiCFBatchGetImpl(Map.Entry> entry String family = getColumnFamilyName(tableName); for (int i = 0; i < values.length; i++) { for (int j = 0; j < columns.length; j++) { - put.add(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i]+k)); + put.addColumn(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i]+k)); } } } @@ -818,7 +822,7 @@ public static void testMultiCFMixBatchImpl(Map.Entry> entry String family = getColumnFamilyName(tableName); for (int i = 0; i < values.length; i++) { for (int j = 0; j < columns.length; j++) { - put.add(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i] + k)); + put.addColumn(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i] + k)); } } } @@ -836,7 +840,8 @@ public static void testMultiCFMixBatchImpl(Map.Entry> entry batchList.add(createPutOp(key, families, columns[0], ts[1], "new_value_"+i)); batchList.add(createGetOp(key, families, columns[0], ts[1])); } - Object[] results = hTable.batch(batchList); + Result[] results = new Result[batchList.size()]; + hTable.batch(batchList, results); Assert.assertEquals(batchSize*3, results.length); for (int i = 0; i < batchSize; i++) { Result get1Result = (Result) results[3*i+0]; @@ -857,7 +862,8 @@ public static void testMultiCFMixBatchImpl(Map.Entry> entry batchList.add(createDeleteOp(key, families, columns[0], ts[1])); batchList.add(createGetOp(key, families, columns[0], ts[1])); } - Object[] results = hTable.batch(batchList); + Result[] results = new Result[batchList.size()]; + hTable.batch(batchList, results); Assert.assertEquals(batchSize*5, results.length); for (int i = 0; i < batchSize; i++) { Result get1Result = (Result) results[5*i+1]; @@ -879,7 +885,8 @@ public static void testMultiCFMixBatchImpl(Map.Entry> entry batchList.add(createDeleteOp(key, families, columns[0], ts[1])); batchList.add(createGetOp(key, families, columns[0], ts[1])); } - Object[] results = hTable.batch(batchList); + Result[] results = new Result[batchList.size()]; + hTable.batch(batchList, results); Assert.assertEquals(batchSize*4, results.length); for (int i = 0; i < batchSize; i++) { Result get1Result = (Result) results[4*i+1]; @@ -899,30 +906,28 @@ private static String[] getAllFamilies(List tableNames) throws Exception return families; } - private static void checkResult(String key, String[] families, String qualifier, - long ts, String value, Result result) { + private static void checkResult(String key, String[] families, String qualifier, long ts, + String value, Result result) { Assert.assertEquals(families.length, result.size()); - for (String family: families) { + for (String family : families) { Cell cell = result.getColumnCells(family.getBytes(), qualifier.getBytes()).get(0); AssertKeyValue(key, family, qualifier, ts, value, cell); } } - - private static Put createPutOp(String key, String[] families, String qualifier, - long ts, String value) { + private static Put createPutOp(String key, String[] families, String qualifier, long ts, + String value) { if (families == null || qualifier == null || value == null) { throw new IllegalArgumentException("input parameters is illegal"); } Put put = new Put(key.getBytes()); for (int i = 0; i < families.length; i++) { - put.add(families[i].getBytes(), qualifier.getBytes(), ts, value.getBytes()); + put.addColumn(families[i].getBytes(), qualifier.getBytes(), ts, value.getBytes()); } return put; } - private static Delete createDeleteOp(String key, String[] families, String qualifier, - long ts) { + private static Delete createDeleteOp(String key, String[] families, String qualifier, long ts) { if (families == null || qualifier == null) { throw new IllegalArgumentException("input parameters is illegal"); } @@ -933,8 +938,8 @@ private static Delete createDeleteOp(String key, String[] families, String quali return delete; } - private static Get createGetOp(String key, String[] families, String qualifier, - long ts) throws IOException { + private static Get createGetOp(String key, String[] families, String qualifier, long ts) + throws IOException { if (families == null || qualifier == null) { throw new IllegalArgumentException("input parameters is illegal"); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java index bf9b178a..95811d8d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCellTTLTest.java @@ -284,7 +284,7 @@ public static void testCellTTL(String tableName) throws Exception { hTable.increment(increment); r = hTable.get(get); assertEquals(r.size(), 1); - assertEquals(Bytes.toLong(r.raw()[0].getValue()), 12L); + assertEquals(Bytes.toLong(CellUtil.cloneValue(r.rawCells()[0])), 12L); get.setMaxVersions(10); r = hTable.get(get); assertEquals(r.size(), 2); @@ -297,7 +297,7 @@ public static void testCellTTL(String tableName) throws Exception { get.setMaxVersions(10); r = hTable.get(get); assertEquals(r.size(), 1); - assertEquals(Bytes.toLong(r.raw()[0].getValue()), 11L); + assertEquals(Bytes.toLong(CellUtil.cloneValue(r.rawCells()[0])), 11L); hTable.close(); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java index d2f6d3a3..14383ba2 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartCheckAndMutateTest.java @@ -71,9 +71,9 @@ public static void testCheckAndMutate(String tableName) throws Throwable { final byte[] ROW = Bytes.toBytes("12345"); final byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); Put put = new Put(ROW); - put.add(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); - put.add(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); - put.add(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); + put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); + put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); + put.addColumn(FAMILY, Bytes.toBytes("C"), Bytes.toBytes("c")); hTable.put(put); // get row back and assert the values Get get = new Get(ROW); @@ -89,11 +89,11 @@ public static void testCheckAndMutate(String tableName) throws Throwable { // put the same row again with C column deleted RowMutations rm = new RowMutations(ROW); put = new Put(ROW); - put.add(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); - put.add(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); + put.addColumn(FAMILY, Bytes.toBytes("A"), Bytes.toBytes("a")); + put.addColumn(FAMILY, Bytes.toBytes("B"), Bytes.toBytes("b")); rm.add(put); Delete del = new Delete(ROW); - del.deleteColumn(FAMILY, Bytes.toBytes("C")); + del.addColumn(FAMILY, Bytes.toBytes("C")); rm.add(del); boolean res = hTable.checkAndMutate(ROW, FAMILY, Bytes.toBytes("A"), CompareFilter.CompareOp.EQUAL, Bytes.toBytes("a"), rm); @@ -112,7 +112,7 @@ public static void testCheckAndMutate(String tableName) throws Throwable { //Test that we get a hTable level exception try { Put p = new Put(ROW); - p.add(new byte[] { 'b', 'o', 'g', 'u', 's' }, new byte[] { 'A' }, new byte[0]); + p.addColumn(new byte[] { 'b', 'o', 'g', 'u', 's' }, new byte[] { 'A' }, new byte[0]); rm = new RowMutations(ROW); rm.add(p); hTable.checkAndMutate(ROW, FAMILY, Bytes.toBytes("A"), CompareFilter.CompareOp.EQUAL, @@ -131,7 +131,7 @@ public static void testCheckAndPut(String tableName) throws Throwable { byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); Put put1 = new Put(ROW); - put1.add(FAMILY, QUALIFIER, VALUE_1); + put1.addColumn(FAMILY, QUALIFIER, VALUE_1); // row doesn't exist, so using non-null value should be considered "not match". boolean ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE_1, put1); @@ -146,14 +146,14 @@ public static void testCheckAndPut(String tableName) throws Throwable { assertEquals(ok, false); Put put2 = new Put(ROW); - put2.add(FAMILY, QUALIFIER, VALUE_2); + put2.addColumn(FAMILY, QUALIFIER, VALUE_2); // row now exists, use the matching value to check ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, VALUE_1, put2); assertEquals(ok, true); Put put3 = new Put(ROW_1); - put3.add(FAMILY, QUALIFIER, VALUE_1); + put3.addColumn(FAMILY, QUALIFIER, VALUE_1); // try to do CheckAndPut on different rows try { @@ -174,10 +174,10 @@ public static void testCheckAndPutWithCompareOp(String tableName) throws Throwab byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); Put put2 = new Put(ROW); - put2.add(FAMILY, QUALIFIER, value2); + put2.addColumn(FAMILY, QUALIFIER, value2); Put put3 = new Put(ROW); - put3.add(FAMILY, QUALIFIER, value3); + put3.addColumn(FAMILY, QUALIFIER, value3); // row doesn't exist, so using "null" to check for existence should be considered "match". boolean ok = hTable.checkAndPut(ROW, FAMILY, QUALIFIER, null, put2); @@ -255,14 +255,14 @@ public static void testCheckAndDeleteWithCompareOp(String tableName) throws Thro byte[] FAMILY = getColumnFamilyName(tableName).getBytes(); Put put2 = new Put(ROW); - put2.add(FAMILY, QUALIFIER, value2); + put2.addColumn(FAMILY, QUALIFIER, value2); hTable.put(put2); Put put3 = new Put(ROW); - put3.add(FAMILY, QUALIFIER, value3); + put3.addColumn(FAMILY, QUALIFIER, value3); Delete delete = new Delete(ROW); - delete.deleteColumns(FAMILY, QUALIFIER); + delete.addColumns(FAMILY, QUALIFIER); // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL // turns out "match" @@ -350,7 +350,7 @@ private static void testCheckAndMutateMultiCF(Map.Entry> en List tableNames = entry.getValue(); byte[] FAMILY_1 = getColumnFamilyName(tableNames.get(0)).getBytes(); Put put2 = new Put(ROW); - put2.add(FAMILY_1, QUALIFIER, VALUE_2); + put2.addColumn(FAMILY_1, QUALIFIER, VALUE_2); hTable.put(put2); RowMutations mutations = new RowMutations(ROW); @@ -372,7 +372,7 @@ private static void testCheckAndMutateMultiCF(Map.Entry> en mutations = new RowMutations(ROW); byte[] FAMILY_2 = tableNames.get(1).getBytes(); Put put3 = new Put(ROW); - put3.add(FAMILY_2, QUALIFIER, VALUE_1); + put3.addColumn(FAMILY_2, QUALIFIER, VALUE_1); mutations.add(put2); mutations.add(put3); try { @@ -393,10 +393,10 @@ private static void testCheckAndMutateSeires(String tableName) throws Exception byte[] ROW = "appendKey".getBytes(); RowMutations mutations = new RowMutations(ROW); Put put2 = new Put(ROW); - put2.add(FAMILY, QUALIFIER, VALUE_2); + put2.addColumn(FAMILY, QUALIFIER, VALUE_2); hTable.put(put2); Put put3 = new Put(ROW); - put3.add(FAMILY, QUALIFIER, VALUE_1); + put3.addColumn(FAMILY, QUALIFIER, VALUE_1); mutations.add(put3); try { hTable.checkAndMutate(ROW, FAMILY, QUALIFIER, CompareFilter.CompareOp.GREATER, VALUE_2, diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java index 226d4fa0..f1b574d1 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartDeleteTest.java @@ -15,7 +15,6 @@ * #L% */ - package com.alipay.oceanbase.hbase.secondary; import com.alipay.oceanbase.hbase.OHTableClient; @@ -35,9 +34,8 @@ import static org.apache.hadoop.hbase.util.Bytes.toBytes; public class OHTableSecondaryPartDeleteTest { - private static List tableNames = new LinkedList(); - private static Map> group2tableNames = new LinkedHashMap<>(); - + private static List tableNames = new LinkedList(); + private static Map> group2tableNames = new LinkedHashMap>(); @BeforeClass public static void before() throws Exception { @@ -56,7 +54,7 @@ public static void finish() throws Exception { public void prepareCase() throws Exception { truncateTables(tableNames, group2tableNames); } - + public static void testDeleteLastVersionImpl(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); @@ -72,14 +70,14 @@ public static void testDeleteLastVersionImpl(String tableName) throws Exception Long ts4 = ts3 + 1000; { // delete last version Put put = new Put(toBytes(key)); - put.add(toBytes(family), toBytes(column), ts1, toBytes(value)); - put.add(toBytes(family), toBytes(column), ts2, toBytes(value)); - put.add(toBytes(family), toBytes(column), ts3, toBytes(value)); - put.add(toBytes(family), toBytes(column), ts4, toBytes(value)); + put.addColumn(toBytes(family), toBytes(column), ts1, toBytes(value)); + put.addColumn(toBytes(family), toBytes(column), ts2, toBytes(value)); + put.addColumn(toBytes(family), toBytes(column), ts3, toBytes(value)); + put.addColumn(toBytes(family), toBytes(column), ts4, toBytes(value)); hTable.put(put); Delete delete = new Delete(toBytes(key)); - delete.deleteColumn(toBytes(family), toBytes(column)); + delete.addColumn(toBytes(family), toBytes(column)); hTable.delete(delete); Get get = new Get(toBytes(key)); @@ -94,7 +92,7 @@ public static void testDeleteLastVersionImpl(String tableName) throws Exception } hTable.close(); } - + public static void testDeleteSpecifiedImpl(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); @@ -111,10 +109,10 @@ public static void testDeleteSpecifiedImpl(String tableName) throws Exception { { Put put = new Put(toBytes(key)); - put.add(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); - put.add(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); - put.add(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); - put.add(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + put.addColumn(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); hTable.put(put); Delete delete = new Delete(toBytes(key)); @@ -133,7 +131,7 @@ public static void testDeleteSpecifiedImpl(String tableName) throws Exception { } hTable.close(); } - + public static void testDeleteColumnImpl(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); @@ -150,10 +148,10 @@ public static void testDeleteColumnImpl(String tableName) throws Exception { { Put put = new Put(toBytes(key)); - put.add(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); - put.add(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); - put.add(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); - put.add(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + put.addColumn(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); hTable.put(put); Delete delete = new Delete(toBytes(key)); @@ -172,7 +170,7 @@ public static void testDeleteColumnImpl(String tableName) throws Exception { } hTable.close(); } - + public static void testDeleteFamilyImpl(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); @@ -189,10 +187,10 @@ public static void testDeleteFamilyImpl(String tableName) throws Exception { { Put put = new Put(toBytes(key)); - put.add(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); - put.add(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); - put.add(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); - put.add(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + put.addColumn(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); hTable.put(put); Delete delete = new Delete(toBytes(key)); @@ -207,7 +205,7 @@ public static void testDeleteFamilyImpl(String tableName) throws Exception { } hTable.close(); } - + public static void testDeleteFamilyVersionImpl(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); @@ -224,10 +222,10 @@ public static void testDeleteFamilyVersionImpl(String tableName) throws Exceptio { Put put = new Put(toBytes(key)); - put.add(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); - put.add(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); - put.add(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); - put.add(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); + put.addColumn(toBytes(family), toBytes(column), ts1, toBytes(value + ts1)); + put.addColumn(toBytes(family), toBytes(column), ts2, toBytes(value + ts2)); + put.addColumn(toBytes(family), toBytes(column), ts3, toBytes(value + ts3)); + put.addColumn(toBytes(family), toBytes(column), ts4, toBytes(value + ts4)); hTable.put(put); Delete delete = new Delete(toBytes(key)); @@ -334,7 +332,7 @@ public static void testMultiCFDeleteSpecifiedImpl(Map.Entry } hTable.close(); } - + public static void testMultiCFDeleteColumnImpl(Map.Entry> entry) throws Exception { String key = "putKey"; String value = "value"; @@ -376,9 +374,9 @@ public static void testMultiCFDeleteColumnImpl(Map.Entry> e } hTable.close(); } - - - public static void testMultiCFDeleteFamilyImpl(Map.Entry> entry) throws Exception { + + public static void testMultiCFDeleteFamilyImpl(Map.Entry> entry) + throws Exception { String key = "putKey"; String value = "value"; String column = "putColumn"; @@ -403,7 +401,6 @@ public static void testMultiCFDeleteFamilyImpl(Map.Entry> e hTable.put(put); hTable.delete(delete); - Get get = new Get(toBytes(key)); get.setMaxVersions(); for (String tableName : entry.getValue()) { @@ -414,7 +411,7 @@ public static void testMultiCFDeleteFamilyImpl(Map.Entry> e Assert.assertEquals(0, result.size()); hTable.close(); } - + public static void testMultiCFDeleteFamilyVersionImpl(Map.Entry> entry) throws Exception { String key = "putKey"; String value = "value"; @@ -451,7 +448,7 @@ public static void testMultiCFDeleteFamilyVersionImpl(Map.EntryAssert.assertEquals(6, result.size())); hTable.close(); } - + public static void testDeleteAllImpl(Map.Entry> entry) throws Exception { String key = "putKey"; String value = "value"; @@ -487,58 +484,57 @@ public static void testDeleteAllImpl(Map.Entry> entry) thro Assert(entry.getValue(), ()->Assert.assertEquals(0, result.size())); hTable.close(); } - + @Test public void testDelete() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteSpecifiedImpl); } + @Test public void testDeleteLastVersion() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteLastVersionImpl); } - + @Test public void testDeleteColumn() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteColumnImpl); } - + @Test public void testDeleteFamily() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteFamilyImpl); } - + @Test public void testDeleteFamilyVersion() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartDeleteTest::testDeleteFamilyVersionImpl); } - - + @Test public void testMultiCFDelete() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteLastVersionImpl); } - + @Test public void testMultiCFDeleteSpecified() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteSpecifiedImpl); } - + @Test public void testMultiCFDeleteColumn() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteColumnImpl); } - + @Test public void testMultiCFDeleteFamily() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteFamilyImpl); } - + @Test public void testMultiCFDeleteFamilyVersion() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testMultiCFDeleteFamilyVersionImpl); } - - + @Test public void testDeleteAll() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartDeleteTest::testDeleteAllImpl); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java index edda1e18..cae33d1e 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartGetTest.java @@ -80,7 +80,7 @@ public static void testGetImpl(String tableName) throws Exception { for (int i = 0; i < values.length; i++) { for (int j = 0; j < columns.length; j++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i])); + put.addColumn(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i])); hTable.put(put); } } @@ -91,7 +91,7 @@ public static void testGetImpl(String tableName) throws Exception { Get get = new Get(key.getBytes()); get.addColumn(family.getBytes(), columns[index].getBytes()); Result r = hTable.get(get); - Assert.assertEquals(1, r.raw().length); + Assert.assertEquals(1, r.size()); AssertKeyValue(key, columns[index], lastTs, latestValue, r.rawCells()[0]); } @@ -115,9 +115,9 @@ public static void testGetImpl(String tableName) throws Exception { get.addColumn(family.getBytes(), columns[index].getBytes()); get.setMaxVersions(2); Result r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - AssertKeyValue(key, columns[index], ts[1], values[1], r.raw()[0]); - AssertKeyValue(key, columns[index], ts[0], values[0], r.raw()[1]); + Assert.assertEquals(2, r.size()); + AssertKeyValue(key, columns[index], ts[1], values[1], r.rawCells()[0]); + AssertKeyValue(key, columns[index], ts[0], values[0], r.rawCells()[1]); } // 4. get specify time range @@ -127,9 +127,9 @@ public static void testGetImpl(String tableName) throws Exception { get.setMaxVersions(2); get.setTimeStamp(ts[1]); Result r = hTable.get(get); - Assert.assertEquals(columns.length, r.raw().length); + Assert.assertEquals(columns.length, r.size()); for (int i = 0; i < columns.length; i++) { - AssertKeyValue(key, columns[i], values[1], r.raw()[i]); + AssertKeyValue(key, columns[i], values[1], r.rawCells()[i]); } } @@ -142,9 +142,9 @@ public static void testGetImpl(String tableName) throws Exception { new BinaryComparator(toBytes(values[0]))); get.setFilter(valueFilter); Result r = hTable.get(get); - Assert.assertEquals(columns.length, r.raw().length); + Assert.assertEquals(columns.length, r.size()); for (int i = 0; i < columns.length; i++) { - AssertKeyValue(key, columns[i], values[0], r.raw()[i]); + AssertKeyValue(key, columns[i], values[0], r.rawCells()[i]); } } @@ -171,7 +171,8 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr for (int i = 0; i < values.length; i++) { for (int j = 0; j < columns.length; j++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns[j].getBytes(), ts[i], toBytes(values[i])); + put.addColumn(family.getBytes(), columns[j].getBytes(), ts[i], + toBytes(values[i])); hTable.put(put); } } @@ -185,7 +186,7 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr Get get = new Get(key.getBytes()); get.addColumn(family.getBytes(), columns[columnIndex].getBytes()); Result r = hTable.get(get); - Assert.assertEquals(1, r.raw().length); + Assert.assertEquals(1, r.size()); AssertKeyValue(key, columns[columnIndex], lastTs, latestValue, r.rawCells()[0]); } } @@ -210,11 +211,11 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr { Get get = new Get(key.getBytes()); Result r = hTable.get(get); - Assert.assertEquals(tableNames.size() * columns.length, r.raw().length); + Assert.assertEquals(tableNames.size() * columns.length, r.size()); int cur = 0; for (String tableName : tableNames) { for (int i = 0; i < columns.length; i++) { - AssertKeyValue(key, columns[i], lastTs, latestValue, r.raw()[cur]); + AssertKeyValue(key, columns[i], lastTs, latestValue, r.rawCells()[cur]); cur++; } } @@ -240,13 +241,13 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr Get get = new Get(key.getBytes()); get.setMaxVersions(2); Result r = hTable.get(get); - Assert.assertEquals(tableNames.size() * columns.length * ts.length, r.raw().length); + Assert.assertEquals(tableNames.size() * columns.length * ts.length, r.size()); int cur = 0; for (String tableName : tableNames) { String family = getColumnFamilyName(tableName); for (int i = 0; i < columns.length; i++) { for (int k = ts.length - 1; k >= 0; k--) { - AssertKeyValue(key, family, columns[i], ts[k], values[k], r.raw()[cur]); + AssertKeyValue(key, family, columns[i], ts[k], values[k], r.rawCells()[cur]); cur++; } } @@ -259,12 +260,12 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr get.setMaxVersions(2); get.setTimeStamp(ts[1]); Result r = hTable.get(get); - Assert.assertEquals(tableNames.size() * columns.length, r.raw().length); + Assert.assertEquals(tableNames.size() * columns.length, r.size()); int cur = 0; for (String tableName : tableNames) { String family = getColumnFamilyName(tableName); for (int i = 0; i < columns.length; i++) { - AssertKeyValue(key, family, columns[i], ts[1], values[1], r.raw()[cur]); + AssertKeyValue(key, family, columns[i], ts[1], values[1], r.rawCells()[cur]); cur++; } } @@ -278,12 +279,12 @@ public static void testMultiCFGetImpl(Map.Entry> entry) thr new BinaryComparator(toBytes(values[0]))); get.setFilter(valueFilter); Result r = hTable.get(get); - Assert.assertEquals(tableNames.size() * columns.length, r.raw().length); + Assert.assertEquals(tableNames.size() * columns.length, r.size()); int cur = 0; for (String tableName : tableNames) { String family = getColumnFamilyName(tableName); for (int i = 0; i < columns.length; i++) { - AssertKeyValue(key, family, columns[i], ts[0], values[0], r.raw()[cur]); + AssertKeyValue(key, family, columns[i], ts[0], values[0], r.rawCells()[cur]); cur++; } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java index 4204ddf7..8770b1d3 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartIncrementTest.java @@ -21,6 +21,7 @@ import com.alipay.oceanbase.hbase.OHTableClient; import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.hbase.util.TableTemplateManager; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.junit.AfterClass; @@ -70,7 +71,7 @@ public static void testIncrement(String tableName) throws Exception { byte[] qualifier2 = Bytes.toBytes("qual2"); byte[] val = Bytes.toBytes(0L); Put p = new Put(row); - p.add(cf, qualifier, val); + p.addColumn(cf, qualifier, val); hTable.put(p); for (int count = 0; count < 13; count++) { @@ -82,7 +83,7 @@ public static void testIncrement(String tableName) throws Exception { get.setMaxVersions(1); get.addFamily(cf); Result result = hTable.get(get); - assertEquals(1300L, Bytes.toLong(result.raw()[0].getValue())); + assertEquals(1300L, Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0]))); get.setMaxVersions(100); result = hTable.get(get); assertEquals(14, result.size()); @@ -93,8 +94,10 @@ public static void testIncrement(String tableName) throws Exception { hTable.increment(inc); get.setMaxVersions(1); result = hTable.get(get); - assertEquals(1200L, Bytes.toLong(result.getColumnCells(cf, qualifier).get(0).getValue())); - assertEquals(-100L, Bytes.toLong(result.getColumnCells(cf, qualifier2).get(0).getValue())); + assertEquals(1200L, + Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(cf, qualifier).get(0)))); + assertEquals(-100L, + Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(cf, qualifier2).get(0)))); hTable.close(); } @@ -132,19 +135,19 @@ private static void testIncBorder(String tableName) throws Exception { get.addFamily(FAMILY); result = hTable.get(get); assertEquals(1, result.size()); - assertEquals(2L, Bytes.toLong(result.raw()[0].getValue())); + assertEquals(2L, Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0]))); inc.addColumn(FAMILY, QUALIFIERS[0], 2L); hTable.increment(inc); get.setMaxVersions(10); get.addFamily(FAMILY); result = hTable.get(get); assertEquals(3, result.size()); - assertEquals(4L, - Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(0).getValue())); - assertEquals(2L, - Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[1]).get(1).getValue())); - assertEquals(2L, - Bytes.toLong(result.getColumnCells(FAMILY, QUALIFIERS[0]).get(0).getValue())); + assertEquals(4L, Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(FAMILY, + QUALIFIERS[1]).get(0)))); + assertEquals(2L, Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(FAMILY, + QUALIFIERS[1]).get(1)))); + assertEquals(2L, Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(FAMILY, + QUALIFIERS[0]).get(0)))); } finally { hTable.close(); } @@ -185,7 +188,7 @@ private static void testIncCon(String tableName) throws Exception { get.setMaxVersions(1); get.addColumn(FAMILY, column.getBytes()); Result result = hTable.get(get); - assertEquals(expect, Bytes.toLong(result.getColumnCells(FAMILY, column.getBytes()).get(0).getValue())); + assertEquals(expect, Bytes.toLong(CellUtil.cloneValue(result.getColumnCells(FAMILY, column.getBytes()).get(0)))); } finally { hTable.close(); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java index 36247fe9..fef4dce0 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartPutTest.java @@ -15,7 +15,6 @@ * #L% */ - package com.alipay.oceanbase.hbase.secondary; import com.alipay.oceanbase.hbase.OHTableClient; @@ -36,8 +35,7 @@ public class OHTableSecondaryPartPutTest { private static List tableNames = new LinkedList(); - private static Map> group2tableNames = new LinkedHashMap<>(); - + private static Map> group2tableNames = new LinkedHashMap>(); @BeforeClass public static void before() throws Exception { @@ -70,8 +68,8 @@ public static void testPutImpl(String tableName) throws Exception { { // put new key and get long timestamp = System.currentTimeMillis(); Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value)); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value)); hTable.put(put); Get get = new Get(toBytes(key)); @@ -85,7 +83,7 @@ public static void testPutImpl(String tableName) throws Exception { { // put exist key and get long timestamp = System.currentTimeMillis(); Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); hTable.put(put); Get get = new Get(toBytes(key)); @@ -98,7 +96,7 @@ public static void testPutImpl(String tableName) throws Exception { { // test timestamp update long timestamp = System.currentTimeMillis(); Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); hTable.put(put); Get get = new Get(toBytes(key)); @@ -109,7 +107,7 @@ public static void testPutImpl(String tableName) throws Exception { Assert(tableName, ()->Assert.assertEquals(timestamp, r.rawCells()[0].getTimestamp())); Put put1 = new Put(toBytes(key)); - put1.add(family.getBytes(), column1.getBytes(), timestamp + 100, toBytes(column1 + value)); + put1.addColumn(family.getBytes(), column1.getBytes(), timestamp + 100, toBytes(column1 + value)); hTable.put(put1); Result r2 = hTable.get(get); @@ -122,7 +120,7 @@ public static void testPutImpl(String tableName) throws Exception { hTable.close(); } - + public static void testBatchPutImpl(String tableName) throws Exception { OHTableClient hTable = ObHTableTestUtil.newOHTableClient(getTableName(tableName)); hTable.init(); @@ -137,14 +135,14 @@ public static void testBatchPutImpl(String tableName) throws Exception { Get get = new Get(toBytes(key)); for (int i = 0; i < 10; ++i) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp + i)); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp + i)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp + i)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp + i)); puts.add(put); get.addColumn(family.getBytes(), column1.getBytes()); get.addColumn(family.getBytes(), column2.getBytes()); } - - hTable.batch(puts); + Result[] results = new Result[puts.size()]; + hTable.batch(puts, results); Result result = hTable.get(get); Assert(tableName, ()->Assert.assertEquals(2, result.size())); for (Cell cell : result.rawCells()) { @@ -159,16 +157,17 @@ public static void testBatchPutImpl(String tableName) throws Exception { for (int i = 0; i < 10; ++i) { Put put = new Put(toBytes(key + i)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp + i)); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp + i)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp + i)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp + i)); puts.add(put); Get get = new Get(toBytes(key + i)); get.addColumn(family.getBytes(), column1.getBytes()); get.addColumn(family.getBytes(), column2.getBytes()); gets.add(get); } - - hTable.batch(puts); + + Result[] results = new Result[puts.size()]; + hTable.batch(puts, results); for (int i = 0; i < 10; ++i) { Result result = hTable.get(gets.get(i)); @@ -198,8 +197,8 @@ public static void testMultiCFPutImpl(Map.Entry> entry) thr Get get = new Get(toBytes(key)); for (String tableName : entry.getValue()) { String family = getColumnFamilyName(tableName); - put.add(family.getBytes(), column1.getBytes(), currentTime, toBytes(column1 + value)); - put.add(family.getBytes(), column2.getBytes(), currentTime, toBytes(column2 + value)); + put.addColumn(family.getBytes(), column1.getBytes(), currentTime, toBytes(column1 + value)); + put.addColumn(family.getBytes(), column2.getBytes(), currentTime, toBytes(column2 + value)); get.addColumn(family.getBytes(), column1.getBytes()); get.addColumn(family.getBytes(), column2.getBytes()); } @@ -219,8 +218,8 @@ public static void testMultiCFPutImpl(Map.Entry> entry) thr Get get = new Get(toBytes(key)); for (String tableName : entry.getValue()) { String family = getColumnFamilyName(tableName); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value)); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value)); get.addColumn(family.getBytes(), column1.getBytes()); get.addColumn(family.getBytes(), column2.getBytes()); } @@ -256,13 +255,14 @@ public static void testMltiCFPutBatchImpl(Map.Entry> entry) for (String tableName : entry.getValue()) { String family = getColumnFamilyName(tableName); Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp)); puts.add(put); get.addColumn(family.getBytes(), column1.getBytes()); get.addColumn(family.getBytes(), column2.getBytes()); } - hTable.batch(puts); + Result[] results = new Result[puts.size()]; + hTable.batch(puts, results); Result result = hTable.get(get); Assert(entry.getValue(), ()->Assert.assertEquals(entry.getValue().size() * 2, result.size())); for (Cell cell : result.rawCells()) { @@ -285,8 +285,8 @@ public static void testMltiCFPutBatchImpl(Map.Entry> entry) String family = getColumnFamilyName(tableName); for (int i = 0; i < 10; ++i) { Put put = new Put(toBytes(key + i)); - put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); - put.add(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp)); + put.addColumn(family.getBytes(), column1.getBytes(), timestamp, toBytes(column1 + value + timestamp)); + put.addColumn(family.getBytes(), column2.getBytes(), timestamp, toBytes(column2 + value + timestamp)); puts.add(put); Get get = new Get(toBytes(key + i)); get.addColumn(family.getBytes(), column1.getBytes()); @@ -294,7 +294,8 @@ public static void testMltiCFPutBatchImpl(Map.Entry> entry) gets.add(new Pair<>(get, family)); } } - hTable.batch(puts); + Result[] results = new Result[puts.size()]; + hTable.batch(puts, results); for (int i = 0; i < 10; ++i) { Result result = hTable.get(gets.get(i).getFirst()); Assert(entry.getValue(), () -> Assert.assertEquals(2, result.size())); @@ -309,26 +310,24 @@ public static void testMltiCFPutBatchImpl(Map.Entry> entry) } hTable.close(); } - - + @Test public void testPut() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartPutTest::testPutImpl); } - + @Test public void testBatchPut() throws Throwable { FOR_EACH(tableNames, OHTableSecondaryPartPutTest::testBatchPutImpl); } - + @Test public void testMultiCFPut() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMultiCFPutImpl); } - + @Test public void testMultiCFPutBatch() throws Throwable { FOR_EACH(group2tableNames, OHTableSecondaryPartPutTest::testMltiCFPutBatchImpl); } - } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java index 34a966ad..d267f39c 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartScanTest.java @@ -80,7 +80,8 @@ public static void testScanImpl(String tableName) throws Exception { for (String column : columns) { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), tss[i], + values[i].getBytes()); hTable.put(put); } } @@ -229,7 +230,12 @@ public static void testScanImpl(String tableName) throws Exception { try { hTable.getScanner(scan); } catch (Exception e) { - Assert.assertTrue(e.getCause().getMessage().contains("secondary partitioned hbase table with reverse query not supported")); + Assert + .assertTrue(e + .getCause() + .getMessage() + .contains( + "secondary partitioned hbase table with reverse query not supported")); } } hTable.close(); @@ -261,7 +267,8 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th for (String column : columns) { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), tss[i], + values[i].getBytes()); hTable.put(put); } } @@ -316,7 +323,6 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th for (String tableName : tableNames) { String family = getColumnFamilyName(tableName); for (String column : columns) { - Cell cell = cells.get(cellIndex); AssertKeyValue(keys[i], family, column, lastTs, latestValue, cells.get(cellIndex)); cellIndex++; @@ -488,12 +494,12 @@ public static void testMultiCFScanImpl(Map.Entry> entry) th } @Test - public void testScan() throws Throwable { - FOR_EACH(tableNames, OHTableSecondaryPartScanTest::testScanImpl); - } + public void testScan() throws Throwable { + FOR_EACH(tableNames, OHTableSecondaryPartScanTest::testScanImpl); + } @Test - public void testMultiCFScan() throws Throwable { - FOR_EACH(group2tableNames, OHTableSecondaryPartScanTest::testMultiCFScanImpl); - } + public void testMultiCFScan() throws Throwable { + FOR_EACH(group2tableNames, OHTableSecondaryPartScanTest::testMultiCFScanImpl); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java index 8b0c0109..5a392586 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableSecondaryPartTTLTest.java @@ -56,7 +56,7 @@ public static void testTTLImpl(List tableNames) throws Exception { for (String column : columns) { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), values[i].getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), values[i].getBytes()); hTable.put(put); } } @@ -132,7 +132,7 @@ public static void testMultiCFTTLImpl(Map> group2tableNames for (String column : columns) { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), values[i].getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), values[i].getBytes()); hTable.put(put); } } @@ -203,7 +203,7 @@ void testRowkeyTTL(List tableNames, Boolean useScan, Boolean isReversed) for (String column : columns) { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), values[i].getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), values[i].getBytes()); hTable.put(put); } } @@ -296,7 +296,7 @@ void testMultiCFRowkeyTTL(Map> group2tableNames, Boolean us for (String column : columns) { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), values[i].getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), values[i].getBytes()); hTable.put(put); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java index fc025e8b..1e99f130 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesGetTest.java @@ -79,7 +79,7 @@ public static void testGetImpl(String tableName) throws Exception { for (int i = 0; i < values.length; i++) { for (int j = 0; j < columns1.length; j++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), columns1[j].getBytes(), ts[i], toBytes(values[i])); + put.addColumn(family.getBytes(), columns1[j].getBytes(), ts[i], toBytes(values[i])); hTable.put(put); } } @@ -87,7 +87,7 @@ public static void testGetImpl(String tableName) throws Exception { Put put = new Put(toBytes(key)); for (int i = 0; i < values.length; i++) { for (int j = 0; j < columns2.length; j++) { - put.add(family.getBytes(), columns2[j].getBytes(), ts[i], toBytes(values[i])); + put.addColumn(family.getBytes(), columns2[j].getBytes(), ts[i], toBytes(values[i])); } } hTable.put(put); diff --git a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java index d0765c2f..28c5a84f 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/secondary/OHTableTimeSeriesScanTest.java @@ -80,7 +80,7 @@ public static void testScanImpl(String tableName) throws Exception { Put put = new Put(toBytes(key)); for (String column : columns) { for (int i = 0; i < values.length; i++) { - put.add(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); } } hTable.put(put); @@ -90,7 +90,7 @@ public static void testScanImpl(String tableName) throws Exception { for (String column : columns) { for (int i = 0; i < values.length; i++) { Put put = new Put(toBytes(key)); - put.add(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); + put.addColumn(family.getBytes(), column.getBytes(), tss[i], values[i].getBytes()); hTable.put(put); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java index e7195e55..03b1691b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableSecondaryPartUtil.java @@ -18,6 +18,7 @@ package com.alipay.oceanbase.hbase.util; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.util.Bytes; @@ -309,25 +310,25 @@ public static void disableTTL() throws Exception { public static void AssertKeyValue(String key, String qualifier, long timestamp, String value, Cell cell) { - Assert.assertEquals(key, Bytes.toString(cell.getRow())); - Assert.assertEquals(qualifier, Bytes.toString(cell.getQualifier())); + Assert.assertEquals(key, Bytes.toString(CellUtil.cloneRow(cell))); + Assert.assertEquals(qualifier, Bytes.toString(CellUtil.cloneQualifier(cell))); Assert.assertEquals(timestamp, cell.getTimestamp()); - Assert.assertEquals(value, Bytes.toString(cell.getValue())); + Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(cell))); } public static void AssertKeyValue(String key, String qualifier, String value, Cell cell) { - Assert.assertEquals(key, Bytes.toString(cell.getRow())); - Assert.assertEquals(qualifier, Bytes.toString(cell.getQualifier())); - Assert.assertEquals(value, Bytes.toString(cell.getValue())); + Assert.assertEquals(key, Bytes.toString(CellUtil.cloneRow(cell))); + Assert.assertEquals(qualifier, Bytes.toString(CellUtil.cloneQualifier(cell))); + Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(cell))); } public static void AssertKeyValue(String key, String family, String qualifier, long timestamp, String value, Cell cell) { - Assert.assertEquals(key, Bytes.toString(cell.getRow())); - Assert.assertEquals(family, Bytes.toString(cell.getFamily())); - Assert.assertEquals(qualifier, Bytes.toString(cell.getQualifier())); + Assert.assertEquals(key, Bytes.toString(CellUtil.cloneRow(cell))); + Assert.assertEquals(family, Bytes.toString(CellUtil.cloneFamily(cell))); + Assert.assertEquals(qualifier, Bytes.toString(CellUtil.cloneQualifier(cell))); Assert.assertEquals(timestamp, cell.getTimestamp()); - Assert.assertEquals(value, Bytes.toString(cell.getValue())); + Assert.assertEquals(value, Bytes.toString(CellUtil.cloneValue(cell))); } public static List getCellsFromScanner(ResultScanner scanner) { @@ -352,21 +353,31 @@ public static void checkUtilTimeout(List tableNames, Supplier f } public static void sortCells(Cell[] cells) { - if (cells == null || cells.length <= 1) { return; } + if (cells == null || cells.length <= 1) { + return; + } Arrays.sort(cells, new Comparator() { @Override public int compare(Cell c1, Cell c2) { - if (c1 == null) return 1; - if (c2 == null) return -1; - int cmpRet = Bytes.compareTo(c1.getRow(), c2.getRow()); - if (cmpRet != 0) { return cmpRet; } + if (c1 == null) + return 1; + if (c2 == null) + return -1; + int cmpRet = Bytes.compareTo(CellUtil.cloneRow(c1), CellUtil.cloneRow(c2)); + if (cmpRet != 0) { + return cmpRet; + } - cmpRet = Bytes.compareTo(c1.getFamily(), c2.getFamily()); - if (cmpRet != 0) { return cmpRet; } + cmpRet = Bytes.compareTo(CellUtil.cloneFamily(c1), CellUtil.cloneFamily(c2)); + if (cmpRet != 0) { + return cmpRet; + } - cmpRet = Bytes.compareTo(c1.getQualifier(), c2.getQualifier()); - if (cmpRet != 0) { return cmpRet; } + cmpRet = Bytes.compareTo(CellUtil.cloneQualifier(c1), CellUtil.cloneQualifier(c2)); + if (cmpRet != 0) { + return cmpRet; + } cmpRet = Long.compare(c2.getTimestamp(), c1.getTimestamp()); return cmpRet; diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java index effeb457..81fd3688 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -20,7 +20,6 @@ import com.alipay.oceanbase.hbase.OHTableClient; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HTableDescriptor; import java.sql.Connection; @@ -53,17 +52,16 @@ public class ObHTableTestUtil { + "characterEncoding=utf-8&" + "socketTimeout=3000000&" + "connectTimeout=60000"; public static String SYS_JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " - + "oceanbase?" + "useUnicode=TRUE&" + + "oceanbase?" + "useUnicode=TRUE&" + "characterEncoding=utf-8&" + "socketTimeout=3000000&" + "connectTimeout=60000"; public static String SQL_FORMAT = "truncate %s"; - public static List tableNameList; + public static List tableNameList = new LinkedList(); public static Connection conn; public static Statement stmt; static { - tableNameList = new LinkedList<>(); conn = getConnection(); try { stmt = conn.createStatement(); @@ -125,29 +123,28 @@ public static OHTableClient newOHTableClient(String tableName) { } static public List getOHTableNameList(String tableGroup) throws IOException { - // 读取建表语句 - List res = new LinkedList<>(); - String sql = new String(Files.readAllBytes(Paths.get(NativeHBaseUtil.SQL_PATH))); - String[] sqlList = sql.split(";"); - Map tableMap = new LinkedHashMap<>(); - for (String singleSql : sqlList) { - String realTableName; - if (singleSql.contains("CREATE TABLE ")) { - singleSql.trim(); - String[] splits = singleSql.split(" "); - String tableGroupName = splits[2].substring(1, splits[2].length() - 1); - if (tableGroupName.contains(":")) { - String[] tmpStr = tableGroupName.split(":", 2); - tableGroupName = tmpStr[1]; - } - realTableName = tableGroupName.split("\\$", 2)[0]; - if (realTableName.equals(tableGroup)) { - res.add(tableGroupName); + // 读取建表语句 + List res = new LinkedList<>(); + String sql = new String(Files.readAllBytes(Paths.get(NativeHBaseUtil.SQL_PATH))); + String[] sqlList = sql.split(";"); + for (String singleSql : sqlList) { + String realTableName; + if (singleSql.contains("CREATE TABLE ")) { + singleSql.trim(); + String[] splits = singleSql.split(" "); + String tableGroupName = splits[2].substring(1, splits[2].length() - 1); + if (tableGroupName.contains(":")) { + String[] tmpStr = tableGroupName.split(":", 2); + tableGroupName = tmpStr[1]; + } + realTableName = tableGroupName.split("\\$", 2)[0]; + if (realTableName.equals(tableGroup)) { + res.add(tableGroupName); + } } } + return res; } - return res; - } static public Connection getConnection() { try { @@ -164,7 +161,8 @@ static public Connection getConnection() { static public Connection getSysConnection() { try { Class.forName("com.mysql.cj.jdbc.Driver"); - Connection conn = DriverManager.getConnection(SYS_JDBC_URL, SYS_USER_NAME, SYS_PASSWORD); + Connection conn = DriverManager + .getConnection(SYS_JDBC_URL, SYS_USER_NAME, SYS_PASSWORD); return conn; } catch (Exception e) { @@ -175,28 +173,25 @@ static public Connection getSysConnection() { @FunctionalInterface public interface CheckedConsumer { void accept(T t) throws Throwable; - default CheckedConsumer andThen(CheckedConsumer after) throws Exception { - if (after == null) { throw new NullPointerException();} - return (T t) -> { - accept(t); - after.accept(t); - }; - } } - - public static void FOR_EACH(List tableNames, CheckedConsumer consumer) throws Throwable { + + public static void FOR_EACH(List tableNames, CheckedConsumer consumer) + throws Throwable { for (String tableName : tableNames) { - System.out.println("============================= table::{" + tableName + "} ============================="); + System.out.println("============================= table::{" + tableName + + "} ============================="); consumer.accept(tableName); } } - - public static void FOR_EACH(Map> group2Tables, CheckedConsumer>> consumer) throws Throwable { + + public static void FOR_EACH(Map> group2Tables, + CheckedConsumer>> consumer) + throws Throwable { for (Map.Entry> entry : group2Tables.entrySet()) { consumer.accept(entry); } } - + public static void Assert(String tableName, Runnable assertMethod) throws SQLException { try { assertMethod.run(); @@ -209,7 +204,7 @@ public static void Assert(String tableName, Runnable assertMethod) throws SQLExc throw e; } } - + public static void Assert(List tableNames, Runnable assertMethod) throws SQLException { try { assertMethod.run(); @@ -224,8 +219,7 @@ public static void Assert(List tableNames, Runnable assertMethod) throws throw e; } } - - + public static boolean secureCompare(byte[] a, byte[] b) { int diff = a.length ^ b.length; for (int i = 0; i < a.length && i < b.length; i++) { diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java b/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java index 65b3964c..0ae65bb2 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ResultSetPrinter.java @@ -23,48 +23,43 @@ import java.util.*; public class ResultSetPrinter { - private static final int MAX_COL_WIDTH = 30; - private static final String VERTICAL = "│"; - private static final String HORIZONTAL = "─"; - private static final String TOP_LEFT = "┌"; - private static final String TOP_RIGHT = "┐"; - private static final String MID_LEFT = "├"; - private static final String MID_RIGHT = "┤"; - private static final String BOTTOM_LEFT = "└"; - private static final String BOTTOM_RIGHT = "┘"; - private static final String CROSS = "┼"; - private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private static final int MAX_COL_WIDTH = 30; + private static final String VERTICAL = "│"; + private static final String HORIZONTAL = "─"; + private static final String TOP_LEFT = "┌"; + private static final String TOP_RIGHT = "┐"; + private static final String MID_LEFT = "├"; + private static final String MID_RIGHT = "┤"; + private static final String BOTTOM_LEFT = "└"; + private static final String BOTTOM_RIGHT = "┘"; + private static final String CROSS = "┼"; + private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); public static void print(ResultSet rs) throws SQLException { ResultSetMetaData meta = rs.getMetaData(); int colCount = meta.getColumnCount(); - + List columns = new ArrayList(); for (int i = 1; i <= colCount; i++) { - columns.add(new ColumnMeta( - meta.getColumnLabel(i), - meta.getColumnType(i), - meta.getPrecision(i) - )); + columns.add(new ColumnMeta(meta.getColumnLabel(i), meta.getColumnType(i), meta + .getPrecision(i))); } // 计算初始列宽 List widths = new ArrayList(); for (ColumnMeta col : columns) { - int width = Math.min( - Math.max(col.name.length(), getTypeWidth(col)), - MAX_COL_WIDTH - ); + int width = Math.min(Math.max(col.name.length(), getTypeWidth(col)), MAX_COL_WIDTH); widths.add(width); } - + List> rows = new ArrayList>(); while (rs.next()) { List row = new ArrayList(); for (int i = 0; i < colCount; i++) { Object value = rs.getObject(i + 1); String str = formatValue(value, widths.get(i)); - + int actualWidth = str.length(); if (actualWidth > widths.get(i)) { widths.set(i, Math.min(actualWidth, MAX_COL_WIDTH)); @@ -73,12 +68,12 @@ public static void print(ResultSet rs) throws SQLException { } rows.add(row); } - + StringBuilder fmt = new StringBuilder(VERTICAL); for (int w : widths) { fmt.append(" %-").append(w).append("s ").append(VERTICAL); } - + printDivider(widths, TOP_LEFT, CROSS, TOP_RIGHT); System.out.printf(fmt.toString() + "%n", getHeaders(columns)); printDivider(widths, MID_LEFT, CROSS, MID_RIGHT); @@ -90,8 +85,8 @@ public static void print(ResultSet rs) throws SQLException { private static class ColumnMeta { final String name; - final int type; - final int precision; + final int type; + final int precision; ColumnMeta(String name, int type, int precision) { this.name = name; @@ -102,17 +97,23 @@ private static class ColumnMeta { private static int getTypeWidth(ColumnMeta col) { switch (col.type) { - case Types.DATE: return 10; - case Types.TIMESTAMP: return 19; - case Types.INTEGER: return 11; + case Types.DATE: + return 10; + case Types.TIMESTAMP: + return 19; + case Types.INTEGER: + return 11; case Types.DECIMAL: - case Types.NUMERIC: return col.precision + 2; - default: return Math.min(col.precision, MAX_COL_WIDTH); + case Types.NUMERIC: + return col.precision + 2; + default: + return Math.min(col.precision, MAX_COL_WIDTH); } } private static String formatValue(Object value, int maxWidth) { - if (value == null) return "NULL"; + if (value == null) + return "NULL"; if (value instanceof byte[]) { return formatByteArray((byte[]) value, maxWidth); @@ -124,21 +125,23 @@ private static String formatValue(Object value, int maxWidth) { } private static String formatByteArray(byte[] bytes, int maxWidth) { - if (bytes.length == 0) return "[NULL]"; + if (bytes.length == 0) + return "[NULL]"; if (isPrintable(bytes)) { return truncateString(new String(bytes), maxWidth); } - + if (bytes.length <= 4) { return hexFormat(bytes, maxWidth); } - + return base64Format(bytes, maxWidth); } private static boolean isPrintable(byte[] bytes) { for (byte b : bytes) { - if (b < 0x20 || b > 0x7E) return false; + if (b < 0x20 || b > 0x7E) + return false; } return true; } @@ -151,7 +154,8 @@ private static String hexFormat(byte[] bytes, int maxWidth) { for (int i = 0; i < Math.min(bytes.length, maxBytes); i++) { sb.append(String.format("%02X ", bytes[i])); } - if (bytes.length > maxBytes) sb.append("..."); + if (bytes.length > maxBytes) + sb.append("..."); sb.setLength(sb.length() - 1); return sb.append("]").toString(); } @@ -162,7 +166,8 @@ private static String base64Format(byte[] bytes, int maxWidth) { } private static String truncateString(String str, int maxWidth) { - if (str.length() <= maxWidth) return str; + if (str.length() <= maxWidth) + return str; return str.substring(0, maxWidth - 3) + "..."; } @@ -182,5 +187,5 @@ private static void printDivider(List widths, String left, String mid, } System.out.println(sb); } - + } diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java index a0d85231..75a3e32d 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/TableTemplateManager.java @@ -84,10 +84,12 @@ public enum TableType { SECONDARY_PARTITIONED_KEY_RANGE_CELL_TTL, SECONDARY_PARTITIONED_KEY_RANGE_GEN_CELL_TTL); - public static List TIMESERIES_TABLES = Arrays.asList(NON_PARTITIONED_TIME_SERIES, - SINGLE_PARTITIONED_TIME_SERIES, - SECONDARY_PARTITIONED_TIME_RANGE_KEY, - SECONDARY_PARTITIONED_TIME_KEY_RANGE); + public static List TIMESERIES_TABLES = Arrays + .asList( + NON_PARTITIONED_TIME_SERIES, + SINGLE_PARTITIONED_TIME_SERIES, + SECONDARY_PARTITIONED_TIME_RANGE_KEY, + SECONDARY_PARTITIONED_TIME_KEY_RANGE); private static final Map SQL_TEMPLATES = new EnumMap( TableType.class);