From 31264727cf89fb845a0384b28793820dda56a988 Mon Sep 17 00:00:00 2001 From: stuBirdFly <1065492934@qq.com> Date: Thu, 24 Oct 2024 15:09:21 +0800 Subject: [PATCH] add native hbase test --- .../com/alipay/oceanbase/hbase/OHTable.java | 7 +- ...lyTest.java => HTableMultiCFTestBase.java} | 406 +++++++++++++++--- .../oceanbase/hbase/HTableTestBase.java | 160 +------ .../alipay/oceanbase/hbase/LoggerTest.java | 5 +- .../oceanbase/hbase/NativeHBaseTest.java | 46 ++ .../oceanbase/hbase/OHConnectionTest.java | 56 ++- .../oceanbase/hbase/OHMultiNamespaceTest.java | 82 ---- .../hbase/OHTableAdminInterfaceTest.java | 1 + .../oceanbase/hbase/OHTableClientTest.java | 26 +- .../hbase/OHTableClientTestLoadTest.java | 30 +- .../hbase/OHTableDeleteFamilyVersionTest.java | 348 --------------- .../oceanbase/hbase/OHTablePoolLoadTest.java | 31 +- .../oceanbase/hbase/OHTablePoolTest.java | 33 +- .../alipay/oceanbase/hbase/OHTableTest.java | 30 +- .../oceanbase/hbase/ObHTableTestUtil.java | 59 --- .../oceanbase/hbase/util/NativeHBaseUtil.java | 105 +++++ .../hbase/util/OHTableHotkeyThrottleUtil.java | 1 - .../hbase/util/ObHTableTestUtil.java | 148 +++++++ 18 files changed, 843 insertions(+), 731 deletions(-) rename src/test/java/com/alipay/oceanbase/hbase/{OHTableMultiColumnFamilyTest.java => HTableMultiCFTestBase.java} (67%) create mode 100644 src/test/java/com/alipay/oceanbase/hbase/NativeHBaseTest.java delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java delete mode 100644 src/test/java/com/alipay/oceanbase/hbase/ObHTableTestUtil.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/util/NativeHBaseUtil.java create mode 100644 src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java diff --git a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java index 4712527b..630b008f 100644 --- a/src/main/java/com/alipay/oceanbase/hbase/OHTable.java +++ b/src/main/java/com/alipay/oceanbase/hbase/OHTable.java @@ -1558,14 +1558,16 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, boolean includeStart, byte[] stop, boolean includeStop, boolean isReversed) { ObNewRange obNewRange = new ObNewRange(); - + ObBorderFlag obBorderFlag = new ObBorderFlag(); if (Arrays.equals(start, HConstants.EMPTY_BYTE_ARRAY)) { obNewRange.setStartKey(ObRowKey.getInstance(ObObj.getMin(), ObObj.getMin(), ObObj.getMin())); } else if (includeStart) { obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMin(), ObObj.getMin())); + obBorderFlag.setInclusiveStart(); } else { obNewRange.setStartKey(ObRowKey.getInstance(start, ObObj.getMax(), ObObj.getMax())); + obBorderFlag.unsetInclusiveStart(); } if (Arrays.equals(stop, HConstants.EMPTY_BYTE_ARRAY)) { @@ -1573,10 +1575,13 @@ private ObTableQuery buildObTableQuery(ObHTableFilter filter, byte[] start, ObObj.getMax())); } else if (includeStop) { obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMax(), ObObj.getMax())); + obBorderFlag.setInclusiveEnd(); } else { obNewRange.setEndKey(ObRowKey.getInstance(stop, ObObj.getMin(), ObObj.getMin())); + obBorderFlag.unsetInclusiveEnd(); } ObTableQuery obTableQuery = new ObTableQuery(); + obNewRange.setBorderFlag(obBorderFlag); if (isReversed) { obTableQuery.setScanOrder(ObScanOrder.Reverse); } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java similarity index 67% rename from src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java rename to src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java index a649e58e..fa28a810 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableMultiColumnFamilyTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableMultiCFTestBase.java @@ -22,30 +22,329 @@ import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.util.Bytes; import org.junit.*; import org.junit.rules.ExpectedException; -import java.io.IOException; import java.util.*; import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; -public class OHTableMultiColumnFamilyTest { +public abstract class HTableMultiCFTestBase { @Rule - public ExpectedException expectedException = ExpectedException.none(); + public ExpectedException expectedException = ExpectedException.none(); - protected HTableInterface hTable; + protected static Table multiCfHTable; - @Before - public void before() throws Exception { - hTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); - ((OHTableClient) hTable).init(); + public void tryPut(Table multiCfHTable, Put put) throws Exception { + multiCfHTable.put(put); + Thread.sleep(1); } - @After - public void finish() throws IOException { - hTable.close(); + @Test + public void testDeleteFamilyVerison() throws Exception { + String key1 = "scanKey1x"; + String key2 = "scanKey2x"; + String key3 = "scanKey3x"; + String column1 = "column1"; + String column2 = "column2"; + String column3 = "column3"; + String value1 = "value1"; + String value2 = "value2"; + String value3 = "value3"; + String family1 = "family_with_group1"; + String family2 = "family_with_group2"; + // delete previous data + Delete deleteKey1Family = new Delete(toBytes(key1)); + deleteKey1Family.deleteFamily(toBytes(family1)); + deleteKey1Family.deleteFamily(toBytes(family2)); + Delete deleteKey2Family = new Delete(toBytes(key2)); + deleteKey2Family.deleteFamily(toBytes(family1)); + deleteKey2Family.deleteFamily(toBytes(family2)); + Delete deleteKey3Family = new Delete(toBytes(key3)); + deleteKey3Family.deleteFamily(toBytes(family1)); + deleteKey3Family.deleteFamily(toBytes(family2)); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); + + long minTimeStamp = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp1 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp2 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp3 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp4 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp5 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp6 = System.currentTimeMillis(); + Thread.sleep(5); + long timeStamp7 = System.currentTimeMillis(); + Thread.sleep(5); + + Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); + putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, + toBytes(value1)); + + Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); + + Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); + putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, + toBytes(value1)); + + Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); + + Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); + putKey3Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); + + Put putKey3Fam1Column2Ts4 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts4.add(toBytes(family1), toBytes(column2), timeStamp4, toBytes(value1)); + + Put putKey2Fam1Column3Ts3 = new Put(toBytes(key2)); + putKey2Fam1Column3Ts3.add(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); + + tryPut(multiCfHTable, putKey1Fam1Column1MinTs); + tryPut(multiCfHTable, putKey3Fam1Column1Ts1); + tryPut(multiCfHTable, putKey1Fam1Column2MinTs); + tryPut(multiCfHTable, putKey1Fam1Column2Ts3); + tryPut(multiCfHTable, putKey2Fam1Column2Ts3); + tryPut(multiCfHTable, putKey2Fam1Column3Ts1); + tryPut(multiCfHTable, putKey3Fam1Column3Ts1); + tryPut(multiCfHTable, putKey3Fam1Column2Ts4); + tryPut(multiCfHTable, putKey2Fam1Column3Ts3); + + // test DeleteFamilyVersion single cf + Get get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + Result r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(2, r.raw().length); + + Delete delKey1MinTs = new Delete(toBytes(key1)); + delKey1MinTs.deleteFamilyVersion(toBytes(family1), minTimeStamp); + multiCfHTable.delete(delKey1MinTs); + + get = new Get(toBytes(key1)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(minTimeStamp); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts1 = new Delete(toBytes(key3)); + delKey3Ts1.deleteFamilyVersion(toBytes(family1), timeStamp1); + multiCfHTable.delete(delKey3Ts1); + + get = new Get(toBytes(key3)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp1); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Delete delKey2Ts3 = new Delete(toBytes(key2)); + delKey2Ts3.deleteFamilyVersion(family1.getBytes(), timeStamp3); + multiCfHTable.delete(delKey2Ts3); + + get = new Get(toBytes(key2)); + get.addFamily(toBytes(family1)); + get.setTimeStamp(timeStamp3); + get.setMaxVersions(10); + r = multiCfHTable.get(get); + Assert.assertEquals(0, r.raw().length); + + Scan scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.setMaxVersions(10); + ResultScanner scanner = multiCfHTable.getScanner(scan); + int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (key1.equals(Bytes.toString(kv.getRow()))) { + ++key1Cnt; + } else if (key2.equals(Bytes.toString(kv.getRow()))) { + ++key2Cnt; + } else { + ++key3Cnt; + } + } + } + Assert.assertEquals(1, key1Cnt); + Assert.assertEquals(1, key2Cnt); + Assert.assertEquals(1, key3Cnt); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); + + // test DeleteFamilyVersion multiple cf + Put putKey1Fam1Column3Ts4 = new Put(toBytes(key1)); + putKey1Fam1Column3Ts4.add(toBytes(family1), toBytes(column3), timeStamp4, toBytes(value3)); + + Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column2Ts2.add(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); + + Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); + putKey1Fam2Column3Ts2.add(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); + + Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); + putKey1Fam1Column2Ts1.add(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); + + Put putKey2Fam1Column2Ts5 = new Put(toBytes(key2)); + putKey2Fam1Column2Ts5.add(toBytes(family1), toBytes(column2), timeStamp5, toBytes(value2)); + + Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); + putKey2Fam2Column3Ts1.add(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); + + Put putKey2Fam1Column1Ts5 = new Put(toBytes(key2)); + putKey2Fam1Column1Ts5.add(toBytes(family1), toBytes(column1), timeStamp5, toBytes(value1)); + + Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); + putKey2Fam2Column1Ts3.add(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); + + Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value2)); + + Put putKey3Fam2Column3Ts7 = new Put(toBytes(key3)); + putKey3Fam2Column3Ts7 + .add(toBytes(family2), toBytes(column3), timeStamp7, toBytes(value1)); + + Put putKey3Fam2Column1Ts7 = new Put(toBytes(key3)); + putKey3Fam2Column1Ts7 + .add(toBytes(family2), toBytes(column1), timeStamp7, toBytes(value2)); + + Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); + putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); + + tryPut(multiCfHTable, putKey1Fam1Column3Ts4); + tryPut(multiCfHTable, putKey1Fam2Column2Ts2); + tryPut(multiCfHTable, putKey1Fam2Column3Ts2); + tryPut(multiCfHTable, putKey1Fam1Column2Ts1); + tryPut(multiCfHTable, putKey2Fam1Column2Ts5); + tryPut(multiCfHTable, putKey2Fam2Column3Ts1); + tryPut(multiCfHTable, putKey2Fam1Column1Ts5); + tryPut(multiCfHTable, putKey2Fam2Column1Ts3); + tryPut(multiCfHTable, putKey3Fam1Column2Ts6); + tryPut(multiCfHTable, putKey3Fam2Column3Ts7); + tryPut(multiCfHTable, putKey3Fam2Column1Ts7); + tryPut(multiCfHTable, putKey3Fam1Column2Ts2); + + Get getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = multiCfHTable.get(getKey1); + Assert.assertEquals(4, r.raw().length); + + Get getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = multiCfHTable.get(getKey2); + Assert.assertEquals(4, r.raw().length); + + Get getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = multiCfHTable.get(getKey3); + Assert.assertEquals(4, r.raw().length); + + Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family1), timeStamp4); + delKey1Ts_6_2.deleteFamilyVersion(toBytes(family2), timeStamp2); + multiCfHTable.delete(delKey1Ts_6_2); + + getKey1 = new Get(toBytes(key1)); + getKey1.addFamily(toBytes(family1)); + getKey1.addFamily(toBytes(family2)); + getKey1.setMaxVersions(10); + r = multiCfHTable.get(getKey1); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp1, kv.getTimestamp()); + } + + Delete delKey2Ts_5_3 = new Delete(toBytes(key2)); + delKey2Ts_5_3.deleteFamilyVersion(toBytes(family1), timeStamp5); + delKey2Ts_5_3.deleteFamilyVersion(toBytes(family2), timeStamp3); + multiCfHTable.delete(delKey2Ts_5_3); + + getKey2 = new Get(toBytes(key2)); + getKey2.addFamily(toBytes(family1)); + getKey2.addFamily(toBytes(family2)); + getKey2.setMaxVersions(10); + r = multiCfHTable.get(getKey2); + Assert.assertEquals(0, r.raw().length); + + Delete delKey3Ts_2_7 = new Delete(toBytes(key3)); + delKey3Ts_2_7.deleteFamilyVersion(toBytes(family1), timeStamp2); + delKey3Ts_2_7.deleteFamilyVersion(toBytes(family2), timeStamp7); + multiCfHTable.delete(delKey3Ts_2_7); + + getKey3 = new Get(toBytes(key3)); + getKey3.addFamily(toBytes(family1)); + getKey3.addFamily(toBytes(family2)); + getKey3.setMaxVersions(10); + r = multiCfHTable.get(getKey3); + Assert.assertEquals(1, r.raw().length); + for (KeyValue kv : r.raw()) { + Assert.assertEquals(timeStamp6, kv.getTimestamp()); + } + + scan = new Scan(); + scan.setStartRow(toBytes(key1)); + scan.setStopRow("scanKey4x".getBytes()); + scan.addFamily(toBytes(family1)); + scan.addFamily(toBytes(family2)); + scan.setMaxVersions(10); + scanner = multiCfHTable.getScanner(scan); + int ts1Cnt = 0, ts9Cnt = 0; + for (Result result : scanner) { + for (KeyValue kv : result.raw()) { + if (kv.getTimestamp() == timeStamp1) { + ++ts1Cnt; + } else if (kv.getTimestamp() == timeStamp6) { + ++ts9Cnt; + } + } + } + Assert.assertEquals(1, ts1Cnt); + Assert.assertEquals(1, ts9Cnt); + + multiCfHTable.delete(deleteKey1Family); + multiCfHTable.delete(deleteKey2Family); + multiCfHTable.delete(deleteKey3Family); } @Test @@ -84,10 +383,10 @@ public void testMultiColumnFamilyBatch() throws Exception { delete.deleteColumns(family1, family1_column1); delete.deleteColumns(family2, family2_column1); batchLsit.add(delete); - hTable.batch(batchLsit); + multiCfHTable.batch(batchLsit); // f1c2 f1c3 f2c2 f3c1 Get get = new Get(toBytes("Key1")); - Result result = hTable.get(get); + Result result = multiCfHTable.get(get); KeyValue[] keyValues = result.raw(); assertEquals(4, keyValues.length); assertFalse(result.containsColumn(family1, family1_column1)); @@ -109,9 +408,9 @@ public void testMultiColumnFamilyBatch() throws Exception { delete.deleteColumns(family3, family3_column1); batchLsit.add(delete); // null - hTable.batch(batchLsit); + multiCfHTable.batch(batchLsit); get = new Get(toBytes("Key2")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(3, keyValues.length); batchLsit.clear(); @@ -130,9 +429,9 @@ public void testMultiColumnFamilyBatch() throws Exception { delete.deleteColumn(family1, family1_column2); delete.deleteColumn(family2, family2_column1); batchLsit.add(delete); - hTable.batch(batchLsit); + multiCfHTable.batch(batchLsit); get = new Get(toBytes("Key3")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(6, keyValues.length); @@ -142,10 +441,10 @@ public void testMultiColumnFamilyBatch() throws Exception { delete.deleteColumns(family2, family2_column1); delete.deleteFamily(family3); batchLsit.add(delete); - hTable.batch(batchLsit); + multiCfHTable.batch(batchLsit); get = new Get(toBytes("Key4")); get.setMaxVersions(10); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(6, keyValues.length); @@ -166,7 +465,7 @@ public void testMultiColumnFamilyBatch() throws Exception { put.add(family3, family3_column1, family3_value); batchLsit.add(put); } - hTable.batchCallback(batchLsit, new Batch.Callback() { + multiCfHTable.batchCallback(batchLsit, new Batch.Callback() { @Override public void update(byte[] region, byte[] row, MutationResult result) { updateCounter[0]++; @@ -210,14 +509,13 @@ public void testMultiColumnFamilyPut() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } - hTable.flushCommits(); Scan scan = new Scan(); scan.setStartRow(toBytes("Key")); scan.setStopRow(toBytes("Kf")); - ResultScanner scanner = hTable.getScanner(scan); + ResultScanner scanner = multiCfHTable.getScanner(scan); int count = 0; for (Result result : scanner) { @@ -270,14 +568,13 @@ public void testMultiColumnFamilyAppend() throws Exception { append.add(family2, family2_column1, family2_value); append.add(family2, family2_column2, family2_value); append.add(family3, family3_column1, family3_value); - hTable.append(append); + multiCfHTable.append(append); } - hTable.flushCommits(); Scan scan = new Scan(); scan.setStartRow(toBytes("Key")); scan.setStopRow(toBytes("Kf")); - ResultScanner scanner = hTable.getScanner(scan); + ResultScanner scanner = multiCfHTable.getScanner(scan); int count = 0; for (Result result : scanner) { @@ -330,14 +627,14 @@ public void testMultiColumnFamilyReverseScan() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } Scan scan = new Scan(); scan.addFamily(family1); scan.addFamily(family2); scan.setReversed(true); - ResultScanner scanner2 = hTable.getScanner(scan); + ResultScanner scanner2 = multiCfHTable.getScanner(scan); for (Result result : scanner2) { KeyValue[] keyValues = result.raw(); @@ -387,7 +684,7 @@ public void testMultiColumnFamilyScanWithColumns() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } Scan scan = new Scan(); @@ -395,7 +692,7 @@ public void testMultiColumnFamilyScanWithColumns() throws Exception { scan.setStopRow(toBytes("Kf")); scan.addColumn(family1, family1_column1); scan.addColumn(family2, family2_column1); - ResultScanner scanner = hTable.getScanner(scan); + ResultScanner scanner = multiCfHTable.getScanner(scan); for (Result result : scanner) { KeyValue[] keyValues = result.raw(); @@ -419,7 +716,7 @@ public void testMultiColumnFamilyScanWithColumns() throws Exception { scan.addColumn(family1, family1_column3); scan.addColumn(family2, family2_column1); scan.addColumn(family2, family2_column2); - scanner = hTable.getScanner(scan); + scanner = multiCfHTable.getScanner(scan); for (Result result : scanner) { KeyValue[] keyValues = result.raw(); @@ -441,7 +738,7 @@ public void testMultiColumnFamilyScanWithColumns() throws Exception { scan.addFamily(family1); scan.addFamily(family2); - scanner = hTable.getScanner(scan); + scanner = multiCfHTable.getScanner(scan); for (Result result : scanner) { KeyValue[] keyValues = result.raw(); @@ -463,7 +760,7 @@ public void testMultiColumnFamilyScanWithColumns() throws Exception { scan.addFamily(family1); scan.addFamily(family3); - scanner = hTable.getScanner(scan); + scanner = multiCfHTable.getScanner(scan); for (Result result : scanner) { KeyValue[] keyValues = result.raw(); @@ -516,7 +813,7 @@ public void testMultiColumnFamilyScanWithFilter() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } PrefixFilter filter = new PrefixFilter(toBytes("Key1")); @@ -524,7 +821,7 @@ public void testMultiColumnFamilyScanWithFilter() throws Exception { scan.setStartRow(toBytes("Key")); scan.setStopRow(toBytes("Kf")); scan.setFilter(filter); - ResultScanner scanner = hTable.getScanner(scan); + ResultScanner scanner = multiCfHTable.getScanner(scan); // Key1, Key10, Key11, Key12, Key13, Key14, Key15, Key16, Key17, Key18, Key19 int count = 0; @@ -579,14 +876,13 @@ public void testMultiColumnFamilyGet() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } - hTable.flushCommits(); // get with empty family // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 Get get = new Get(toBytes("Key1")); - Result result = hTable.get(get); + Result result = multiCfHTable.get(get); KeyValue[] keyValues = result.raw(); long timestamp = keyValues[0].getTimestamp(); for (int i = 1; i < keyValues.length; ++i) { @@ -604,7 +900,7 @@ public void testMultiColumnFamilyGet() throws Exception { get2.addColumn(family1, family1_column1); get2.addColumn(family2, family2_column1); get2.addColumn(family2, family2_column2); - Result result2 = hTable.get(get2); + Result result2 = multiCfHTable.get(get2); keyValues = result2.raw(); timestamp = keyValues[0].getTimestamp(); for (int i = 1; i < keyValues.length; ++i) { @@ -622,7 +918,7 @@ public void testMultiColumnFamilyGet() throws Exception { get3.addFamily(family1); get3.addColumn(family2, family2_column1); get3.addColumn(family2, family2_column2); - Result result3 = hTable.get(get3); + Result result3 = multiCfHTable.get(get3); keyValues = result3.raw(); timestamp = keyValues[0].getTimestamp(); for (int i = 1; i < keyValues.length; ++i) { @@ -657,24 +953,24 @@ public void testMultiColumnFamilyDelete() throws Exception { for (int i = 0; i < rows; ++i) { Put put = new Put(toBytes("Key" + i)); Delete delete = new Delete(toBytes("Key" + i)); - hTable.delete(delete); + multiCfHTable.delete(delete); put.add(family1, family1_column1, family1_value); put.add(family1, family1_column2, family1_value); put.add(family1, family1_column3, family1_value); put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 Delete delete = new Delete(toBytes("Key1")); delete.deleteColumns(family1, family1_column1); delete.deleteColumns(family2, family2_column1); - hTable.delete(delete); + multiCfHTable.delete(delete); // f1c2 f1c3 f2c2 f3c1 Get get = new Get(toBytes("Key1")); - Result result = hTable.get(get); + Result result = multiCfHTable.get(get); KeyValue[] keyValues = result.raw(); assertEquals(4, keyValues.length); assertFalse(result.containsColumn(family1, family1_column1)); @@ -694,9 +990,9 @@ public void testMultiColumnFamilyDelete() throws Exception { delete.deleteFamily(family1); delete.deleteFamily(family2); // f3c1 - hTable.delete(delete); + multiCfHTable.delete(delete); get = new Get(toBytes("Key2")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(1, keyValues.length); @@ -704,19 +1000,19 @@ public void testMultiColumnFamilyDelete() throws Exception { delete = new Delete(toBytes("Key3")); delete.deleteFamily(family1); delete.deleteColumns(family2, family2_column1); - hTable.delete(delete); + multiCfHTable.delete(delete); // f2c2 f3c1 get = new Get(toBytes("Key3")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(2, keyValues.length); // f1c1 f1c2 f1c3 f2c1 f2c2 f3c1 delete = new Delete(toBytes("Key4")); - hTable.delete(delete); + multiCfHTable.delete(delete); // null get = new Get(toBytes("Key4")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(0, keyValues.length); @@ -725,10 +1021,10 @@ public void testMultiColumnFamilyDelete() throws Exception { delete.deleteColumns(family1, family1_column2); delete.deleteColumns(family1, family1_column3); delete.deleteColumns(family3, family3_column1); - hTable.delete(delete); + multiCfHTable.delete(delete); // null get = new Get(toBytes("Key5")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(3, keyValues.length); @@ -740,15 +1036,15 @@ public void testMultiColumnFamilyDelete() throws Exception { put.add(family2, family2_column1, family2_value); put.add(family2, family2_column2, family2_value); put.add(family3, family3_column1, family3_value); - hTable.put(put); + multiCfHTable.put(put); } delete = new Delete(toBytes("Key6")); delete.deleteColumn(family1, family1_column2); delete.deleteColumn(family2, family2_column1); - hTable.delete(delete); + multiCfHTable.delete(delete); get = new Get(toBytes("Key6")); - result = hTable.get(get); + result = multiCfHTable.get(get); keyValues = result.raw(); assertEquals(6, keyValues.length); diff --git a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java index 9aa2e1f7..4131a572 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java +++ b/src/test/java/com/alipay/oceanbase/hbase/HTableTestBase.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.*; +import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.junit.Assert; @@ -43,12 +44,12 @@ import static org.apache.hadoop.hbase.util.Bytes.toBytes; import static org.junit.Assert.*; -public abstract class HTableTestBase { +public abstract class HTableTestBase extends HTableMultiCFTestBase { @Rule public ExpectedException expectedException = ExpectedException.none(); - protected Table hTable; + protected static Table hTable; @Test public void testTableGroup() throws IOError, IOException { @@ -112,11 +113,11 @@ PRIMARY KEY (`K`, `Q`, `T`) Assert.assertEquals(column1, Bytes.toString(keyValue.getQualifier())); Assert.assertEquals(timestamp, keyValue.getTimestamp()); Assert.assertEquals(value + "1", Bytes.toString(keyValue.getValue())); - System.out.println("rowKey: " + new String(keyValue.getRow()) + " family :" - + new String(keyValue.getFamily()) + " columnQualifier:" - + new String(keyValue.getQualifier()) + " timestamp:" - + keyValue.getTimestamp() + " value:" - + new String(keyValue.getValue())); + System.out.println( + "rowKey: " + new String(keyValue.getRow()) + " family :" + new String( + keyValue.getFamily()) + " columnQualifier:" + new String( + keyValue.getQualifier()) + " timestamp:" + keyValue.getTimestamp() + " value:" + new String( + keyValue.getValue())); } } } @@ -132,9 +133,6 @@ private void testBasic(String family) throws Exception { String column2 = "putColumn2"; String value = "value"; long timestamp = System.currentTimeMillis(); - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(toBytes(key)); put.add(family.getBytes(), column1.getBytes(), timestamp, toBytes(value)); @@ -168,7 +166,7 @@ private void testBasic(String family) throws Exception { r = hTable.get(get); Assert.assertEquals(1, r.raw().length); - delete = new Delete(key.getBytes()); + Delete delete = new Delete(key.getBytes()); delete.deleteFamily(family.getBytes()); hTable.delete(delete); @@ -406,27 +404,7 @@ public void testMultiPartitionDel() throws IOException { String column1 = "column1"; String column2 = "column2"; String column3 = "column3"; - String value = "value"; String family = "familyPartition"; - // delete - { - List deletes = new ArrayList(); - for (String key : keys) { - Delete del = new Delete(Bytes.toBytes(key)); - del.deleteColumns(toBytes(family), toBytes(column1)); - del.deleteColumns(toBytes(family), toBytes(column2), System.currentTimeMillis()); - deletes.add(del); - } - - for (String key : keys) { - // del same k, q, t - Delete del = new Delete(Bytes.toBytes(key)); - del.deleteColumn(toBytes(family), toBytes(column3), 100L); - del.deleteColumn(toBytes(family), toBytes(column3), 100L); - deletes.add(del); - } - hTable.delete(deletes); - } // get { List gets = new ArrayList(); @@ -459,7 +437,6 @@ public void testFilter() throws Exception { String column2 = "def"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "family1"; Delete deleteKey1Family = new Delete(toBytes(key1)); deleteKey1Family.deleteFamily(toBytes(family)); @@ -467,9 +444,6 @@ public void testFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -498,8 +472,6 @@ public void testFilter() throws Exception { Result r; ColumnPrefixFilter filter; - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); tryPut(hTable, putKey1Column1Value1); tryPut(hTable, putKey1Column1Value2); tryPut(hTable, putKey1Column1Value1); @@ -971,9 +943,6 @@ public void testRowRangeFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1227,7 +1196,6 @@ public void testColumnRangeFilter() throws Exception { String column2 = "def"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "family1"; Delete deleteKey1Family = new Delete(toBytes(key1)); deleteKey1Family.deleteFamily(toBytes(family)); @@ -1235,9 +1203,6 @@ public void testColumnRangeFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1262,8 +1227,6 @@ public void testColumnRangeFilter() throws Exception { Put putKey2Column2Value1 = new Put(toBytes(key2)); putKey2Column2Value1.add(toBytes(family), toBytes(column2), toBytes(value1)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); tryPut(hTable, putKey1Column1Value1); tryPut(hTable, putKey1Column1Value2); tryPut(hTable, putKey1Column1Value1); @@ -1295,7 +1258,7 @@ public void testColumnRangeFilter() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 3); + Assert.assertEquals(3, res_count); scanner.close(); scan = new Scan(); @@ -1317,7 +1280,7 @@ public void testColumnRangeFilter() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 3); + Assert.assertEquals(3, res_count); scanner.close(); scan = new Scan(); @@ -1339,7 +1302,7 @@ public void testColumnRangeFilter() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); // MultipleColumnPrefixFilter @@ -1363,12 +1326,13 @@ public void testColumnRangeFilter() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); scan = new Scan(); scan.addFamily(family.getBytes()); scan.setMaxVersions(10); + // 和原生hbase不一致，已知 range = new byte[][] { Bytes.toBytes("de"), Bytes.toBytes("bg"), Bytes.toBytes("nc"), Bytes.toBytes("aa"), Bytes.toBytes("abcd"), Bytes.toBytes("dea"), }; iFilter = new MultipleColumnPrefixFilter(range); @@ -1387,7 +1351,7 @@ public void testColumnRangeFilter() throws Exception { res_count += 1; } } - Assert.assertEquals(res_count, 6); + Assert.assertEquals(6, res_count); scanner.close(); } @@ -1407,9 +1371,6 @@ public void testFilterNullRange() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1489,9 +1450,6 @@ public void testFilter2() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -1796,9 +1754,6 @@ public void testFuzzyRowFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -2083,9 +2038,6 @@ public void testFirstKeyValueMatchingQualifiersFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -2232,9 +2184,6 @@ public void testGetFilter() throws Exception { Delete deleteKey2Family = new Delete(toBytes(key2)); deleteKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -2894,11 +2843,6 @@ public void testScanWithObParams() throws Exception { Delete deleteKey4Family = new Delete(toBytes(key4)); deleteKey4Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteKey4Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -2998,12 +2942,6 @@ public void testScanSessionClean() throws Exception { Delete deleteKey5Family = new Delete(toBytes(key5)); deleteKey5Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteKey4Family); - hTable.delete(deleteKey5Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -3067,7 +3005,6 @@ public void testGet() throws Exception { String column2 = "column2"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "family1"; // delete previous data @@ -3082,12 +3019,6 @@ public void testGet() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -3364,7 +3295,6 @@ public void testScan() throws Exception { String column2 = "column2"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "family1"; // delete previous data @@ -3379,12 +3309,6 @@ public void testScan() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -3866,12 +3790,6 @@ public void testReversedScan() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -4265,7 +4183,6 @@ public void testPartitionScan() throws Exception { String column2 = "column2"; String value1 = "value1"; String value2 = "value2"; - String value3 = "value3"; String family = "partitionFamily1"; // delete previous data @@ -4280,12 +4197,6 @@ public void testPartitionScan() throws Exception { Delete deleteZKey2Family = new Delete(toBytes(zKey2)); deleteZKey2Family.deleteFamily(toBytes(family)); - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - hTable.delete(deleteZKey1Family); - hTable.delete(deleteZKey2Family); - Put putKey1Column1Value1 = new Put(toBytes(key1)); putKey1Column1Value1.add(toBytes(family), toBytes(column1), toBytes(value1)); @@ -4553,9 +4464,6 @@ public void testCheckAndPut() throws IOException, InterruptedException { String column = "checkAndPut"; String value = "value"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Get get = new Get(key.getBytes()); get.setMaxVersions(Integer.MAX_VALUE); get.addColumn(family.getBytes(), column.getBytes()); @@ -4605,15 +4513,12 @@ public void testCheckAndDelete() throws IOException { String column2 = "checkAndDeleteColumn2"; String value = "value"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(key.getBytes()); put.add(family.getBytes(), column.getBytes(), value.getBytes()); hTable.put(put); // check delete column - delete = new Delete(key.getBytes()); + Delete delete = new Delete(key.getBytes()); delete.deleteColumn(family.getBytes(), column.getBytes()); boolean ret = hTable.checkAndDelete(key.getBytes(), family.getBytes(), column.getBytes(), value.getBytes(), delete); @@ -4703,9 +4608,6 @@ public void testCheckAndMutate() throws IOException { String value1 = "value1"; String value2 = "value2"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); long t = System.currentTimeMillis(); // put @@ -4845,9 +4747,6 @@ public void testCheckAndMutate() throws IOException { public void testAppend() throws IOException { String column = "appendColumn"; String key = "appendKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); // append an absent column is not supported yet // Append append = new Append(key.getBytes()); @@ -4874,9 +4773,6 @@ public void testAppend() throws IOException { public void testIncrement() throws IOException { String column = "incrementColumn"; String key = "incrementKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); // increment an absent column is not supported yet // Increment increment = new Increment(key.getBytes()); @@ -4922,9 +4818,6 @@ public void testIncrement() throws IOException { public void testExist() throws IOException { String column = "existColumn"; String key = "existKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); Get get = new Get(key.getBytes()); get.addFamily("family1".getBytes()); @@ -4948,8 +4841,6 @@ public void testExist() throws IOException { get.setTimeStamp(timestamp + 1); Assert.assertFalse(hTable.exists(get)); - - hTable.delete(delete); } @Ignore @@ -4959,7 +4850,6 @@ public void testMutateRow() throws IOException { String column2 = "mutationRowColumn2"; String key = "mutationRowKey"; String family1 = "family1"; - String family2 = "family2"; String value = "value"; Delete deleteFamily = new Delete(key.getBytes()); @@ -5039,9 +4929,6 @@ public void testQualifyNull() throws Exception { String value = "value"; String value1 = "value1"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(key.getBytes()); put.add(family.getBytes(), null, value.getBytes()); hTable.put(put); @@ -5129,6 +5016,8 @@ public void testFamilyBlank() throws Exception { fail(); } catch (IllegalArgumentException e) { Assert.assertTrue(e.getMessage().contains("family is blank")); + } catch (NoSuchColumnFamilyException e) { + Assert.assertTrue(e.getMessage().contains("does not exist")); } Put put = new Put(key.getBytes()); put.add(null, null, value.getBytes()); @@ -5166,9 +5055,6 @@ public void testScannerMultiVersion() throws Exception { String column = "column"; String value1 = "value1"; String family = "family1"; - Delete delete = new Delete(key.getBytes()); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); Put put = new Put(key.getBytes()); put.add(family.getBytes(), Bytes.toBytes(column), value.getBytes()); hTable.put(put); @@ -5439,9 +5325,6 @@ public static byte[] toByteArray(long value) { public void testIncrementConcurrency() throws Exception { String column = "incrementColumn"; String key = "incrementKey"; - Delete delete = new Delete(key.getBytes()); - delete.deleteColumns("family1".getBytes(), column.getBytes()); - hTable.delete(delete); for (int i = 0; i < 100; i++) { Increment increment = new Increment(key.getBytes()); @@ -5487,16 +5370,12 @@ public void testFilterSpecialValue() throws IOException { byte[] columnBytes = specialBytes; byte[] valueBytes = specialBytes; - Delete delete = new Delete(keyBytes); - delete.deleteFamily(family.getBytes()); - hTable.delete(delete); - Put put = new Put(keyBytes); put.add(family.getBytes(), columnBytes, valueBytes); hTable.put(put); // check delete column - delete = new Delete(keyBytes); + Delete delete = new Delete(keyBytes); delete.deleteColumn(family.getBytes(), columnBytes); boolean ret = hTable.checkAndDelete(keyBytes, family.getBytes(), columnBytes, valueBytes, delete); @@ -5532,6 +5411,5 @@ public void testFilterSpecialValue() throws IOException { result = hTable.get(get); Assert.assertEquals(1, result.raw().length); Assert.assertArrayEquals(keyBytes, result.getRow()); - } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java b/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java index d1909ea0..0ab0617b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/LoggerTest.java @@ -17,12 +17,13 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Increment; import org.apache.hadoop.hbase.client.Scan; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.support.membermodification.MemberModifier; @@ -42,7 +43,7 @@ public class LoggerTest { HTableInterface hTableMock; - @Before + @BeforeClass public void setup() throws IOException { Configuration c = ObHTableTestUtil.newConfiguration(); c.set("rs.list.acquire.read.timeout", "10000"); diff --git a/src/test/java/com/alipay/oceanbase/hbase/NativeHBaseTest.java b/src/test/java/com/alipay/oceanbase/hbase/NativeHBaseTest.java new file mode 100644 index 00000000..545d95b9 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/NativeHBaseTest.java @@ -0,0 +1,46 @@ +package com.alipay.oceanbase.hbase; + +import com.alipay.oceanbase.hbase.util.NativeHBaseUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class NativeHBaseTest extends HTableTestBase { + + static Admin admin; + static TableName tableName1 = TableName.valueOf("test"); + static TableName tableName2 = TableName.valueOf("test_multi_cf"); + + static { + try { + admin = NativeHBaseUtil.getAdmin(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @BeforeClass + public static void setup() throws IOException { + hTable = NativeHBaseUtil.getTable(tableName1); + multiCfHTable = NativeHBaseUtil.getTable(tableName2); + } + + @Before + public void cleanData() throws IOException { + admin.disableTable(tableName1); + admin.disableTable(tableName2); + admin.truncateTable(tableName1, true); + admin.truncateTable(tableName2, true); + } + + @AfterClass + public static void finish() throws IOException { + hTable.close(); + multiCfHTable.close(); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java index b40b23ea..79848a1e 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHConnectionTest.java @@ -18,16 +18,16 @@ package com.alipay.oceanbase.hbase; import com.alipay.oceanbase.hbase.exception.FeatureNotSupportedException; -import com.alipay.oceanbase.hbase.util.OHBufferedMutatorImpl; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; -import org.junit.Assert; -import org.junit.Test; +import org.junit.*; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.*; @@ -35,10 +35,11 @@ import static com.alipay.oceanbase.hbase.constants.OHConstants.SOCKET_TIMEOUT; import static org.apache.hadoop.hbase.ipc.RpcClient.SOCKET_TIMEOUT_CONNECT; import static org.apache.hadoop.hbase.util.Bytes.toBytes; +import static org.junit.Assert.*; public class OHConnectionTest { - protected Table hTable; - protected Connection connection; + protected static Table hTable; + protected Connection connection; @Test public void testConnectionBySet() throws Exception { @@ -54,6 +55,7 @@ public void testConnectionBySet() throws Exception { TableName tableName = TableName.valueOf("test"); hTable = connection.getTable(tableName); testBasic(); + hTable.close(); } @Test @@ -66,6 +68,50 @@ public void testConnectionByXml() throws Exception { TableName tableName = TableName.valueOf("test"); hTable = connection.getTable(tableName); testBasic(); + hTable.close(); + } + + @BeforeClass + public static void before() throws Exception { + // use self-defined namespace "n1" + hTable = ObHTableTestUtil.newOHTableClient("n1:test"); + ((OHTableClient) hTable).init(); + } + + @AfterClass + public static void finish() throws IOException { + hTable.close(); + } + + @Test + public void testRefreshTableEntry() throws Exception { + ((OHTableClient) hTable).refreshTableEntry("family1", false); + ((OHTableClient) hTable).refreshTableEntry("family1", true); + } + + @Test + public void testNew() throws Exception { + OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient("n1:test"); + hTable2.init(); + hTable2.getConfiguration().set("rs.list.acquire.read.timeout", "10000"); + + assertTrue(hTable2.isAutoFlush()); + hTable2.setAutoFlush(false); + assertFalse(hTable2.isAutoFlush()); + hTable2.setAutoFlush(true, true); + assertTrue(hTable2.isAutoFlush()); + hTable2.setWriteBufferSize(10000000L); + assertEquals(10000000L, hTable2.getWriteBufferSize()); + assertEquals("n1:test", hTable2.getTableNameString()); + assertEquals("n1:test", new String(hTable2.getTableName())); + hTable2.flushCommits(); + hTable2.close(); + assertTrue(true); + } + + @After + public void after() throws IOException { + hTable.close(); } private void testBasic() throws Exception { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java deleted file mode 100644 index 9c585c12..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/OHMultiNamespaceTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/*- - * #%L - * OBKV HBase Client Framework - * %% - * Copyright (C) 2022 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.hbase; - -import org.apache.hadoop.hbase.client.Get; -import org.junit.*; - -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class OHMultiNamespaceTest extends HTableTestBase { - @Before - public void before() throws Exception { - // use self-defined namespace "n1" - hTable = ObHTableTestUtil.newOHTableClient("n1:test"); - ((OHTableClient) hTable).init(); - } - - @After - public void finish() throws IOException { - hTable.close(); - } - - @Test - public void testRefreshTableEntry() throws Exception { - ((OHTableClient) hTable).refreshTableEntry("family1", false); - ((OHTableClient) hTable).refreshTableEntry("family1", true); - } - - @Test - public void testGetColumnFamilyNotExists() throws Exception { - /** family 不存在时提示不友好，*/ - Get get = new Get(("key_c_f").getBytes()); - get.addFamily("family_not_exists".getBytes()); - expectedException.expect(IOException.class); - expectedException.expectMessage("query table:n1:test family family_not_exists error."); - hTable.get(get); - } - - @Test - public void testNew() throws Exception { - OHTableClient hTable2 = ObHTableTestUtil.newOHTableClient("n1:test"); - hTable2.init(); - hTable2.getConfiguration().set("rs.list.acquire.read.timeout", "10000"); - - assertTrue(hTable2.isAutoFlush()); - hTable2.setAutoFlush(false); - assertFalse(hTable2.isAutoFlush()); - hTable2.setAutoFlush(true, true); - assertTrue(hTable2.isAutoFlush()); - hTable2.setWriteBufferSize(10000000L); - assertEquals(10000000L, hTable2.getWriteBufferSize()); - assertEquals("n1:test", hTable2.getTableNameString()); - assertEquals("n1:test", new String(hTable2.getTableName())); - hTable2.flushCommits(); - hTable2.close(); - assertTrue(true); - } - - @After - public void after() throws IOException { - hTable.close(); - } -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java index 380125f1..6d53616b 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableAdminInterfaceTest.java @@ -17,6 +17,7 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.util.Pair; import org.junit.Assert; diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java index 59d4aac5..01c4057f 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTest.java @@ -17,25 +17,33 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import org.junit.*; -import java.io.IOException; +import java.util.LinkedList; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class OHTableClientTest extends HTableTestBase { - @Before - public void before() throws Exception { + @BeforeClass + public static void before() throws Exception { hTable = ObHTableTestUtil.newOHTableClient("test"); // hTable = ObHTableTestUtil.newOHTableClient("n1:test"); ((OHTableClient) hTable).init(); + multiCfHTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); + ((OHTableClient) multiCfHTable).init(); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); } - @After - public void finish() throws IOException { - hTable.close(); + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -67,8 +75,10 @@ public void testNew() throws Exception { assertTrue(true); } - @After - public void after() throws IOException { + @AfterClass + public static void finish() throws Exception { hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java index 2c636436..ed9df2fa 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableClientTestLoadTest.java @@ -17,24 +17,36 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.rpc.exception.ObTableNotExistException; import org.apache.hadoop.hbase.client.Delete; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.util.LinkedList; +import java.util.List; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_ENABLE; import static com.alipay.oceanbase.hbase.constants.OHConstants.HBASE_HTABLE_TEST_LOAD_SUFFIX; public class OHTableClientTestLoadTest extends HTableTestBase { - @Before - public void before() throws Exception { + @BeforeClass + public static void before() throws Exception { hTable = ObHTableTestUtil.newOHTableClient("test"); ((OHTableClient) hTable).init(); hTable.getConfiguration().set(HBASE_HTABLE_TEST_LOAD_ENABLE, "true"); + multiCfHTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); + ((OHTableClient) multiCfHTable).init(); + multiCfHTable.getConfiguration().set(HBASE_HTABLE_TEST_LOAD_ENABLE, "true"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -53,9 +65,11 @@ public void test_refresh_table_entry() throws Exception { } - @After - public void after() throws IOException { + @AfterClass + public static void after() throws Exception { hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); } @Test diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java deleted file mode 100644 index 2ed4c190..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableDeleteFamilyVersionTest.java +++ /dev/null @@ -1,348 +0,0 @@ -package com.alipay.oceanbase.hbase; - -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.util.Bytes; -import org.junit.*; -import org.junit.rules.ExpectedException; - -import java.io.IOException; - -import static org.apache.hadoop.hbase.util.Bytes.toBytes; - -public class OHTableDeleteFamilyVersionTest { - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - protected HTableInterface hTable; - private static final String key1 = "scanKey1x"; - private static final String key2 = "scanKey2x"; - private static final String key3 = "scanKey3x"; - private static final String column1 = "column1"; - private static final String column2 = "column2"; - private static final String column3 = "column3"; - private static final String value1 = "value1"; - private static final String value2 = "value2"; - private static final String value3 = "value3"; - private static final String family1 = "family_with_group1"; - private static final String family2 = "family_with_group2"; - - @Before - public void before() throws Exception { - hTable = ObHTableTestUtil.newOHTableClient("test_multi_cf"); - ((OHTableClient) hTable).init(); - } - - @After - public void finish() throws IOException { - hTable.close(); - } - - public void tryPut(Table hTable, Put put) throws Exception { - hTable.put(put); - Thread.sleep(1); - } - - @Test - public void testDeleteFamilyVerison() throws Exception { - // delete previous data - Delete deleteKey1Family = new Delete(toBytes(key1)); - deleteKey1Family.deleteFamily(toBytes(family1)); - deleteKey1Family.deleteFamily(toBytes(family2)); - Delete deleteKey2Family = new Delete(toBytes(key2)); - deleteKey2Family.deleteFamily(toBytes(family1)); - deleteKey2Family.deleteFamily(toBytes(family2)); - Delete deleteKey3Family = new Delete(toBytes(key3)); - deleteKey3Family.deleteFamily(toBytes(family1)); - deleteKey3Family.deleteFamily(toBytes(family2)); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - - long minTimeStamp = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp1 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp2 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp3 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp4 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp5 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp6 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp7 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp8 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp9 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp10 = System.currentTimeMillis(); - Thread.sleep(5); - long timeStamp11 = System.currentTimeMillis(); - Thread.sleep(5); - long maxTimeStamp = System.currentTimeMillis(); - - Put putKey1Fam1Column1MinTs = new Put(toBytes(key1)); - putKey1Fam1Column1MinTs.add(toBytes(family1), toBytes(column1), minTimeStamp, - toBytes(value1)); - - Put putKey3Fam1Column1Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp1, toBytes(value2)); - - Put putKey1Fam1Column2MinTs = new Put(toBytes(key1)); - putKey1Fam1Column2MinTs.add(toBytes(family1), toBytes(column2), minTimeStamp, - toBytes(value1)); - - Put putKey1Fam1Column2Ts3 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); - - Put putKey2Fam1Column2Ts3 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts3.add(toBytes(family1), toBytes(column2), timeStamp3, toBytes(value2)); - - Put putKey2Fam1Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); - - Put putKey3Fam1Column3Ts1 = new Put(toBytes(key3)); - putKey3Fam1Column3Ts1.add(toBytes(family1), toBytes(column3), timeStamp1, toBytes(value2)); - - Put putKey3Fam1Column2Ts6 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts6.add(toBytes(family1), toBytes(column2), timeStamp6, toBytes(value1)); - - Put putKey2Fam1Column3Ts6 = new Put(toBytes(key2)); - putKey2Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp3, toBytes(value1)); - - tryPut(hTable, putKey1Fam1Column1MinTs); - tryPut(hTable, putKey3Fam1Column1Ts1); - tryPut(hTable, putKey1Fam1Column2MinTs); - tryPut(hTable, putKey1Fam1Column2Ts3); - tryPut(hTable, putKey2Fam1Column2Ts3); - tryPut(hTable, putKey2Fam1Column3Ts1); - tryPut(hTable, putKey3Fam1Column3Ts1); - tryPut(hTable, putKey3Fam1Column2Ts6); - tryPut(hTable, putKey2Fam1Column3Ts6); - - // test DeleteFamilyVersion single cf - Get get = new Get(toBytes(key1)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(minTimeStamp); - get.setMaxVersions(10); - Result r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - get = new Get(toBytes(key3)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp1); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - get = new Get(toBytes(key2)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp3); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(2, r.raw().length); - - Delete delKey1MinTs = new Delete(toBytes(key1)); - delKey1MinTs.deleteFamilyVersion(toBytes(family1), minTimeStamp); - hTable.delete(delKey1MinTs); - - get = new Get(toBytes(key1)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(minTimeStamp); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Delete delKey3Ts1 = new Delete(toBytes(key3)); - delKey3Ts1.deleteFamilyVersion(toBytes(family1), timeStamp1); - hTable.delete(delKey3Ts1); - - get = new Get(toBytes(key3)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp1); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Delete delKey2Ts3 = new Delete(toBytes(key2)); - delKey2Ts3.deleteFamilyVersion(family1.getBytes(), timeStamp3); - hTable.delete(delKey2Ts3); - - get = new Get(toBytes(key2)); - get.addFamily(toBytes(family1)); - get.setTimeStamp(timeStamp3); - get.setMaxVersions(10); - r = hTable.get(get); - Assert.assertEquals(0, r.raw().length); - - Scan scan = new Scan(); - scan.setStartRow(toBytes(key1)); - scan.setStopRow("scanKey4x".getBytes()); - scan.addFamily(toBytes(family1)); - scan.setMaxVersions(10); - ResultScanner scanner = hTable.getScanner(scan); - int key1Cnt = 0, key2Cnt = 0, key3Cnt = 0; - for (Result result : scanner) { - for (KeyValue kv : result.raw()) { - if (key1.equals(Bytes.toString(kv.getRow()))) { - ++key1Cnt; - } else if (key2.equals(Bytes.toString(kv.getRow()))) { - ++key2Cnt; - } else { - ++key3Cnt; - } - } - } - Assert.assertEquals(1, key1Cnt); - Assert.assertEquals(1, key2Cnt); - Assert.assertEquals(1, key3Cnt); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - - // test DeleteFamilyVersion multiple cf - Put putKey1Fam1Column3Ts6 = new Put(toBytes(key1)); - putKey1Fam1Column3Ts6.add(toBytes(family1), toBytes(column3), timeStamp6, toBytes(value3)); - - Put putKey1Fam2Column2Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column2Ts2.add(toBytes(family2), toBytes(column2), timeStamp2, toBytes(value1)); - - Put putKey1Fam2Column3Ts2 = new Put(toBytes(key1)); - putKey1Fam2Column3Ts2.add(toBytes(family2), toBytes(column3), timeStamp2, toBytes(value1)); - - Put putKey1Fam1Column2Ts1 = new Put(toBytes(key1)); - putKey1Fam1Column2Ts1.add(toBytes(family1), toBytes(column2), timeStamp1, toBytes(value2)); - - Put putKey2Fam1Column2Ts8 = new Put(toBytes(key2)); - putKey2Fam1Column2Ts8.add(toBytes(family1), toBytes(column2), timeStamp8, toBytes(value2)); - - Put putKey2Fam2Column3Ts1 = new Put(toBytes(key2)); - putKey2Fam2Column3Ts1.add(toBytes(family2), toBytes(column3), timeStamp3, toBytes(value3)); - - Put putKey2Fam1Column1Ts1 = new Put(toBytes(key2)); - putKey2Fam1Column1Ts1.add(toBytes(family1), toBytes(column1), timeStamp8, toBytes(value1)); - - Put putKey2Fam2Column1Ts3 = new Put(toBytes(key2)); - putKey2Fam2Column1Ts3.add(toBytes(family2), toBytes(column1), timeStamp3, toBytes(value2)); - - Put putKey3Fam1Column2Ts9 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts9.add(toBytes(family1), toBytes(column2), timeStamp9, toBytes(value2)); - - Put putKey3Fam2Column3Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column3Ts10 - .add(toBytes(family2), toBytes(column3), timeStamp10, toBytes(value1)); - - Put putKey3Fam2Column1Ts10 = new Put(toBytes(key3)); - putKey3Fam2Column1Ts10 - .add(toBytes(family2), toBytes(column1), timeStamp10, toBytes(value2)); - - Put putKey3Fam1Column2Ts2 = new Put(toBytes(key3)); - putKey3Fam1Column2Ts2.add(toBytes(family1), toBytes(column2), timeStamp2, toBytes(value1)); - - tryPut(hTable, putKey1Fam1Column3Ts6); - tryPut(hTable, putKey1Fam2Column2Ts2); - tryPut(hTable, putKey1Fam2Column3Ts2); - tryPut(hTable, putKey1Fam1Column2Ts1); - tryPut(hTable, putKey2Fam1Column2Ts8); - tryPut(hTable, putKey2Fam2Column3Ts1); - tryPut(hTable, putKey2Fam1Column1Ts1); - tryPut(hTable, putKey2Fam2Column1Ts3); - tryPut(hTable, putKey3Fam1Column2Ts9); - tryPut(hTable, putKey3Fam2Column3Ts10); - tryPut(hTable, putKey3Fam2Column1Ts10); - tryPut(hTable, putKey3Fam1Column2Ts2); - - Get getKey1 = new Get(toBytes(key1)); - getKey1.addFamily(toBytes(family1)); - getKey1.addFamily(toBytes(family2)); - getKey1.setMaxVersions(10); - r = hTable.get(getKey1); - Assert.assertEquals(4, r.raw().length); - - Get getKey2 = new Get(toBytes(key2)); - getKey2.addFamily(toBytes(family1)); - getKey2.addFamily(toBytes(family2)); - getKey2.setMaxVersions(10); - r = hTable.get(getKey2); - Assert.assertEquals(4, r.raw().length); - - Get getKey3 = new Get(toBytes(key3)); - getKey3.addFamily(toBytes(family1)); - getKey3.addFamily(toBytes(family2)); - getKey3.setMaxVersions(10); - r = hTable.get(getKey3); - Assert.assertEquals(4, r.raw().length); - - Delete delKey1Ts_6_2 = new Delete(toBytes(key1)); - delKey1Ts_6_2.deleteFamilyVersion(toBytes(family1), timeStamp6); - delKey1Ts_6_2.deleteFamilyVersion(toBytes(family2), timeStamp2); - hTable.delete(delKey1Ts_6_2); - - getKey1 = new Get(toBytes(key1)); - getKey1.addFamily(toBytes(family1)); - getKey1.addFamily(toBytes(family2)); - getKey1.setMaxVersions(10); - r = hTable.get(getKey1); - Assert.assertEquals(1, r.raw().length); - for (KeyValue kv : r.raw()) { - Assert.assertEquals(timeStamp1, kv.getTimestamp()); - } - - Delete delKey2Ts_8_3 = new Delete(toBytes(key2)); - delKey2Ts_8_3.deleteFamilyVersion(toBytes(family1), timeStamp8); - delKey2Ts_8_3.deleteFamilyVersion(toBytes(family2), timeStamp3); - hTable.delete(delKey2Ts_8_3); - - getKey2 = new Get(toBytes(key2)); - getKey2.addFamily(toBytes(family1)); - getKey2.addFamily(toBytes(family2)); - getKey2.setMaxVersions(10); - r = hTable.get(getKey2); - Assert.assertEquals(0, r.raw().length); - - Delete delKey3Ts_2_10 = new Delete(toBytes(key3)); - delKey3Ts_2_10.deleteFamilyVersion(toBytes(family1), timeStamp2); - delKey3Ts_2_10.deleteFamilyVersion(toBytes(family2), timeStamp10); - hTable.delete(delKey3Ts_2_10); - - getKey3 = new Get(toBytes(key3)); - getKey3.addFamily(toBytes(family1)); - getKey3.addFamily(toBytes(family2)); - getKey3.setMaxVersions(10); - r = hTable.get(getKey3); - Assert.assertEquals(1, r.raw().length); - for (KeyValue kv : r.raw()) { - Assert.assertEquals(timeStamp9, kv.getTimestamp()); - } - - scan = new Scan(); - scan.setStartRow(toBytes(key1)); - scan.setStopRow("scanKey4x".getBytes()); - scan.addFamily(toBytes(family1)); - scan.addFamily(toBytes(family2)); - scan.setMaxVersions(10); - scanner = hTable.getScanner(scan); - int ts1Cnt = 0, ts9Cnt = 0; - for (Result result : scanner) { - for (KeyValue kv : result.raw()) { - if (kv.getTimestamp() == timeStamp1) { - ++ts1Cnt; - } else if (kv.getTimestamp() == timeStamp9) { - ++ts9Cnt; - } - } - } - Assert.assertEquals(1, ts1Cnt); - Assert.assertEquals(1, ts9Cnt); - - hTable.delete(deleteKey1Family); - hTable.delete(deleteKey2Family); - hTable.delete(deleteKey3Family); - } -} diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java index 4e66fc0b..d639d4a4 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolLoadTest.java @@ -17,15 +17,17 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.rpc.exception.ObTableNotExistException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HTableInterface; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.Executors; import static com.alipay.oceanbase.hbase.constants.OHConstants.*; @@ -34,10 +36,10 @@ import static org.junit.Assert.assertTrue; public class OHTablePoolLoadTest extends HTableTestBase { - private OHTablePool ohTablePool; + private static OHTablePool ohTablePool; - @Before - public void setup() throws IOException { + @BeforeClass + public static void setup() throws Exception { Configuration c = new Configuration(); c.set(HBASE_HTABLE_TEST_LOAD_ENABLE, "true"); ohTablePool = new OHTablePool(c, 10); @@ -56,6 +58,16 @@ public void setup() throws IOException { } ohTablePool.setRuntimeBatchExecutor("test", Executors.newFixedThreadPool(3)); hTable = ohTablePool.getTable("test"); + multiCfHTable = ohTablePool.getTable("test_multi_cf"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -152,4 +164,11 @@ public void testNew() throws IOException { hTable2.close(); assertTrue(true); } + + @AfterClass + public static void finish() throws IOException, SQLException { + hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); + } } diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java index 7ed492d3..d915afdc 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTablePoolTest.java @@ -17,25 +17,26 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.remoting.util.ConcurrentHashSet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.util.PoolMap; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import static com.alipay.oceanbase.hbase.util.ObTableClientManager.OB_TABLE_CLIENT_INSTANCE; public class OHTablePoolTest extends HTableTestBase { - protected OHTablePool ohTablePool; + protected static OHTablePool ohTablePool; - private OHTablePool newOHTablePool(final int maxSize, final PoolMap.PoolType poolType) { + private static OHTablePool newOHTablePool(final int maxSize, final PoolMap.PoolType poolType) { OHTablePool pool = new OHTablePool(new Configuration(), maxSize, poolType); pool.setFullUserName("test", ObHTableTestUtil.FULL_USER_NAME); pool.setPassword("test", ObHTableTestUtil.PASSWORD); @@ -52,17 +53,29 @@ private OHTablePool newOHTablePool(final int maxSize, final PoolMap.PoolType poo return pool; } - @Before - public void setup() throws IOException { + @BeforeClass + public static void setup() throws Exception { Configuration c = new Configuration(); ohTablePool = newOHTablePool(10, null); ohTablePool.setRuntimeBatchExecutor("test", Executors.newFixedThreadPool(3)); hTable = ohTablePool.getTable("test"); + multiCfHTable = ohTablePool.getTable("test_multi_cf"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); + tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } - @After - public void finish() throws IOException { + @AfterClass + public static void finish() throws IOException, SQLException { hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); } public void test_current_get_close(final OHTablePool ohTablePool, int concurrency, int maxSize) { diff --git a/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java b/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java index b14ca220..f36969ca 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java +++ b/src/test/java/com/alipay/oceanbase/hbase/OHTableTest.java @@ -17,16 +17,18 @@ package com.alipay.oceanbase.hbase; +import com.alipay.oceanbase.hbase.util.ObHTableTestUtil; import com.alipay.oceanbase.rpc.ObTableClient; import com.alipay.oceanbase.rpc.exception.ObTableNotExistException; import com.alipay.sofa.common.thread.SofaThreadPoolExecutor; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.HTableInterface; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.*; import java.io.IOException; +import java.sql.SQLException; +import java.util.LinkedList; +import java.util.List; import java.util.concurrent.SynchronousQueue; import static com.alipay.oceanbase.hbase.util.TableHBaseLoggerFactory.TABLE_HBASE_LOGGER_SPACE; @@ -35,12 +37,23 @@ import static org.junit.Assert.fail; public class OHTableTest extends HTableTestBase { - @Before - public void setup() throws IOException { + @BeforeClass + public static void setup() throws Exception { Configuration c = ObHTableTestUtil.newConfiguration(); c.set("rs.list.acquire.read.timeout", "10000"); + hTable = new OHTable(c, "test"); + multiCfHTable = new OHTable(c, "test_multi_cf"); + List tableGroups = new LinkedList<>(); + tableGroups.add("test"); +// tableGroups.add("test_multi_cf"); + ObHTableTestUtil.prepareClean(tableGroups); + } + + @Before + public void prepareCase() { + ObHTableTestUtil.cleanData(); } @Test @@ -93,4 +106,11 @@ public void testNew() throws Exception { } + @AfterClass + public static void finish() throws IOException, SQLException { + hTable.close(); + multiCfHTable.close(); + ObHTableTestUtil.closeConn(); + } + } diff --git a/src/test/java/com/alipay/oceanbase/hbase/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/ObHTableTestUtil.java deleted file mode 100644 index ba9d25f2..00000000 --- a/src/test/java/com/alipay/oceanbase/hbase/ObHTableTestUtil.java +++ /dev/null @@ -1,59 +0,0 @@ -/*- - * #%L - * OBKV HBase Client Framework - * %% - * Copyright (C) 2022 OceanBase Group - * %% - * OBKV HBase Client Framework is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * #L% - */ - -package com.alipay.oceanbase.hbase; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; - -import static com.alipay.oceanbase.hbase.constants.OHConstants.*; - -public class ObHTableTestUtil { - // please consult your dba for the following configuration. - public static String PARAM_URL = ""; - public static String FULL_USER_NAME = ""; - public static String PASSWORD = ""; - public static String SYS_USER_NAME = ""; - public static String SYS_PASSWORD = ""; - public static String ODP_ADDR = ""; - public static int ODP_PORT = 0; - public static boolean ODP_MODE = false; - public static String DATABASE = ""; - - public static Configuration newConfiguration() { - Configuration conf = HBaseConfiguration.create(); - conf.set(HBASE_OCEANBASE_FULL_USER_NAME, FULL_USER_NAME); - conf.set(HBASE_OCEANBASE_PASSWORD, PASSWORD); - if (ODP_MODE) { - // ODP mode - conf.set(HBASE_OCEANBASE_ODP_ADDR, ODP_ADDR); - conf.setInt(HBASE_OCEANBASE_ODP_PORT, ODP_PORT); - conf.setBoolean(HBASE_OCEANBASE_ODP_MODE, ODP_MODE); - conf.set(HBASE_OCEANBASE_DATABASE, DATABASE); - } else { - // OCP mode - conf.set(HBASE_OCEANBASE_PARAM_URL, PARAM_URL); - conf.set(HBASE_OCEANBASE_SYS_USER_NAME, SYS_USER_NAME); - conf.set(HBASE_OCEANBASE_SYS_PASSWORD, SYS_PASSWORD); - } - return conf; - } - - public static OHTableClient newOHTableClient(String tableName) { - return new OHTableClient(tableName, newConfiguration()); - } -} \ No newline at end of file diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/NativeHBaseUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/NativeHBaseUtil.java new file mode 100644 index 00000000..7d6d82b5 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/NativeHBaseUtil.java @@ -0,0 +1,105 @@ +package com.alipay.oceanbase.hbase.util; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Table; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; + +public class NativeHBaseUtil { + public static String SQL_PATH = "src/test/java/unit_test_db.sql"; + public static String MASTER_IP_PORT = ""; + public static String ZK_QUORUM = ""; + public static String ZK_PORT = ""; + + @Test + public void createTable() throws IOException { + Configuration config = new Configuration(); + if (!MASTER_IP_PORT.isEmpty()) { + config.set("hbase.master", MASTER_IP_PORT); + } + config.set("hbase.zookeeper.quorum", ZK_QUORUM); + config.set("hbase.zookeeper.property.clientPort", ZK_PORT); + // 建立连接 + Connection connection = ConnectionFactory.createConnection(config); + Admin admin = connection.getAdmin(); + + // 读取建表语句 + String sql = new String(Files.readAllBytes(Paths.get(SQL_PATH))); + String[] sqlList = sql.split(";"); + Map tableMap = new LinkedHashMap<>(); + for (String singleSql : sqlList) { + String namespace = null; + String realTableName; + String family; + + if (singleSql.contains("CREATE TABLE ")) { + singleSql.trim(); + String[] splits = singleSql.split(" "); + realTableName = splits[2].substring(1, splits[2].length() - 1); + if (realTableName.contains(":")) { + String[] tmpStr = realTableName.split(":", 2); + namespace = tmpStr[0]; + realTableName = tmpStr[1]; + } + String[] tmpStr = realTableName.split("\\$", 2); + realTableName = tmpStr[0]; + family = tmpStr[1]; + HTableDescriptor hTableDescriptor = tableMap.get(realTableName); + if (hTableDescriptor == null) { + hTableDescriptor = new HTableDescriptor(TableName.valueOf(namespace, realTableName)); + tableMap.put(realTableName, hTableDescriptor); + } + if (family != null) { + try { + HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(family); + hColumnDescriptor.setMaxVersions(1000); + hTableDescriptor.addFamily(hColumnDescriptor); + + } catch (Exception e) { + System.out.println("family has exist"); + } + } + // 不分区，结果不会有变化 + } + } + for (Map.Entry entry : tableMap.entrySet()) { + if (admin.tableExists(entry.getValue().getTableName())) { + admin.disableTable(entry.getValue().getTableName()); + admin.deleteTable(entry.getValue().getTableName()); + } + admin.createTable(entry.getValue()); + } + } + + static public Table getTable(TableName tableName) throws IOException { + Configuration config = new Configuration(); + config.set("hbase.master", MASTER_IP_PORT); + config.set("hbase.zookeeper.quorum", ZK_QUORUM); + config.set("hbase.zookeeper.property.clientPort", ZK_PORT); + // 建立连接 + Connection connection = ConnectionFactory.createConnection(config); + return connection.getTable(tableName); + } + + static public Admin getAdmin() throws IOException { + Configuration config = new Configuration(); + config.set("hbase.master", MASTER_IP_PORT); + config.set("hbase.zookeeper.quorum", ZK_QUORUM); + config.set("hbase.zookeeper.property.clientPort", ZK_PORT); + // 建立连接 + Connection connection = ConnectionFactory.createConnection(config); + return connection.getAdmin(); + } + +} diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java index 950ce5db..aaa3c5a9 100644 --- a/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java +++ b/src/test/java/com/alipay/oceanbase/hbase/util/OHTableHotkeyThrottleUtil.java @@ -18,7 +18,6 @@ package com.alipay.oceanbase.hbase.util; import com.alipay.oceanbase.hbase.OHTableClient; -import com.alipay.oceanbase.hbase.ObHTableTestUtil; import com.alipay.oceanbase.rpc.exception.ObTableUnexpectedException; import org.apache.hadoop.hbase.client.*; import org.junit.Assert; diff --git a/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java new file mode 100644 index 00000000..2e5e8f10 --- /dev/null +++ b/src/test/java/com/alipay/oceanbase/hbase/util/ObHTableTestUtil.java @@ -0,0 +1,148 @@ +/*- + * #%L + * OBKV HBase Client Framework + * %% + * Copyright (C) 2022 OceanBase Group + * %% + * OBKV HBase Client Framework is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * #L% + */ + +package com.alipay.oceanbase.hbase.util; + +import com.alipay.oceanbase.hbase.OHTableClient; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HTableDescriptor; + +import java.sql.Connection; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static com.alipay.oceanbase.hbase.constants.OHConstants.*; + +public class ObHTableTestUtil { + // please consult your dba for the following configuration. + public static String PARAM_URL = ""; + public static String FULL_USER_NAME = ""; + public static String PASSWORD = ""; + public static String SYS_USER_NAME = ""; + public static String SYS_PASSWORD = ""; + public static String ODP_ADDR = ""; + public static int ODP_PORT = 0; + public static boolean ODP_MODE = false; + public static String DATABASE = ""; + public static String JDBC_IP = ""; + public static String JDBC_PORT = ""; + public static String JDBC_DATABASE = ""; + public static String JDBC_URL = "jdbc:mysql://" + JDBC_IP + ":" + JDBC_PORT + "/ " + JDBC_DATABASE + "?" + "useUnicode=TRUE&" + "characterEncoding=utf-8&" + "socketTimeout=3000000&" + "connectTimeout=60000"; + + public static String SQL_FORMAT = "truncate %s"; + public static List tableNameList = new LinkedList<>(); + public static Connection conn; + public static Statement stmt; + + public static void prepareClean(List tableGroupList) throws Exception { + for (String tableGroup : tableGroupList) { + tableNameList.addAll(getOTableNameList(tableGroup)); + } + conn = getConnection(); + stmt = conn.createStatement(); + } + + public static void cleanData() { + try { + for (String realTableName : tableNameList) { + try { + stmt.execute(String.format(SQL_FORMAT, realTableName)); + } catch (Exception e) { + System.out.println( + "clean table data error ." + realTableName + " exception:" + e); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static void closeConn() throws SQLException { + stmt.close(); + conn.close(); + } + + public static Configuration newConfiguration() { + Configuration conf = HBaseConfiguration.create(); + conf.set(HBASE_OCEANBASE_FULL_USER_NAME, FULL_USER_NAME); + conf.set(HBASE_OCEANBASE_PASSWORD, PASSWORD); + if (ODP_MODE) { + // ODP mode + conf.set(HBASE_OCEANBASE_ODP_ADDR, ODP_ADDR); + conf.setInt(HBASE_OCEANBASE_ODP_PORT, ODP_PORT); + conf.setBoolean(HBASE_OCEANBASE_ODP_MODE, ODP_MODE); + conf.set(HBASE_OCEANBASE_DATABASE, DATABASE); + } else { + // OCP mode + conf.set(HBASE_OCEANBASE_PARAM_URL, PARAM_URL); + conf.set(HBASE_OCEANBASE_SYS_USER_NAME, SYS_USER_NAME); + conf.set(HBASE_OCEANBASE_SYS_PASSWORD, SYS_PASSWORD); + } + return conf; + } + + public static OHTableClient newOHTableClient(String tableName) { + return new OHTableClient(tableName, newConfiguration()); + } + + static public List getOTableNameList(String tableGroup) throws IOException { + // 读取建表语句 + List res = new LinkedList<>(); + String sql = new String(Files.readAllBytes(Paths.get(NativeHBaseUtil.SQL_PATH))); + String[] sqlList = sql.split(";"); + Map tableMap = new LinkedHashMap<>(); + for (String singleSql : sqlList) { + String realTableName; + if (singleSql.contains("CREATE TABLE ")) { + singleSql.trim(); + String[] splits = singleSql.split(" "); + String tableGroupName = splits[2].substring(1, splits[2].length() - 1); + if (tableGroupName.contains(":")) { + String[] tmpStr = tableGroupName.split(":", 2); + tableGroupName = tmpStr[1]; + } + realTableName = tableGroupName.split("\\$", 2)[0]; + if (realTableName.equals(tableGroup)) { + res.add(tableGroupName); + } + } + } + return res; + } + + static public Connection getConnection() { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + String[] userNames = FULL_USER_NAME.split("#"); + Connection conn = DriverManager.getConnection(JDBC_URL, userNames[0], PASSWORD); + + return conn; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file